1 /*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_inet.h" 32 #include "opt_inet6.h" 33 34 #include <sys/param.h> 35 #include <sys/conf.h> 36 #include <sys/priv.h> 37 #include <sys/kernel.h> 38 #include <sys/bus.h> 39 #include <sys/module.h> 40 #include <sys/malloc.h> 41 #include <sys/queue.h> 42 #include <sys/taskqueue.h> 43 #include <sys/pciio.h> 44 #include <dev/pci/pcireg.h> 45 #include <dev/pci/pcivar.h> 46 #include <dev/pci/pci_private.h> 47 #include <sys/firmware.h> 48 #include <sys/sbuf.h> 49 #include <sys/smp.h> 50 #include <sys/socket.h> 51 #include <sys/sockio.h> 52 #include <sys/sysctl.h> 53 #include <net/ethernet.h> 54 #include <net/if.h> 55 #include <net/if_types.h> 56 #include <net/if_dl.h> 57 #include <net/if_vlan_var.h> 58 #if defined(__i386__) || defined(__amd64__) 59 #include <vm/vm.h> 60 #include <vm/pmap.h> 61 #endif 62 63 #include "common/common.h" 64 #include "common/t4_msg.h" 65 #include "common/t4_regs.h" 66 #include "common/t4_regs_values.h" 67 #include "t4_ioctl.h" 68 #include "t4_l2t.h" 69 #include "t4_mp_ring.h" 70 71 /* T4 bus driver interface */ 72 static int t4_probe(device_t); 73 static int t4_attach(device_t); 74 static int t4_detach(device_t); 75 static device_method_t t4_methods[] = { 76 DEVMETHOD(device_probe, t4_probe), 77 DEVMETHOD(device_attach, t4_attach), 78 DEVMETHOD(device_detach, t4_detach), 79 80 DEVMETHOD_END 81 }; 82 static driver_t t4_driver = { 83 "t4nex", 84 t4_methods, 85 sizeof(struct adapter) 86 }; 87 88 89 /* T4 port (cxgbe) interface */ 90 static int cxgbe_probe(device_t); 91 static int cxgbe_attach(device_t); 92 static int cxgbe_detach(device_t); 93 static device_method_t cxgbe_methods[] = { 94 DEVMETHOD(device_probe, cxgbe_probe), 95 DEVMETHOD(device_attach, cxgbe_attach), 96 DEVMETHOD(device_detach, cxgbe_detach), 97 { 0, 0 } 98 }; 99 static driver_t cxgbe_driver = { 100 "cxgbe", 101 cxgbe_methods, 102 sizeof(struct port_info) 103 }; 104 105 static d_ioctl_t t4_ioctl; 106 static d_open_t t4_open; 107 static d_close_t t4_close; 108 109 static struct cdevsw t4_cdevsw = { 110 .d_version = D_VERSION, 111 .d_flags = 0, 112 .d_open = t4_open, 113 .d_close = t4_close, 114 .d_ioctl = t4_ioctl, 115 .d_name = "t4nex", 116 }; 117 118 /* T5 bus driver interface */ 119 static int t5_probe(device_t); 120 static device_method_t t5_methods[] = { 121 DEVMETHOD(device_probe, t5_probe), 122 DEVMETHOD(device_attach, t4_attach), 123 DEVMETHOD(device_detach, t4_detach), 124 125 DEVMETHOD_END 126 }; 127 static driver_t t5_driver = { 128 "t5nex", 129 t5_methods, 130 sizeof(struct adapter) 131 }; 132 133 134 /* T5 port (cxl) interface */ 135 static driver_t cxl_driver = { 136 "cxl", 137 cxgbe_methods, 138 sizeof(struct port_info) 139 }; 140 141 static struct cdevsw t5_cdevsw = { 142 .d_version = D_VERSION, 143 .d_flags = 0, 144 .d_open = t4_open, 145 .d_close = t4_close, 146 .d_ioctl = t4_ioctl, 147 .d_name = "t5nex", 148 }; 149 150 /* ifnet + media interface */ 151 static void cxgbe_init(void *); 152 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 153 static int cxgbe_transmit(struct ifnet *, struct mbuf *); 154 static void cxgbe_qflush(struct ifnet *); 155 static uint64_t cxgbe_get_counter(struct ifnet *, ift_counter); 156 static int cxgbe_media_change(struct ifnet *); 157 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 158 159 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 160 161 /* 162 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 163 * then ADAPTER_LOCK, then t4_uld_list_lock. 164 */ 165 static struct sx t4_list_lock; 166 SLIST_HEAD(, adapter) t4_list; 167 #ifdef TCP_OFFLOAD 168 static struct sx t4_uld_list_lock; 169 SLIST_HEAD(, uld_info) t4_uld_list; 170 #endif 171 172 /* 173 * Tunables. See tweak_tunables() too. 174 * 175 * Each tunable is set to a default value here if it's known at compile-time. 176 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 177 * provide a reasonable default when the driver is loaded. 178 * 179 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 180 * T5 are under hw.cxl. 181 */ 182 183 /* 184 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 185 */ 186 #define NTXQ_10G 16 187 static int t4_ntxq10g = -1; 188 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 189 190 #define NRXQ_10G 8 191 static int t4_nrxq10g = -1; 192 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 193 194 #define NTXQ_1G 4 195 static int t4_ntxq1g = -1; 196 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 197 198 #define NRXQ_1G 2 199 static int t4_nrxq1g = -1; 200 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 201 202 static int t4_rsrv_noflowq = 0; 203 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq); 204 205 #ifdef TCP_OFFLOAD 206 #define NOFLDTXQ_10G 8 207 static int t4_nofldtxq10g = -1; 208 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 209 210 #define NOFLDRXQ_10G 2 211 static int t4_nofldrxq10g = -1; 212 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 213 214 #define NOFLDTXQ_1G 2 215 static int t4_nofldtxq1g = -1; 216 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 217 218 #define NOFLDRXQ_1G 1 219 static int t4_nofldrxq1g = -1; 220 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 221 #endif 222 223 #ifdef DEV_NETMAP 224 #define NNMTXQ_10G 2 225 static int t4_nnmtxq10g = -1; 226 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g); 227 228 #define NNMRXQ_10G 2 229 static int t4_nnmrxq10g = -1; 230 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g); 231 232 #define NNMTXQ_1G 1 233 static int t4_nnmtxq1g = -1; 234 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g); 235 236 #define NNMRXQ_1G 1 237 static int t4_nnmrxq1g = -1; 238 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g); 239 #endif 240 241 /* 242 * Holdoff parameters for 10G and 1G ports. 243 */ 244 #define TMR_IDX_10G 1 245 static int t4_tmr_idx_10g = TMR_IDX_10G; 246 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 247 248 #define PKTC_IDX_10G (-1) 249 static int t4_pktc_idx_10g = PKTC_IDX_10G; 250 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 251 252 #define TMR_IDX_1G 1 253 static int t4_tmr_idx_1g = TMR_IDX_1G; 254 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 255 256 #define PKTC_IDX_1G (-1) 257 static int t4_pktc_idx_1g = PKTC_IDX_1G; 258 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 259 260 /* 261 * Size (# of entries) of each tx and rx queue. 262 */ 263 static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 264 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 265 266 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 267 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 268 269 /* 270 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 271 */ 272 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 273 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 274 275 /* 276 * Configuration file. 277 */ 278 #define DEFAULT_CF "default" 279 #define FLASH_CF "flash" 280 #define UWIRE_CF "uwire" 281 #define FPGA_CF "fpga" 282 static char t4_cfg_file[32] = DEFAULT_CF; 283 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 284 285 /* 286 * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively). 287 * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them. 288 * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water 289 * mark or when signalled to do so, 0 to never emit PAUSE. 290 */ 291 static int t4_pause_settings = PAUSE_TX | PAUSE_RX; 292 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings); 293 294 /* 295 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 296 * encouraged respectively). 297 */ 298 static unsigned int t4_fw_install = 1; 299 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 300 301 /* 302 * ASIC features that will be used. Disable the ones you don't want so that the 303 * chip resources aren't wasted on features that will not be used. 304 */ 305 static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 306 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 307 308 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 309 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 310 311 static int t4_toecaps_allowed = -1; 312 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 313 314 static int t4_rdmacaps_allowed = 0; 315 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 316 317 static int t4_iscsicaps_allowed = 0; 318 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 319 320 static int t4_fcoecaps_allowed = 0; 321 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 322 323 static int t5_write_combine = 0; 324 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 325 326 struct intrs_and_queues { 327 uint16_t intr_type; /* INTx, MSI, or MSI-X */ 328 uint16_t nirq; /* Total # of vectors */ 329 uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */ 330 uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */ 331 uint16_t ntxq10g; /* # of NIC txq's for each 10G port */ 332 uint16_t nrxq10g; /* # of NIC rxq's for each 10G port */ 333 uint16_t ntxq1g; /* # of NIC txq's for each 1G port */ 334 uint16_t nrxq1g; /* # of NIC rxq's for each 1G port */ 335 uint16_t rsrv_noflowq; /* Flag whether to reserve queue 0 */ 336 #ifdef TCP_OFFLOAD 337 uint16_t nofldtxq10g; /* # of TOE txq's for each 10G port */ 338 uint16_t nofldrxq10g; /* # of TOE rxq's for each 10G port */ 339 uint16_t nofldtxq1g; /* # of TOE txq's for each 1G port */ 340 uint16_t nofldrxq1g; /* # of TOE rxq's for each 1G port */ 341 #endif 342 #ifdef DEV_NETMAP 343 uint16_t nnmtxq10g; /* # of netmap txq's for each 10G port */ 344 uint16_t nnmrxq10g; /* # of netmap rxq's for each 10G port */ 345 uint16_t nnmtxq1g; /* # of netmap txq's for each 1G port */ 346 uint16_t nnmrxq1g; /* # of netmap rxq's for each 1G port */ 347 #endif 348 }; 349 350 struct filter_entry { 351 uint32_t valid:1; /* filter allocated and valid */ 352 uint32_t locked:1; /* filter is administratively locked */ 353 uint32_t pending:1; /* filter action is pending firmware reply */ 354 uint32_t smtidx:8; /* Source MAC Table index for smac */ 355 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 356 357 struct t4_filter_specification fs; 358 }; 359 360 static int map_bars_0_and_4(struct adapter *); 361 static int map_bar_2(struct adapter *); 362 static void setup_memwin(struct adapter *); 363 static int validate_mem_range(struct adapter *, uint32_t, int); 364 static int fwmtype_to_hwmtype(int); 365 static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 366 uint32_t *); 367 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 368 static uint32_t position_memwin(struct adapter *, int, uint32_t); 369 static int cfg_itype_and_nqueues(struct adapter *, int, int, 370 struct intrs_and_queues *); 371 static int prep_firmware(struct adapter *); 372 static int partition_resources(struct adapter *, const struct firmware *, 373 const char *); 374 static int get_params__pre_init(struct adapter *); 375 static int get_params__post_init(struct adapter *); 376 static int set_params__post_init(struct adapter *); 377 static void t4_set_desc(struct adapter *); 378 static void build_medialist(struct port_info *, struct ifmedia *); 379 static int cxgbe_init_synchronized(struct port_info *); 380 static int cxgbe_uninit_synchronized(struct port_info *); 381 static int setup_intr_handlers(struct adapter *); 382 static void quiesce_txq(struct adapter *, struct sge_txq *); 383 static void quiesce_wrq(struct adapter *, struct sge_wrq *); 384 static void quiesce_iq(struct adapter *, struct sge_iq *); 385 static void quiesce_fl(struct adapter *, struct sge_fl *); 386 static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 387 driver_intr_t *, void *, char *); 388 static int t4_free_irq(struct adapter *, struct irq *); 389 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 390 unsigned int); 391 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 392 static void cxgbe_refresh_stats(struct adapter *, struct port_info *); 393 static void cxgbe_tick(void *); 394 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 395 static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 396 struct mbuf *); 397 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 398 static int fw_msg_not_handled(struct adapter *, const __be64 *); 399 static int t4_sysctls(struct adapter *); 400 static int cxgbe_sysctls(struct port_info *); 401 static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 402 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 403 static int sysctl_btphy(SYSCTL_HANDLER_ARGS); 404 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); 405 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 406 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 407 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 408 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 409 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS); 410 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 411 static int sysctl_temperature(SYSCTL_HANDLER_ARGS); 412 #ifdef SBUF_DRAIN 413 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 414 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 415 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 416 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); 417 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); 418 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 419 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 420 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 421 static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 422 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 423 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 424 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 425 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); 426 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 427 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); 428 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 429 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 430 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 431 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 432 static int sysctl_tids(SYSCTL_HANDLER_ARGS); 433 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 434 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 435 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 436 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 437 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 438 #endif 439 static uint32_t fconf_to_mode(uint32_t); 440 static uint32_t mode_to_fconf(uint32_t); 441 static uint32_t fspec_to_fconf(struct t4_filter_specification *); 442 static int get_filter_mode(struct adapter *, uint32_t *); 443 static int set_filter_mode(struct adapter *, uint32_t); 444 static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 445 static int get_filter(struct adapter *, struct t4_filter *); 446 static int set_filter(struct adapter *, struct t4_filter *); 447 static int del_filter(struct adapter *, struct t4_filter *); 448 static void clear_filter(struct filter_entry *); 449 static int set_filter_wr(struct adapter *, int); 450 static int del_filter_wr(struct adapter *, int); 451 static int get_sge_context(struct adapter *, struct t4_sge_context *); 452 static int load_fw(struct adapter *, struct t4_data *); 453 static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 454 static int read_i2c(struct adapter *, struct t4_i2c_data *); 455 static int set_sched_class(struct adapter *, struct t4_sched_params *); 456 static int set_sched_queue(struct adapter *, struct t4_sched_queue *); 457 #ifdef TCP_OFFLOAD 458 static int toe_capability(struct port_info *, int); 459 #endif 460 static int mod_event(module_t, int, void *); 461 462 struct { 463 uint16_t device; 464 char *desc; 465 } t4_pciids[] = { 466 {0xa000, "Chelsio Terminator 4 FPGA"}, 467 {0x4400, "Chelsio T440-dbg"}, 468 {0x4401, "Chelsio T420-CR"}, 469 {0x4402, "Chelsio T422-CR"}, 470 {0x4403, "Chelsio T440-CR"}, 471 {0x4404, "Chelsio T420-BCH"}, 472 {0x4405, "Chelsio T440-BCH"}, 473 {0x4406, "Chelsio T440-CH"}, 474 {0x4407, "Chelsio T420-SO"}, 475 {0x4408, "Chelsio T420-CX"}, 476 {0x4409, "Chelsio T420-BT"}, 477 {0x440a, "Chelsio T404-BT"}, 478 {0x440e, "Chelsio T440-LP-CR"}, 479 }, t5_pciids[] = { 480 {0xb000, "Chelsio Terminator 5 FPGA"}, 481 {0x5400, "Chelsio T580-dbg"}, 482 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ 483 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ 484 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ 485 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ 486 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ 487 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ 488 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ 489 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ 490 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 491 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ 492 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ 493 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ 494 {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */ 495 #ifdef notyet 496 {0x5404, "Chelsio T520-BCH"}, 497 {0x5405, "Chelsio T540-BCH"}, 498 {0x5406, "Chelsio T540-CH"}, 499 {0x5408, "Chelsio T520-CX"}, 500 {0x540b, "Chelsio B520-SR"}, 501 {0x540c, "Chelsio B504-BT"}, 502 {0x540f, "Chelsio Amsterdam"}, 503 {0x5413, "Chelsio T580-CHR"}, 504 #endif 505 }; 506 507 #ifdef TCP_OFFLOAD 508 /* 509 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 510 * exactly the same for both rxq and ofld_rxq. 511 */ 512 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 513 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 514 #endif 515 516 /* No easy way to include t4_msg.h before adapter.h so we check this way */ 517 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 518 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 519 520 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); 521 522 static int 523 t4_probe(device_t dev) 524 { 525 int i; 526 uint16_t v = pci_get_vendor(dev); 527 uint16_t d = pci_get_device(dev); 528 uint8_t f = pci_get_function(dev); 529 530 if (v != PCI_VENDOR_ID_CHELSIO) 531 return (ENXIO); 532 533 /* Attach only to PF0 of the FPGA */ 534 if (d == 0xa000 && f != 0) 535 return (ENXIO); 536 537 for (i = 0; i < nitems(t4_pciids); i++) { 538 if (d == t4_pciids[i].device) { 539 device_set_desc(dev, t4_pciids[i].desc); 540 return (BUS_PROBE_DEFAULT); 541 } 542 } 543 544 return (ENXIO); 545 } 546 547 static int 548 t5_probe(device_t dev) 549 { 550 int i; 551 uint16_t v = pci_get_vendor(dev); 552 uint16_t d = pci_get_device(dev); 553 uint8_t f = pci_get_function(dev); 554 555 if (v != PCI_VENDOR_ID_CHELSIO) 556 return (ENXIO); 557 558 /* Attach only to PF0 of the FPGA */ 559 if (d == 0xb000 && f != 0) 560 return (ENXIO); 561 562 for (i = 0; i < nitems(t5_pciids); i++) { 563 if (d == t5_pciids[i].device) { 564 device_set_desc(dev, t5_pciids[i].desc); 565 return (BUS_PROBE_DEFAULT); 566 } 567 } 568 569 return (ENXIO); 570 } 571 572 static int 573 t4_attach(device_t dev) 574 { 575 struct adapter *sc; 576 int rc = 0, i, n10g, n1g, rqidx, tqidx; 577 struct intrs_and_queues iaq; 578 struct sge *s; 579 #ifdef TCP_OFFLOAD 580 int ofld_rqidx, ofld_tqidx; 581 #endif 582 #ifdef DEV_NETMAP 583 int nm_rqidx, nm_tqidx; 584 #endif 585 586 sc = device_get_softc(dev); 587 sc->dev = dev; 588 #ifdef INVARIANTS 589 sc->debug_flags = DF_DUMP_MBOX; 590 #endif 591 592 pci_enable_busmaster(dev); 593 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 594 uint32_t v; 595 596 pci_set_max_read_req(dev, 4096); 597 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 598 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 599 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 600 601 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5); 602 } 603 604 sc->traceq = -1; 605 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); 606 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", 607 device_get_nameunit(dev)); 608 609 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 610 device_get_nameunit(dev)); 611 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 612 sx_xlock(&t4_list_lock); 613 SLIST_INSERT_HEAD(&t4_list, sc, link); 614 sx_xunlock(&t4_list_lock); 615 616 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 617 TAILQ_INIT(&sc->sfl); 618 callout_init(&sc->sfl_callout, 1); 619 620 mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF); 621 622 rc = map_bars_0_and_4(sc); 623 if (rc != 0) 624 goto done; /* error message displayed already */ 625 626 /* 627 * This is the real PF# to which we're attaching. Works from within PCI 628 * passthrough environments too, where pci_get_function() could return a 629 * different PF# depending on the passthrough configuration. We need to 630 * use the real PF# in all our communication with the firmware. 631 */ 632 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 633 sc->mbox = sc->pf; 634 635 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 636 sc->an_handler = an_not_handled; 637 for (i = 0; i < nitems(sc->cpl_handler); i++) 638 sc->cpl_handler[i] = cpl_not_handled; 639 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 640 sc->fw_msg_handler[i] = fw_msg_not_handled; 641 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 642 t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt); 643 t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt); 644 t4_init_sge_cpl_handlers(sc); 645 646 /* Prepare the adapter for operation */ 647 rc = -t4_prep_adapter(sc); 648 if (rc != 0) { 649 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 650 goto done; 651 } 652 653 /* 654 * Do this really early, with the memory windows set up even before the 655 * character device. The userland tool's register i/o and mem read 656 * will work even in "recovery mode". 657 */ 658 setup_memwin(sc); 659 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 660 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 661 device_get_nameunit(dev)); 662 if (sc->cdev == NULL) 663 device_printf(dev, "failed to create nexus char device.\n"); 664 else 665 sc->cdev->si_drv1 = sc; 666 667 /* Go no further if recovery mode has been requested. */ 668 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 669 device_printf(dev, "recovery mode.\n"); 670 goto done; 671 } 672 673 #if defined(__i386__) 674 if ((cpu_feature & CPUID_CX8) == 0) { 675 device_printf(dev, "64 bit atomics not available.\n"); 676 rc = ENOTSUP; 677 goto done; 678 } 679 #endif 680 681 /* Prepare the firmware for operation */ 682 rc = prep_firmware(sc); 683 if (rc != 0) 684 goto done; /* error message displayed already */ 685 686 rc = get_params__post_init(sc); 687 if (rc != 0) 688 goto done; /* error message displayed already */ 689 690 rc = set_params__post_init(sc); 691 if (rc != 0) 692 goto done; /* error message displayed already */ 693 694 rc = map_bar_2(sc); 695 if (rc != 0) 696 goto done; /* error message displayed already */ 697 698 rc = t4_create_dma_tag(sc); 699 if (rc != 0) 700 goto done; /* error message displayed already */ 701 702 /* 703 * First pass over all the ports - allocate VIs and initialize some 704 * basic parameters like mac address, port type, etc. We also figure 705 * out whether a port is 10G or 1G and use that information when 706 * calculating how many interrupts to attempt to allocate. 707 */ 708 n10g = n1g = 0; 709 for_each_port(sc, i) { 710 struct port_info *pi; 711 712 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 713 sc->port[i] = pi; 714 715 /* These must be set before t4_port_init */ 716 pi->adapter = sc; 717 pi->port_id = i; 718 719 /* Allocate the vi and initialize parameters like mac addr */ 720 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 721 if (rc != 0) { 722 device_printf(dev, "unable to initialize port %d: %d\n", 723 i, rc); 724 free(pi, M_CXGBE); 725 sc->port[i] = NULL; 726 goto done; 727 } 728 729 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX); 730 pi->link_cfg.requested_fc |= t4_pause_settings; 731 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX); 732 pi->link_cfg.fc |= t4_pause_settings; 733 734 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 735 if (rc != 0) { 736 device_printf(dev, "port %d l1cfg failed: %d\n", i, rc); 737 free(pi, M_CXGBE); 738 sc->port[i] = NULL; 739 goto done; 740 } 741 742 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 743 device_get_nameunit(dev), i); 744 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 745 sc->chan_map[pi->tx_chan] = i; 746 747 if (is_10G_port(pi) || is_40G_port(pi)) { 748 n10g++; 749 pi->tmr_idx = t4_tmr_idx_10g; 750 pi->pktc_idx = t4_pktc_idx_10g; 751 } else { 752 n1g++; 753 pi->tmr_idx = t4_tmr_idx_1g; 754 pi->pktc_idx = t4_pktc_idx_1g; 755 } 756 757 pi->xact_addr_filt = -1; 758 pi->linkdnrc = -1; 759 760 pi->qsize_rxq = t4_qsize_rxq; 761 pi->qsize_txq = t4_qsize_txq; 762 763 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 764 if (pi->dev == NULL) { 765 device_printf(dev, 766 "failed to add device for port %d.\n", i); 767 rc = ENXIO; 768 goto done; 769 } 770 device_set_softc(pi->dev, pi); 771 } 772 773 /* 774 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 775 */ 776 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 777 if (rc != 0) 778 goto done; /* error message displayed already */ 779 780 sc->intr_type = iaq.intr_type; 781 sc->intr_count = iaq.nirq; 782 783 s = &sc->sge; 784 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 785 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 786 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 787 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 788 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 789 #ifdef TCP_OFFLOAD 790 if (is_offload(sc)) { 791 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 792 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 793 s->neq += s->nofldtxq + s->nofldrxq; 794 s->niq += s->nofldrxq; 795 796 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 797 M_CXGBE, M_ZERO | M_WAITOK); 798 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 799 M_CXGBE, M_ZERO | M_WAITOK); 800 } 801 #endif 802 #ifdef DEV_NETMAP 803 s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g; 804 s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g; 805 s->neq += s->nnmtxq + s->nnmrxq; 806 s->niq += s->nnmrxq; 807 808 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq), 809 M_CXGBE, M_ZERO | M_WAITOK); 810 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq), 811 M_CXGBE, M_ZERO | M_WAITOK); 812 #endif 813 814 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 815 M_ZERO | M_WAITOK); 816 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 817 M_ZERO | M_WAITOK); 818 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 819 M_ZERO | M_WAITOK); 820 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 821 M_ZERO | M_WAITOK); 822 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 823 M_ZERO | M_WAITOK); 824 825 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 826 M_ZERO | M_WAITOK); 827 828 t4_init_l2t(sc, M_WAITOK); 829 830 /* 831 * Second pass over the ports. This time we know the number of rx and 832 * tx queues that each port should get. 833 */ 834 rqidx = tqidx = 0; 835 #ifdef TCP_OFFLOAD 836 ofld_rqidx = ofld_tqidx = 0; 837 #endif 838 #ifdef DEV_NETMAP 839 nm_rqidx = nm_tqidx = 0; 840 #endif 841 for_each_port(sc, i) { 842 struct port_info *pi = sc->port[i]; 843 844 if (pi == NULL) 845 continue; 846 847 pi->first_rxq = rqidx; 848 pi->first_txq = tqidx; 849 if (is_10G_port(pi) || is_40G_port(pi)) { 850 pi->flags |= iaq.intr_flags_10g; 851 pi->nrxq = iaq.nrxq10g; 852 pi->ntxq = iaq.ntxq10g; 853 } else { 854 pi->flags |= iaq.intr_flags_1g; 855 pi->nrxq = iaq.nrxq1g; 856 pi->ntxq = iaq.ntxq1g; 857 } 858 859 if (pi->ntxq > 1) 860 pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0; 861 else 862 pi->rsrv_noflowq = 0; 863 864 rqidx += pi->nrxq; 865 tqidx += pi->ntxq; 866 #ifdef TCP_OFFLOAD 867 if (is_offload(sc)) { 868 pi->first_ofld_rxq = ofld_rqidx; 869 pi->first_ofld_txq = ofld_tqidx; 870 if (is_10G_port(pi) || is_40G_port(pi)) { 871 pi->nofldrxq = iaq.nofldrxq10g; 872 pi->nofldtxq = iaq.nofldtxq10g; 873 } else { 874 pi->nofldrxq = iaq.nofldrxq1g; 875 pi->nofldtxq = iaq.nofldtxq1g; 876 } 877 ofld_rqidx += pi->nofldrxq; 878 ofld_tqidx += pi->nofldtxq; 879 } 880 #endif 881 #ifdef DEV_NETMAP 882 pi->first_nm_rxq = nm_rqidx; 883 pi->first_nm_txq = nm_tqidx; 884 if (is_10G_port(pi) || is_40G_port(pi)) { 885 pi->nnmrxq = iaq.nnmrxq10g; 886 pi->nnmtxq = iaq.nnmtxq10g; 887 } else { 888 pi->nnmrxq = iaq.nnmrxq1g; 889 pi->nnmtxq = iaq.nnmtxq1g; 890 } 891 nm_rqidx += pi->nnmrxq; 892 nm_tqidx += pi->nnmtxq; 893 #endif 894 } 895 896 rc = setup_intr_handlers(sc); 897 if (rc != 0) { 898 device_printf(dev, 899 "failed to setup interrupt handlers: %d\n", rc); 900 goto done; 901 } 902 903 rc = bus_generic_attach(dev); 904 if (rc != 0) { 905 device_printf(dev, 906 "failed to attach all child ports: %d\n", rc); 907 goto done; 908 } 909 910 device_printf(dev, 911 "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 912 sc->params.pci.speed, sc->params.pci.width, sc->params.nports, 913 sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" : 914 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 915 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 916 917 t4_set_desc(sc); 918 919 done: 920 if (rc != 0 && sc->cdev) { 921 /* cdev was created and so cxgbetool works; recover that way. */ 922 device_printf(dev, 923 "error during attach, adapter is now in recovery mode.\n"); 924 rc = 0; 925 } 926 927 if (rc != 0) 928 t4_detach(dev); 929 else 930 t4_sysctls(sc); 931 932 return (rc); 933 } 934 935 /* 936 * Idempotent 937 */ 938 static int 939 t4_detach(device_t dev) 940 { 941 struct adapter *sc; 942 struct port_info *pi; 943 int i, rc; 944 945 sc = device_get_softc(dev); 946 947 if (sc->flags & FULL_INIT_DONE) 948 t4_intr_disable(sc); 949 950 if (sc->cdev) { 951 destroy_dev(sc->cdev); 952 sc->cdev = NULL; 953 } 954 955 rc = bus_generic_detach(dev); 956 if (rc) { 957 device_printf(dev, 958 "failed to detach child devices: %d\n", rc); 959 return (rc); 960 } 961 962 for (i = 0; i < sc->intr_count; i++) 963 t4_free_irq(sc, &sc->irq[i]); 964 965 for (i = 0; i < MAX_NPORTS; i++) { 966 pi = sc->port[i]; 967 if (pi) { 968 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid); 969 if (pi->dev) 970 device_delete_child(dev, pi->dev); 971 972 mtx_destroy(&pi->pi_lock); 973 free(pi, M_CXGBE); 974 } 975 } 976 977 if (sc->flags & FULL_INIT_DONE) 978 adapter_full_uninit(sc); 979 980 if (sc->flags & FW_OK) 981 t4_fw_bye(sc, sc->mbox); 982 983 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 984 pci_release_msi(dev); 985 986 if (sc->regs_res) 987 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 988 sc->regs_res); 989 990 if (sc->udbs_res) 991 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 992 sc->udbs_res); 993 994 if (sc->msix_res) 995 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 996 sc->msix_res); 997 998 if (sc->l2t) 999 t4_free_l2t(sc->l2t); 1000 1001 #ifdef TCP_OFFLOAD 1002 free(sc->sge.ofld_rxq, M_CXGBE); 1003 free(sc->sge.ofld_txq, M_CXGBE); 1004 #endif 1005 #ifdef DEV_NETMAP 1006 free(sc->sge.nm_rxq, M_CXGBE); 1007 free(sc->sge.nm_txq, M_CXGBE); 1008 #endif 1009 free(sc->irq, M_CXGBE); 1010 free(sc->sge.rxq, M_CXGBE); 1011 free(sc->sge.txq, M_CXGBE); 1012 free(sc->sge.ctrlq, M_CXGBE); 1013 free(sc->sge.iqmap, M_CXGBE); 1014 free(sc->sge.eqmap, M_CXGBE); 1015 free(sc->tids.ftid_tab, M_CXGBE); 1016 t4_destroy_dma_tag(sc); 1017 if (mtx_initialized(&sc->sc_lock)) { 1018 sx_xlock(&t4_list_lock); 1019 SLIST_REMOVE(&t4_list, sc, adapter, link); 1020 sx_xunlock(&t4_list_lock); 1021 mtx_destroy(&sc->sc_lock); 1022 } 1023 1024 if (mtx_initialized(&sc->tids.ftid_lock)) 1025 mtx_destroy(&sc->tids.ftid_lock); 1026 if (mtx_initialized(&sc->sfl_lock)) 1027 mtx_destroy(&sc->sfl_lock); 1028 if (mtx_initialized(&sc->ifp_lock)) 1029 mtx_destroy(&sc->ifp_lock); 1030 if (mtx_initialized(&sc->regwin_lock)) 1031 mtx_destroy(&sc->regwin_lock); 1032 1033 bzero(sc, sizeof(*sc)); 1034 1035 return (0); 1036 } 1037 1038 static int 1039 cxgbe_probe(device_t dev) 1040 { 1041 char buf[128]; 1042 struct port_info *pi = device_get_softc(dev); 1043 1044 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 1045 device_set_desc_copy(dev, buf); 1046 1047 return (BUS_PROBE_DEFAULT); 1048 } 1049 1050 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 1051 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 1052 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS) 1053 #define T4_CAP_ENABLE (T4_CAP) 1054 1055 static int 1056 cxgbe_attach(device_t dev) 1057 { 1058 struct port_info *pi = device_get_softc(dev); 1059 struct ifnet *ifp; 1060 char *s; 1061 int n, o; 1062 1063 /* Allocate an ifnet and set it up */ 1064 ifp = if_alloc(IFT_ETHER); 1065 if (ifp == NULL) { 1066 device_printf(dev, "Cannot allocate ifnet\n"); 1067 return (ENOMEM); 1068 } 1069 pi->ifp = ifp; 1070 ifp->if_softc = pi; 1071 1072 callout_init(&pi->tick, 1); 1073 1074 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1075 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1076 1077 ifp->if_init = cxgbe_init; 1078 ifp->if_ioctl = cxgbe_ioctl; 1079 ifp->if_transmit = cxgbe_transmit; 1080 ifp->if_qflush = cxgbe_qflush; 1081 ifp->if_get_counter = cxgbe_get_counter; 1082 1083 ifp->if_capabilities = T4_CAP; 1084 #ifdef TCP_OFFLOAD 1085 if (is_offload(pi->adapter)) 1086 ifp->if_capabilities |= IFCAP_TOE; 1087 #endif 1088 ifp->if_capenable = T4_CAP_ENABLE; 1089 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1090 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1091 1092 ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1093 ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS; 1094 ifp->if_hw_tsomaxsegsize = 65536; 1095 1096 /* Initialize ifmedia for this port */ 1097 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 1098 cxgbe_media_status); 1099 build_medialist(pi, &pi->media); 1100 1101 pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 1102 EVENTHANDLER_PRI_ANY); 1103 1104 ether_ifattach(ifp, pi->hw_addr); 1105 1106 n = 128; 1107 s = malloc(n, M_CXGBE, M_WAITOK); 1108 o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq); 1109 MPASS(n > o); 1110 #ifdef TCP_OFFLOAD 1111 if (is_offload(pi->adapter)) { 1112 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)", 1113 pi->nofldtxq, pi->nofldrxq); 1114 MPASS(n > o); 1115 } 1116 #endif 1117 #ifdef DEV_NETMAP 1118 o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq, 1119 pi->nnmrxq); 1120 MPASS(n > o); 1121 #endif 1122 device_printf(dev, "%s\n", s); 1123 free(s, M_CXGBE); 1124 1125 #ifdef DEV_NETMAP 1126 /* nm_media handled here to keep implementation private to this file */ 1127 ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change, 1128 cxgbe_media_status); 1129 build_medialist(pi, &pi->nm_media); 1130 create_netmap_ifnet(pi); /* logs errors it something fails */ 1131 #endif 1132 cxgbe_sysctls(pi); 1133 1134 return (0); 1135 } 1136 1137 static int 1138 cxgbe_detach(device_t dev) 1139 { 1140 struct port_info *pi = device_get_softc(dev); 1141 struct adapter *sc = pi->adapter; 1142 struct ifnet *ifp = pi->ifp; 1143 1144 /* Tell if_ioctl and if_init that the port is going away */ 1145 ADAPTER_LOCK(sc); 1146 SET_DOOMED(pi); 1147 wakeup(&sc->flags); 1148 while (IS_BUSY(sc)) 1149 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 1150 SET_BUSY(sc); 1151 #ifdef INVARIANTS 1152 sc->last_op = "t4detach"; 1153 sc->last_op_thr = curthread; 1154 #endif 1155 ADAPTER_UNLOCK(sc); 1156 1157 if (pi->flags & HAS_TRACEQ) { 1158 sc->traceq = -1; /* cloner should not create ifnet */ 1159 t4_tracer_port_detach(sc); 1160 } 1161 1162 if (pi->vlan_c) 1163 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c); 1164 1165 PORT_LOCK(pi); 1166 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1167 callout_stop(&pi->tick); 1168 PORT_UNLOCK(pi); 1169 callout_drain(&pi->tick); 1170 1171 /* Let detach proceed even if these fail. */ 1172 cxgbe_uninit_synchronized(pi); 1173 port_full_uninit(pi); 1174 1175 ifmedia_removeall(&pi->media); 1176 ether_ifdetach(pi->ifp); 1177 if_free(pi->ifp); 1178 1179 #ifdef DEV_NETMAP 1180 /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */ 1181 destroy_netmap_ifnet(pi); 1182 #endif 1183 1184 ADAPTER_LOCK(sc); 1185 CLR_BUSY(sc); 1186 wakeup(&sc->flags); 1187 ADAPTER_UNLOCK(sc); 1188 1189 return (0); 1190 } 1191 1192 static void 1193 cxgbe_init(void *arg) 1194 { 1195 struct port_info *pi = arg; 1196 struct adapter *sc = pi->adapter; 1197 1198 if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0) 1199 return; 1200 cxgbe_init_synchronized(pi); 1201 end_synchronized_op(sc, 0); 1202 } 1203 1204 static int 1205 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1206 { 1207 int rc = 0, mtu, flags, can_sleep; 1208 struct port_info *pi = ifp->if_softc; 1209 struct adapter *sc = pi->adapter; 1210 struct ifreq *ifr = (struct ifreq *)data; 1211 uint32_t mask; 1212 1213 switch (cmd) { 1214 case SIOCSIFMTU: 1215 mtu = ifr->ifr_mtu; 1216 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1217 return (EINVAL); 1218 1219 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu"); 1220 if (rc) 1221 return (rc); 1222 ifp->if_mtu = mtu; 1223 if (pi->flags & PORT_INIT_DONE) { 1224 t4_update_fl_bufsize(ifp); 1225 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1226 rc = update_mac_settings(ifp, XGMAC_MTU); 1227 } 1228 end_synchronized_op(sc, 0); 1229 break; 1230 1231 case SIOCSIFFLAGS: 1232 can_sleep = 0; 1233 redo_sifflags: 1234 rc = begin_synchronized_op(sc, pi, 1235 can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg"); 1236 if (rc) 1237 return (rc); 1238 1239 if (ifp->if_flags & IFF_UP) { 1240 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1241 flags = pi->if_flags; 1242 if ((ifp->if_flags ^ flags) & 1243 (IFF_PROMISC | IFF_ALLMULTI)) { 1244 if (can_sleep == 1) { 1245 end_synchronized_op(sc, 0); 1246 can_sleep = 0; 1247 goto redo_sifflags; 1248 } 1249 rc = update_mac_settings(ifp, 1250 XGMAC_PROMISC | XGMAC_ALLMULTI); 1251 } 1252 } else { 1253 if (can_sleep == 0) { 1254 end_synchronized_op(sc, LOCK_HELD); 1255 can_sleep = 1; 1256 goto redo_sifflags; 1257 } 1258 rc = cxgbe_init_synchronized(pi); 1259 } 1260 pi->if_flags = ifp->if_flags; 1261 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1262 if (can_sleep == 0) { 1263 end_synchronized_op(sc, LOCK_HELD); 1264 can_sleep = 1; 1265 goto redo_sifflags; 1266 } 1267 rc = cxgbe_uninit_synchronized(pi); 1268 } 1269 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD); 1270 break; 1271 1272 case SIOCADDMULTI: 1273 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1274 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi"); 1275 if (rc) 1276 return (rc); 1277 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1278 rc = update_mac_settings(ifp, XGMAC_MCADDRS); 1279 end_synchronized_op(sc, LOCK_HELD); 1280 break; 1281 1282 case SIOCSIFCAP: 1283 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap"); 1284 if (rc) 1285 return (rc); 1286 1287 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1288 if (mask & IFCAP_TXCSUM) { 1289 ifp->if_capenable ^= IFCAP_TXCSUM; 1290 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1291 1292 if (IFCAP_TSO4 & ifp->if_capenable && 1293 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1294 ifp->if_capenable &= ~IFCAP_TSO4; 1295 if_printf(ifp, 1296 "tso4 disabled due to -txcsum.\n"); 1297 } 1298 } 1299 if (mask & IFCAP_TXCSUM_IPV6) { 1300 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1301 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1302 1303 if (IFCAP_TSO6 & ifp->if_capenable && 1304 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1305 ifp->if_capenable &= ~IFCAP_TSO6; 1306 if_printf(ifp, 1307 "tso6 disabled due to -txcsum6.\n"); 1308 } 1309 } 1310 if (mask & IFCAP_RXCSUM) 1311 ifp->if_capenable ^= IFCAP_RXCSUM; 1312 if (mask & IFCAP_RXCSUM_IPV6) 1313 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1314 1315 /* 1316 * Note that we leave CSUM_TSO alone (it is always set). The 1317 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1318 * sending a TSO request our way, so it's sufficient to toggle 1319 * IFCAP_TSOx only. 1320 */ 1321 if (mask & IFCAP_TSO4) { 1322 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1323 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1324 if_printf(ifp, "enable txcsum first.\n"); 1325 rc = EAGAIN; 1326 goto fail; 1327 } 1328 ifp->if_capenable ^= IFCAP_TSO4; 1329 } 1330 if (mask & IFCAP_TSO6) { 1331 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1332 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1333 if_printf(ifp, "enable txcsum6 first.\n"); 1334 rc = EAGAIN; 1335 goto fail; 1336 } 1337 ifp->if_capenable ^= IFCAP_TSO6; 1338 } 1339 if (mask & IFCAP_LRO) { 1340 #if defined(INET) || defined(INET6) 1341 int i; 1342 struct sge_rxq *rxq; 1343 1344 ifp->if_capenable ^= IFCAP_LRO; 1345 for_each_rxq(pi, i, rxq) { 1346 if (ifp->if_capenable & IFCAP_LRO) 1347 rxq->iq.flags |= IQ_LRO_ENABLED; 1348 else 1349 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1350 } 1351 #endif 1352 } 1353 #ifdef TCP_OFFLOAD 1354 if (mask & IFCAP_TOE) { 1355 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1356 1357 rc = toe_capability(pi, enable); 1358 if (rc != 0) 1359 goto fail; 1360 1361 ifp->if_capenable ^= mask; 1362 } 1363 #endif 1364 if (mask & IFCAP_VLAN_HWTAGGING) { 1365 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1366 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1367 rc = update_mac_settings(ifp, XGMAC_VLANEX); 1368 } 1369 if (mask & IFCAP_VLAN_MTU) { 1370 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1371 1372 /* Need to find out how to disable auto-mtu-inflation */ 1373 } 1374 if (mask & IFCAP_VLAN_HWTSO) 1375 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1376 if (mask & IFCAP_VLAN_HWCSUM) 1377 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1378 1379 #ifdef VLAN_CAPABILITIES 1380 VLAN_CAPABILITIES(ifp); 1381 #endif 1382 fail: 1383 end_synchronized_op(sc, 0); 1384 break; 1385 1386 case SIOCSIFMEDIA: 1387 case SIOCGIFMEDIA: 1388 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1389 break; 1390 1391 case SIOCGI2C: { 1392 struct ifi2creq i2c; 1393 1394 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); 1395 if (rc != 0) 1396 break; 1397 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) { 1398 rc = EPERM; 1399 break; 1400 } 1401 if (i2c.len > sizeof(i2c.data)) { 1402 rc = EINVAL; 1403 break; 1404 } 1405 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4i2c"); 1406 if (rc) 1407 return (rc); 1408 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr, 1409 i2c.offset, i2c.len, &i2c.data[0]); 1410 end_synchronized_op(sc, 0); 1411 if (rc == 0) 1412 rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); 1413 break; 1414 } 1415 1416 default: 1417 rc = ether_ioctl(ifp, cmd, data); 1418 } 1419 1420 return (rc); 1421 } 1422 1423 static int 1424 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1425 { 1426 struct port_info *pi = ifp->if_softc; 1427 struct adapter *sc = pi->adapter; 1428 struct sge_txq *txq; 1429 void *items[1]; 1430 int rc; 1431 1432 M_ASSERTPKTHDR(m); 1433 MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */ 1434 1435 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1436 m_freem(m); 1437 return (ENETDOWN); 1438 } 1439 1440 rc = parse_pkt(&m); 1441 if (__predict_false(rc != 0)) { 1442 MPASS(m == NULL); /* was freed already */ 1443 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */ 1444 return (rc); 1445 } 1446 1447 /* Select a txq. */ 1448 txq = &sc->sge.txq[pi->first_txq]; 1449 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 1450 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) + 1451 pi->rsrv_noflowq); 1452 1453 items[0] = m; 1454 rc = mp_ring_enqueue(txq->r, items, 1, 4096); 1455 if (__predict_false(rc != 0)) 1456 m_freem(m); 1457 1458 return (rc); 1459 } 1460 1461 static void 1462 cxgbe_qflush(struct ifnet *ifp) 1463 { 1464 struct port_info *pi = ifp->if_softc; 1465 struct sge_txq *txq; 1466 int i; 1467 1468 /* queues do not exist if !PORT_INIT_DONE. */ 1469 if (pi->flags & PORT_INIT_DONE) { 1470 for_each_txq(pi, i, txq) { 1471 TXQ_LOCK(txq); 1472 txq->eq.flags &= ~EQ_ENABLED; 1473 TXQ_UNLOCK(txq); 1474 while (!mp_ring_is_idle(txq->r)) { 1475 mp_ring_check_drainage(txq->r, 0); 1476 pause("qflush", 1); 1477 } 1478 } 1479 } 1480 if_qflush(ifp); 1481 } 1482 1483 static uint64_t 1484 cxgbe_get_counter(struct ifnet *ifp, ift_counter c) 1485 { 1486 struct port_info *pi = ifp->if_softc; 1487 struct adapter *sc = pi->adapter; 1488 struct port_stats *s = &pi->stats; 1489 1490 cxgbe_refresh_stats(sc, pi); 1491 1492 switch (c) { 1493 case IFCOUNTER_IPACKETS: 1494 return (s->rx_frames - s->rx_pause); 1495 1496 case IFCOUNTER_IERRORS: 1497 return (s->rx_jabber + s->rx_runt + s->rx_too_long + 1498 s->rx_fcs_err + s->rx_len_err); 1499 1500 case IFCOUNTER_OPACKETS: 1501 return (s->tx_frames - s->tx_pause); 1502 1503 case IFCOUNTER_OERRORS: 1504 return (s->tx_error_frames); 1505 1506 case IFCOUNTER_IBYTES: 1507 return (s->rx_octets - s->rx_pause * 64); 1508 1509 case IFCOUNTER_OBYTES: 1510 return (s->tx_octets - s->tx_pause * 64); 1511 1512 case IFCOUNTER_IMCASTS: 1513 return (s->rx_mcast_frames - s->rx_pause); 1514 1515 case IFCOUNTER_OMCASTS: 1516 return (s->tx_mcast_frames - s->tx_pause); 1517 1518 case IFCOUNTER_IQDROPS: 1519 return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 1520 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 1521 s->rx_trunc3 + pi->tnl_cong_drops); 1522 1523 case IFCOUNTER_OQDROPS: { 1524 uint64_t drops; 1525 1526 drops = s->tx_drop; 1527 if (pi->flags & PORT_INIT_DONE) { 1528 int i; 1529 struct sge_txq *txq; 1530 1531 for_each_txq(pi, i, txq) 1532 drops += counter_u64_fetch(txq->r->drops); 1533 } 1534 1535 return (drops); 1536 1537 } 1538 1539 default: 1540 return (if_get_counter_default(ifp, c)); 1541 } 1542 } 1543 1544 static int 1545 cxgbe_media_change(struct ifnet *ifp) 1546 { 1547 struct port_info *pi = ifp->if_softc; 1548 1549 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1550 1551 return (EOPNOTSUPP); 1552 } 1553 1554 static void 1555 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1556 { 1557 struct port_info *pi = ifp->if_softc; 1558 struct ifmedia *media = NULL; 1559 struct ifmedia_entry *cur; 1560 int speed = pi->link_cfg.speed; 1561 1562 if (ifp == pi->ifp) 1563 media = &pi->media; 1564 #ifdef DEV_NETMAP 1565 else if (ifp == pi->nm_ifp) 1566 media = &pi->nm_media; 1567 #endif 1568 MPASS(media != NULL); 1569 1570 cur = media->ifm_cur; 1571 1572 ifmr->ifm_status = IFM_AVALID; 1573 if (!pi->link_cfg.link_ok) 1574 return; 1575 1576 ifmr->ifm_status |= IFM_ACTIVE; 1577 1578 /* active and current will differ iff current media is autoselect. */ 1579 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1580 return; 1581 1582 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1583 if (speed == SPEED_10000) 1584 ifmr->ifm_active |= IFM_10G_T; 1585 else if (speed == SPEED_1000) 1586 ifmr->ifm_active |= IFM_1000_T; 1587 else if (speed == SPEED_100) 1588 ifmr->ifm_active |= IFM_100_TX; 1589 else if (speed == SPEED_10) 1590 ifmr->ifm_active |= IFM_10_T; 1591 else 1592 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1593 speed)); 1594 } 1595 1596 void 1597 t4_fatal_err(struct adapter *sc) 1598 { 1599 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1600 t4_intr_disable(sc); 1601 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1602 device_get_nameunit(sc->dev)); 1603 } 1604 1605 static int 1606 map_bars_0_and_4(struct adapter *sc) 1607 { 1608 sc->regs_rid = PCIR_BAR(0); 1609 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1610 &sc->regs_rid, RF_ACTIVE); 1611 if (sc->regs_res == NULL) { 1612 device_printf(sc->dev, "cannot map registers.\n"); 1613 return (ENXIO); 1614 } 1615 sc->bt = rman_get_bustag(sc->regs_res); 1616 sc->bh = rman_get_bushandle(sc->regs_res); 1617 sc->mmio_len = rman_get_size(sc->regs_res); 1618 setbit(&sc->doorbells, DOORBELL_KDB); 1619 1620 sc->msix_rid = PCIR_BAR(4); 1621 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1622 &sc->msix_rid, RF_ACTIVE); 1623 if (sc->msix_res == NULL) { 1624 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1625 return (ENXIO); 1626 } 1627 1628 return (0); 1629 } 1630 1631 static int 1632 map_bar_2(struct adapter *sc) 1633 { 1634 1635 /* 1636 * T4: only iWARP driver uses the userspace doorbells. There is no need 1637 * to map it if RDMA is disabled. 1638 */ 1639 if (is_t4(sc) && sc->rdmacaps == 0) 1640 return (0); 1641 1642 sc->udbs_rid = PCIR_BAR(2); 1643 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1644 &sc->udbs_rid, RF_ACTIVE); 1645 if (sc->udbs_res == NULL) { 1646 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1647 return (ENXIO); 1648 } 1649 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1650 1651 if (is_t5(sc)) { 1652 setbit(&sc->doorbells, DOORBELL_UDB); 1653 #if defined(__i386__) || defined(__amd64__) 1654 if (t5_write_combine) { 1655 int rc; 1656 1657 /* 1658 * Enable write combining on BAR2. This is the 1659 * userspace doorbell BAR and is split into 128B 1660 * (UDBS_SEG_SIZE) doorbell regions, each associated 1661 * with an egress queue. The first 64B has the doorbell 1662 * and the second 64B can be used to submit a tx work 1663 * request with an implicit doorbell. 1664 */ 1665 1666 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1667 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1668 if (rc == 0) { 1669 clrbit(&sc->doorbells, DOORBELL_UDB); 1670 setbit(&sc->doorbells, DOORBELL_WCWR); 1671 setbit(&sc->doorbells, DOORBELL_UDBWC); 1672 } else { 1673 device_printf(sc->dev, 1674 "couldn't enable write combining: %d\n", 1675 rc); 1676 } 1677 1678 t4_write_reg(sc, A_SGE_STAT_CFG, 1679 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1680 } 1681 #endif 1682 } 1683 1684 return (0); 1685 } 1686 1687 static const struct memwin t4_memwin[] = { 1688 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1689 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1690 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1691 }; 1692 1693 static const struct memwin t5_memwin[] = { 1694 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1695 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1696 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1697 }; 1698 1699 static void 1700 setup_memwin(struct adapter *sc) 1701 { 1702 const struct memwin *mw; 1703 int i, n; 1704 uint32_t bar0; 1705 1706 if (is_t4(sc)) { 1707 /* 1708 * Read low 32b of bar0 indirectly via the hardware backdoor 1709 * mechanism. Works from within PCI passthrough environments 1710 * too, where rman_get_start() can return a different value. We 1711 * need to program the T4 memory window decoders with the actual 1712 * addresses that will be coming across the PCIe link. 1713 */ 1714 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1715 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1716 1717 mw = &t4_memwin[0]; 1718 n = nitems(t4_memwin); 1719 } else { 1720 /* T5 uses the relative offset inside the PCIe BAR */ 1721 bar0 = 0; 1722 1723 mw = &t5_memwin[0]; 1724 n = nitems(t5_memwin); 1725 } 1726 1727 for (i = 0; i < n; i++, mw++) { 1728 t4_write_reg(sc, 1729 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 1730 (mw->base + bar0) | V_BIR(0) | 1731 V_WINDOW(ilog2(mw->aperture) - 10)); 1732 } 1733 1734 /* flush */ 1735 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1736 } 1737 1738 /* 1739 * Verify that the memory range specified by the addr/len pair is valid and lies 1740 * entirely within a single region (EDCx or MCx). 1741 */ 1742 static int 1743 validate_mem_range(struct adapter *sc, uint32_t addr, int len) 1744 { 1745 uint32_t em, addr_len, maddr, mlen; 1746 1747 /* Memory can only be accessed in naturally aligned 4 byte units */ 1748 if (addr & 3 || len & 3 || len == 0) 1749 return (EINVAL); 1750 1751 /* Enabled memories */ 1752 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1753 if (em & F_EDRAM0_ENABLE) { 1754 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1755 maddr = G_EDRAM0_BASE(addr_len) << 20; 1756 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1757 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1758 addr + len <= maddr + mlen) 1759 return (0); 1760 } 1761 if (em & F_EDRAM1_ENABLE) { 1762 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1763 maddr = G_EDRAM1_BASE(addr_len) << 20; 1764 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1765 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1766 addr + len <= maddr + mlen) 1767 return (0); 1768 } 1769 if (em & F_EXT_MEM_ENABLE) { 1770 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1771 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1772 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1773 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1774 addr + len <= maddr + mlen) 1775 return (0); 1776 } 1777 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 1778 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1779 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1780 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1781 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1782 addr + len <= maddr + mlen) 1783 return (0); 1784 } 1785 1786 return (EFAULT); 1787 } 1788 1789 static int 1790 fwmtype_to_hwmtype(int mtype) 1791 { 1792 1793 switch (mtype) { 1794 case FW_MEMTYPE_EDC0: 1795 return (MEM_EDC0); 1796 case FW_MEMTYPE_EDC1: 1797 return (MEM_EDC1); 1798 case FW_MEMTYPE_EXTMEM: 1799 return (MEM_MC0); 1800 case FW_MEMTYPE_EXTMEM1: 1801 return (MEM_MC1); 1802 default: 1803 panic("%s: cannot translate fw mtype %d.", __func__, mtype); 1804 } 1805 } 1806 1807 /* 1808 * Verify that the memory range specified by the memtype/offset/len pair is 1809 * valid and lies entirely within the memtype specified. The global address of 1810 * the start of the range is returned in addr. 1811 */ 1812 static int 1813 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 1814 uint32_t *addr) 1815 { 1816 uint32_t em, addr_len, maddr, mlen; 1817 1818 /* Memory can only be accessed in naturally aligned 4 byte units */ 1819 if (off & 3 || len & 3 || len == 0) 1820 return (EINVAL); 1821 1822 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1823 switch (fwmtype_to_hwmtype(mtype)) { 1824 case MEM_EDC0: 1825 if (!(em & F_EDRAM0_ENABLE)) 1826 return (EINVAL); 1827 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1828 maddr = G_EDRAM0_BASE(addr_len) << 20; 1829 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1830 break; 1831 case MEM_EDC1: 1832 if (!(em & F_EDRAM1_ENABLE)) 1833 return (EINVAL); 1834 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1835 maddr = G_EDRAM1_BASE(addr_len) << 20; 1836 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1837 break; 1838 case MEM_MC: 1839 if (!(em & F_EXT_MEM_ENABLE)) 1840 return (EINVAL); 1841 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1842 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1843 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1844 break; 1845 case MEM_MC1: 1846 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 1847 return (EINVAL); 1848 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1849 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1850 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1851 break; 1852 default: 1853 return (EINVAL); 1854 } 1855 1856 if (mlen > 0 && off < mlen && off + len <= mlen) { 1857 *addr = maddr + off; /* global address */ 1858 return (0); 1859 } 1860 1861 return (EFAULT); 1862 } 1863 1864 static void 1865 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 1866 { 1867 const struct memwin *mw; 1868 1869 if (is_t4(sc)) { 1870 KASSERT(win >= 0 && win < nitems(t4_memwin), 1871 ("%s: incorrect memwin# (%d)", __func__, win)); 1872 mw = &t4_memwin[win]; 1873 } else { 1874 KASSERT(win >= 0 && win < nitems(t5_memwin), 1875 ("%s: incorrect memwin# (%d)", __func__, win)); 1876 mw = &t5_memwin[win]; 1877 } 1878 1879 if (base != NULL) 1880 *base = mw->base; 1881 if (aperture != NULL) 1882 *aperture = mw->aperture; 1883 } 1884 1885 /* 1886 * Positions the memory window such that it can be used to access the specified 1887 * address in the chip's address space. The return value is the offset of addr 1888 * from the start of the window. 1889 */ 1890 static uint32_t 1891 position_memwin(struct adapter *sc, int n, uint32_t addr) 1892 { 1893 uint32_t start, pf; 1894 uint32_t reg; 1895 1896 KASSERT(n >= 0 && n <= 3, 1897 ("%s: invalid window %d.", __func__, n)); 1898 KASSERT((addr & 3) == 0, 1899 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 1900 1901 if (is_t4(sc)) { 1902 pf = 0; 1903 start = addr & ~0xf; /* start must be 16B aligned */ 1904 } else { 1905 pf = V_PFNUM(sc->pf); 1906 start = addr & ~0x7f; /* start must be 128B aligned */ 1907 } 1908 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 1909 1910 t4_write_reg(sc, reg, start | pf); 1911 t4_read_reg(sc, reg); 1912 1913 return (addr - start); 1914 } 1915 1916 static int 1917 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1918 struct intrs_and_queues *iaq) 1919 { 1920 int rc, itype, navail, nrxq10g, nrxq1g, n; 1921 int nofldrxq10g = 0, nofldrxq1g = 0; 1922 int nnmrxq10g = 0, nnmrxq1g = 0; 1923 1924 bzero(iaq, sizeof(*iaq)); 1925 1926 iaq->ntxq10g = t4_ntxq10g; 1927 iaq->ntxq1g = t4_ntxq1g; 1928 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1929 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1930 iaq->rsrv_noflowq = t4_rsrv_noflowq; 1931 #ifdef TCP_OFFLOAD 1932 if (is_offload(sc)) { 1933 iaq->nofldtxq10g = t4_nofldtxq10g; 1934 iaq->nofldtxq1g = t4_nofldtxq1g; 1935 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1936 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1937 } 1938 #endif 1939 #ifdef DEV_NETMAP 1940 iaq->nnmtxq10g = t4_nnmtxq10g; 1941 iaq->nnmtxq1g = t4_nnmtxq1g; 1942 iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g; 1943 iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g; 1944 #endif 1945 1946 for (itype = INTR_MSIX; itype; itype >>= 1) { 1947 1948 if ((itype & t4_intr_types) == 0) 1949 continue; /* not allowed */ 1950 1951 if (itype == INTR_MSIX) 1952 navail = pci_msix_count(sc->dev); 1953 else if (itype == INTR_MSI) 1954 navail = pci_msi_count(sc->dev); 1955 else 1956 navail = 1; 1957 restart: 1958 if (navail == 0) 1959 continue; 1960 1961 iaq->intr_type = itype; 1962 iaq->intr_flags_10g = 0; 1963 iaq->intr_flags_1g = 0; 1964 1965 /* 1966 * Best option: an interrupt vector for errors, one for the 1967 * firmware event queue, and one for every rxq (NIC, TOE, and 1968 * netmap). 1969 */ 1970 iaq->nirq = T4_EXTRA_INTR; 1971 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g); 1972 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g); 1973 if (iaq->nirq <= navail && 1974 (itype != INTR_MSI || powerof2(iaq->nirq))) { 1975 iaq->intr_flags_10g = INTR_ALL; 1976 iaq->intr_flags_1g = INTR_ALL; 1977 goto allocate; 1978 } 1979 1980 /* 1981 * Second best option: a vector for errors, one for the firmware 1982 * event queue, and vectors for either all the NIC rx queues or 1983 * all the TOE rx queues. The queues that don't get vectors 1984 * will forward their interrupts to those that do. 1985 * 1986 * Note: netmap rx queues cannot be created early and so they 1987 * can't be setup to receive forwarded interrupts for others. 1988 */ 1989 iaq->nirq = T4_EXTRA_INTR; 1990 if (nrxq10g >= nofldrxq10g) { 1991 iaq->intr_flags_10g = INTR_RXQ; 1992 iaq->nirq += n10g * nrxq10g; 1993 #ifdef DEV_NETMAP 1994 iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g); 1995 #endif 1996 } else { 1997 iaq->intr_flags_10g = INTR_OFLD_RXQ; 1998 iaq->nirq += n10g * nofldrxq10g; 1999 #ifdef DEV_NETMAP 2000 iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g); 2001 #endif 2002 } 2003 if (nrxq1g >= nofldrxq1g) { 2004 iaq->intr_flags_1g = INTR_RXQ; 2005 iaq->nirq += n1g * nrxq1g; 2006 #ifdef DEV_NETMAP 2007 iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g); 2008 #endif 2009 } else { 2010 iaq->intr_flags_1g = INTR_OFLD_RXQ; 2011 iaq->nirq += n1g * nofldrxq1g; 2012 #ifdef DEV_NETMAP 2013 iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g); 2014 #endif 2015 } 2016 if (iaq->nirq <= navail && 2017 (itype != INTR_MSI || powerof2(iaq->nirq))) 2018 goto allocate; 2019 2020 /* 2021 * Next best option: an interrupt vector for errors, one for the 2022 * firmware event queue, and at least one per port. At this 2023 * point we know we'll have to downsize nrxq and/or nofldrxq 2024 * and/or nnmrxq to fit what's available to us. 2025 */ 2026 iaq->nirq = T4_EXTRA_INTR; 2027 iaq->nirq += n10g + n1g; 2028 if (iaq->nirq <= navail) { 2029 int leftover = navail - iaq->nirq; 2030 2031 if (n10g > 0) { 2032 int target = max(nrxq10g, nofldrxq10g); 2033 2034 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ? 2035 INTR_RXQ : INTR_OFLD_RXQ; 2036 2037 n = 1; 2038 while (n < target && leftover >= n10g) { 2039 leftover -= n10g; 2040 iaq->nirq += n10g; 2041 n++; 2042 } 2043 iaq->nrxq10g = min(n, nrxq10g); 2044 #ifdef TCP_OFFLOAD 2045 iaq->nofldrxq10g = min(n, nofldrxq10g); 2046 #endif 2047 #ifdef DEV_NETMAP 2048 iaq->nnmrxq10g = min(n, nnmrxq10g); 2049 #endif 2050 } 2051 2052 if (n1g > 0) { 2053 int target = max(nrxq1g, nofldrxq1g); 2054 2055 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ? 2056 INTR_RXQ : INTR_OFLD_RXQ; 2057 2058 n = 1; 2059 while (n < target && leftover >= n1g) { 2060 leftover -= n1g; 2061 iaq->nirq += n1g; 2062 n++; 2063 } 2064 iaq->nrxq1g = min(n, nrxq1g); 2065 #ifdef TCP_OFFLOAD 2066 iaq->nofldrxq1g = min(n, nofldrxq1g); 2067 #endif 2068 #ifdef DEV_NETMAP 2069 iaq->nnmrxq1g = min(n, nnmrxq1g); 2070 #endif 2071 } 2072 2073 if (itype != INTR_MSI || powerof2(iaq->nirq)) 2074 goto allocate; 2075 } 2076 2077 /* 2078 * Least desirable option: one interrupt vector for everything. 2079 */ 2080 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 2081 iaq->intr_flags_10g = iaq->intr_flags_1g = 0; 2082 #ifdef TCP_OFFLOAD 2083 if (is_offload(sc)) 2084 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 2085 #endif 2086 #ifdef DEV_NETMAP 2087 iaq->nnmrxq10g = iaq->nnmrxq1g = 1; 2088 #endif 2089 2090 allocate: 2091 navail = iaq->nirq; 2092 rc = 0; 2093 if (itype == INTR_MSIX) 2094 rc = pci_alloc_msix(sc->dev, &navail); 2095 else if (itype == INTR_MSI) 2096 rc = pci_alloc_msi(sc->dev, &navail); 2097 2098 if (rc == 0) { 2099 if (navail == iaq->nirq) 2100 return (0); 2101 2102 /* 2103 * Didn't get the number requested. Use whatever number 2104 * the kernel is willing to allocate (it's in navail). 2105 */ 2106 device_printf(sc->dev, "fewer vectors than requested, " 2107 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 2108 itype, iaq->nirq, navail); 2109 pci_release_msi(sc->dev); 2110 goto restart; 2111 } 2112 2113 device_printf(sc->dev, 2114 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 2115 itype, rc, iaq->nirq, navail); 2116 } 2117 2118 device_printf(sc->dev, 2119 "failed to find a usable interrupt type. " 2120 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 2121 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 2122 2123 return (ENXIO); 2124 } 2125 2126 #define FW_VERSION(chip) ( \ 2127 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ 2128 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ 2129 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ 2130 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) 2131 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) 2132 2133 struct fw_info { 2134 uint8_t chip; 2135 char *kld_name; 2136 char *fw_mod_name; 2137 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 2138 } fw_info[] = { 2139 { 2140 .chip = CHELSIO_T4, 2141 .kld_name = "t4fw_cfg", 2142 .fw_mod_name = "t4fw", 2143 .fw_hdr = { 2144 .chip = FW_HDR_CHIP_T4, 2145 .fw_ver = htobe32_const(FW_VERSION(T4)), 2146 .intfver_nic = FW_INTFVER(T4, NIC), 2147 .intfver_vnic = FW_INTFVER(T4, VNIC), 2148 .intfver_ofld = FW_INTFVER(T4, OFLD), 2149 .intfver_ri = FW_INTFVER(T4, RI), 2150 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 2151 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 2152 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 2153 .intfver_fcoe = FW_INTFVER(T4, FCOE), 2154 }, 2155 }, { 2156 .chip = CHELSIO_T5, 2157 .kld_name = "t5fw_cfg", 2158 .fw_mod_name = "t5fw", 2159 .fw_hdr = { 2160 .chip = FW_HDR_CHIP_T5, 2161 .fw_ver = htobe32_const(FW_VERSION(T5)), 2162 .intfver_nic = FW_INTFVER(T5, NIC), 2163 .intfver_vnic = FW_INTFVER(T5, VNIC), 2164 .intfver_ofld = FW_INTFVER(T5, OFLD), 2165 .intfver_ri = FW_INTFVER(T5, RI), 2166 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 2167 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 2168 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 2169 .intfver_fcoe = FW_INTFVER(T5, FCOE), 2170 }, 2171 } 2172 }; 2173 2174 static struct fw_info * 2175 find_fw_info(int chip) 2176 { 2177 int i; 2178 2179 for (i = 0; i < nitems(fw_info); i++) { 2180 if (fw_info[i].chip == chip) 2181 return (&fw_info[i]); 2182 } 2183 return (NULL); 2184 } 2185 2186 /* 2187 * Is the given firmware API compatible with the one the driver was compiled 2188 * with? 2189 */ 2190 static int 2191 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 2192 { 2193 2194 /* short circuit if it's the exact same firmware version */ 2195 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 2196 return (1); 2197 2198 /* 2199 * XXX: Is this too conservative? Perhaps I should limit this to the 2200 * features that are supported in the driver. 2201 */ 2202 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 2203 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 2204 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 2205 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 2206 return (1); 2207 #undef SAME_INTF 2208 2209 return (0); 2210 } 2211 2212 /* 2213 * The firmware in the KLD is usable, but should it be installed? This routine 2214 * explains itself in detail if it indicates the KLD firmware should be 2215 * installed. 2216 */ 2217 static int 2218 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 2219 { 2220 const char *reason; 2221 2222 if (!card_fw_usable) { 2223 reason = "incompatible or unusable"; 2224 goto install; 2225 } 2226 2227 if (k > c) { 2228 reason = "older than the version bundled with this driver"; 2229 goto install; 2230 } 2231 2232 if (t4_fw_install == 2 && k != c) { 2233 reason = "different than the version bundled with this driver"; 2234 goto install; 2235 } 2236 2237 return (0); 2238 2239 install: 2240 if (t4_fw_install == 0) { 2241 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2242 "but the driver is prohibited from installing a different " 2243 "firmware on the card.\n", 2244 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2245 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 2246 2247 return (0); 2248 } 2249 2250 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2251 "installing firmware %u.%u.%u.%u on card.\n", 2252 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2253 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 2254 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2255 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2256 2257 return (1); 2258 } 2259 /* 2260 * Establish contact with the firmware and determine if we are the master driver 2261 * or not, and whether we are responsible for chip initialization. 2262 */ 2263 static int 2264 prep_firmware(struct adapter *sc) 2265 { 2266 const struct firmware *fw = NULL, *default_cfg; 2267 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 2268 enum dev_state state; 2269 struct fw_info *fw_info; 2270 struct fw_hdr *card_fw; /* fw on the card */ 2271 const struct fw_hdr *kld_fw; /* fw in the KLD */ 2272 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 2273 against */ 2274 2275 /* Contact firmware. */ 2276 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 2277 if (rc < 0 || state == DEV_STATE_ERR) { 2278 rc = -rc; 2279 device_printf(sc->dev, 2280 "failed to connect to the firmware: %d, %d.\n", rc, state); 2281 return (rc); 2282 } 2283 pf = rc; 2284 if (pf == sc->mbox) 2285 sc->flags |= MASTER_PF; 2286 else if (state == DEV_STATE_UNINIT) { 2287 /* 2288 * We didn't get to be the master so we definitely won't be 2289 * configuring the chip. It's a bug if someone else hasn't 2290 * configured it already. 2291 */ 2292 device_printf(sc->dev, "couldn't be master(%d), " 2293 "device not already initialized either(%d).\n", rc, state); 2294 return (EDOOFUS); 2295 } 2296 2297 /* This is the firmware whose headers the driver was compiled against */ 2298 fw_info = find_fw_info(chip_id(sc)); 2299 if (fw_info == NULL) { 2300 device_printf(sc->dev, 2301 "unable to look up firmware information for chip %d.\n", 2302 chip_id(sc)); 2303 return (EINVAL); 2304 } 2305 drv_fw = &fw_info->fw_hdr; 2306 2307 /* 2308 * The firmware KLD contains many modules. The KLD name is also the 2309 * name of the module that contains the default config file. 2310 */ 2311 default_cfg = firmware_get(fw_info->kld_name); 2312 2313 /* Read the header of the firmware on the card */ 2314 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 2315 rc = -t4_read_flash(sc, FLASH_FW_START, 2316 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 2317 if (rc == 0) 2318 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 2319 else { 2320 device_printf(sc->dev, 2321 "Unable to read card's firmware header: %d\n", rc); 2322 card_fw_usable = 0; 2323 } 2324 2325 /* This is the firmware in the KLD */ 2326 fw = firmware_get(fw_info->fw_mod_name); 2327 if (fw != NULL) { 2328 kld_fw = (const void *)fw->data; 2329 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 2330 } else { 2331 kld_fw = NULL; 2332 kld_fw_usable = 0; 2333 } 2334 2335 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2336 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) { 2337 /* 2338 * Common case: the firmware on the card is an exact match and 2339 * the KLD is an exact match too, or the KLD is 2340 * absent/incompatible. Note that t4_fw_install = 2 is ignored 2341 * here -- use cxgbetool loadfw if you want to reinstall the 2342 * same firmware as the one on the card. 2343 */ 2344 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2345 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2346 be32toh(card_fw->fw_ver))) { 2347 2348 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 2349 if (rc != 0) { 2350 device_printf(sc->dev, 2351 "failed to install firmware: %d\n", rc); 2352 goto done; 2353 } 2354 2355 /* Installed successfully, update the cached header too. */ 2356 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2357 card_fw_usable = 1; 2358 need_fw_reset = 0; /* already reset as part of load_fw */ 2359 } 2360 2361 if (!card_fw_usable) { 2362 uint32_t d, c, k; 2363 2364 d = ntohl(drv_fw->fw_ver); 2365 c = ntohl(card_fw->fw_ver); 2366 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2367 2368 device_printf(sc->dev, "Cannot find a usable firmware: " 2369 "fw_install %d, chip state %d, " 2370 "driver compiled with %d.%d.%d.%d, " 2371 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2372 t4_fw_install, state, 2373 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2374 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2375 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2376 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2377 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2378 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2379 rc = EINVAL; 2380 goto done; 2381 } 2382 2383 /* We're using whatever's on the card and it's known to be good. */ 2384 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2385 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2386 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2387 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2388 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2389 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2390 t4_get_tp_version(sc, &sc->params.tp_vers); 2391 2392 /* Reset device */ 2393 if (need_fw_reset && 2394 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2395 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2396 if (rc != ETIMEDOUT && rc != EIO) 2397 t4_fw_bye(sc, sc->mbox); 2398 goto done; 2399 } 2400 sc->flags |= FW_OK; 2401 2402 rc = get_params__pre_init(sc); 2403 if (rc != 0) 2404 goto done; /* error message displayed already */ 2405 2406 /* Partition adapter resources as specified in the config file. */ 2407 if (state == DEV_STATE_UNINIT) { 2408 2409 KASSERT(sc->flags & MASTER_PF, 2410 ("%s: trying to change chip settings when not master.", 2411 __func__)); 2412 2413 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2414 if (rc != 0) 2415 goto done; /* error message displayed already */ 2416 2417 t4_tweak_chip_settings(sc); 2418 2419 /* get basic stuff going */ 2420 rc = -t4_fw_initialize(sc, sc->mbox); 2421 if (rc != 0) { 2422 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2423 goto done; 2424 } 2425 } else { 2426 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2427 sc->cfcsum = 0; 2428 } 2429 2430 done: 2431 free(card_fw, M_CXGBE); 2432 if (fw != NULL) 2433 firmware_put(fw, FIRMWARE_UNLOAD); 2434 if (default_cfg != NULL) 2435 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2436 2437 return (rc); 2438 } 2439 2440 #define FW_PARAM_DEV(param) \ 2441 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2442 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2443 #define FW_PARAM_PFVF(param) \ 2444 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2445 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2446 2447 /* 2448 * Partition chip resources for use between various PFs, VFs, etc. 2449 */ 2450 static int 2451 partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2452 const char *name_prefix) 2453 { 2454 const struct firmware *cfg = NULL; 2455 int rc = 0; 2456 struct fw_caps_config_cmd caps; 2457 uint32_t mtype, moff, finicsum, cfcsum; 2458 2459 /* 2460 * Figure out what configuration file to use. Pick the default config 2461 * file for the card if the user hasn't specified one explicitly. 2462 */ 2463 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2464 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2465 /* Card specific overrides go here. */ 2466 if (pci_get_device(sc->dev) == 0x440a) 2467 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2468 if (is_fpga(sc)) 2469 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2470 } 2471 2472 /* 2473 * We need to load another module if the profile is anything except 2474 * "default" or "flash". 2475 */ 2476 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2477 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2478 char s[32]; 2479 2480 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2481 cfg = firmware_get(s); 2482 if (cfg == NULL) { 2483 if (default_cfg != NULL) { 2484 device_printf(sc->dev, 2485 "unable to load module \"%s\" for " 2486 "configuration profile \"%s\", will use " 2487 "the default config file instead.\n", 2488 s, sc->cfg_file); 2489 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2490 "%s", DEFAULT_CF); 2491 } else { 2492 device_printf(sc->dev, 2493 "unable to load module \"%s\" for " 2494 "configuration profile \"%s\", will use " 2495 "the config file on the card's flash " 2496 "instead.\n", s, sc->cfg_file); 2497 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2498 "%s", FLASH_CF); 2499 } 2500 } 2501 } 2502 2503 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2504 default_cfg == NULL) { 2505 device_printf(sc->dev, 2506 "default config file not available, will use the config " 2507 "file on the card's flash instead.\n"); 2508 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2509 } 2510 2511 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2512 u_int cflen, i, n; 2513 const uint32_t *cfdata; 2514 uint32_t param, val, addr, off, mw_base, mw_aperture; 2515 2516 KASSERT(cfg != NULL || default_cfg != NULL, 2517 ("%s: no config to upload", __func__)); 2518 2519 /* 2520 * Ask the firmware where it wants us to upload the config file. 2521 */ 2522 param = FW_PARAM_DEV(CF); 2523 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2524 if (rc != 0) { 2525 /* No support for config file? Shouldn't happen. */ 2526 device_printf(sc->dev, 2527 "failed to query config file location: %d.\n", rc); 2528 goto done; 2529 } 2530 mtype = G_FW_PARAMS_PARAM_Y(val); 2531 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2532 2533 /* 2534 * XXX: sheer laziness. We deliberately added 4 bytes of 2535 * useless stuffing/comments at the end of the config file so 2536 * it's ok to simply throw away the last remaining bytes when 2537 * the config file is not an exact multiple of 4. This also 2538 * helps with the validate_mt_off_len check. 2539 */ 2540 if (cfg != NULL) { 2541 cflen = cfg->datasize & ~3; 2542 cfdata = cfg->data; 2543 } else { 2544 cflen = default_cfg->datasize & ~3; 2545 cfdata = default_cfg->data; 2546 } 2547 2548 if (cflen > FLASH_CFG_MAX_SIZE) { 2549 device_printf(sc->dev, 2550 "config file too long (%d, max allowed is %d). " 2551 "Will try to use the config on the card, if any.\n", 2552 cflen, FLASH_CFG_MAX_SIZE); 2553 goto use_config_on_flash; 2554 } 2555 2556 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2557 if (rc != 0) { 2558 device_printf(sc->dev, 2559 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2560 "Will try to use the config on the card, if any.\n", 2561 __func__, mtype, moff, cflen, rc); 2562 goto use_config_on_flash; 2563 } 2564 2565 memwin_info(sc, 2, &mw_base, &mw_aperture); 2566 while (cflen) { 2567 off = position_memwin(sc, 2, addr); 2568 n = min(cflen, mw_aperture - off); 2569 for (i = 0; i < n; i += 4) 2570 t4_write_reg(sc, mw_base + off + i, *cfdata++); 2571 cflen -= n; 2572 addr += n; 2573 } 2574 } else { 2575 use_config_on_flash: 2576 mtype = FW_MEMTYPE_FLASH; 2577 moff = t4_flash_cfg_addr(sc); 2578 } 2579 2580 bzero(&caps, sizeof(caps)); 2581 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2582 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2583 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2584 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2585 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2586 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2587 if (rc != 0) { 2588 device_printf(sc->dev, 2589 "failed to pre-process config file: %d " 2590 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2591 goto done; 2592 } 2593 2594 finicsum = be32toh(caps.finicsum); 2595 cfcsum = be32toh(caps.cfcsum); 2596 if (finicsum != cfcsum) { 2597 device_printf(sc->dev, 2598 "WARNING: config file checksum mismatch: %08x %08x\n", 2599 finicsum, cfcsum); 2600 } 2601 sc->cfcsum = cfcsum; 2602 2603 #define LIMIT_CAPS(x) do { \ 2604 caps.x &= htobe16(t4_##x##_allowed); \ 2605 } while (0) 2606 2607 /* 2608 * Let the firmware know what features will (not) be used so it can tune 2609 * things accordingly. 2610 */ 2611 LIMIT_CAPS(linkcaps); 2612 LIMIT_CAPS(niccaps); 2613 LIMIT_CAPS(toecaps); 2614 LIMIT_CAPS(rdmacaps); 2615 LIMIT_CAPS(iscsicaps); 2616 LIMIT_CAPS(fcoecaps); 2617 #undef LIMIT_CAPS 2618 2619 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2620 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2621 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2622 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2623 if (rc != 0) { 2624 device_printf(sc->dev, 2625 "failed to process config file: %d.\n", rc); 2626 } 2627 done: 2628 if (cfg != NULL) 2629 firmware_put(cfg, FIRMWARE_UNLOAD); 2630 return (rc); 2631 } 2632 2633 /* 2634 * Retrieve parameters that are needed (or nice to have) very early. 2635 */ 2636 static int 2637 get_params__pre_init(struct adapter *sc) 2638 { 2639 int rc; 2640 uint32_t param[2], val[2]; 2641 struct fw_devlog_cmd cmd; 2642 struct devlog_params *dlog = &sc->params.devlog; 2643 2644 param[0] = FW_PARAM_DEV(PORTVEC); 2645 param[1] = FW_PARAM_DEV(CCLK); 2646 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2647 if (rc != 0) { 2648 device_printf(sc->dev, 2649 "failed to query parameters (pre_init): %d.\n", rc); 2650 return (rc); 2651 } 2652 2653 sc->params.portvec = val[0]; 2654 sc->params.nports = bitcount32(val[0]); 2655 sc->params.vpd.cclk = val[1]; 2656 2657 /* Read device log parameters. */ 2658 bzero(&cmd, sizeof(cmd)); 2659 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2660 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2661 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2662 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2663 if (rc != 0) { 2664 device_printf(sc->dev, 2665 "failed to get devlog parameters: %d.\n", rc); 2666 bzero(dlog, sizeof (*dlog)); 2667 rc = 0; /* devlog isn't critical for device operation */ 2668 } else { 2669 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2670 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2671 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2672 dlog->size = be32toh(cmd.memsize_devlog); 2673 } 2674 2675 return (rc); 2676 } 2677 2678 /* 2679 * Retrieve various parameters that are of interest to the driver. The device 2680 * has been initialized by the firmware at this point. 2681 */ 2682 static int 2683 get_params__post_init(struct adapter *sc) 2684 { 2685 int rc; 2686 uint32_t param[7], val[7]; 2687 struct fw_caps_config_cmd caps; 2688 2689 param[0] = FW_PARAM_PFVF(IQFLINT_START); 2690 param[1] = FW_PARAM_PFVF(EQ_START); 2691 param[2] = FW_PARAM_PFVF(FILTER_START); 2692 param[3] = FW_PARAM_PFVF(FILTER_END); 2693 param[4] = FW_PARAM_PFVF(L2T_START); 2694 param[5] = FW_PARAM_PFVF(L2T_END); 2695 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2696 if (rc != 0) { 2697 device_printf(sc->dev, 2698 "failed to query parameters (post_init): %d.\n", rc); 2699 return (rc); 2700 } 2701 2702 sc->sge.iq_start = val[0]; 2703 sc->sge.eq_start = val[1]; 2704 sc->tids.ftid_base = val[2]; 2705 sc->tids.nftids = val[3] - val[2] + 1; 2706 sc->params.ftid_min = val[2]; 2707 sc->params.ftid_max = val[3]; 2708 sc->vres.l2t.start = val[4]; 2709 sc->vres.l2t.size = val[5] - val[4] + 1; 2710 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2711 ("%s: L2 table size (%u) larger than expected (%u)", 2712 __func__, sc->vres.l2t.size, L2T_SIZE)); 2713 2714 /* get capabilites */ 2715 bzero(&caps, sizeof(caps)); 2716 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2717 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2718 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2719 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2720 if (rc != 0) { 2721 device_printf(sc->dev, 2722 "failed to get card capabilities: %d.\n", rc); 2723 return (rc); 2724 } 2725 2726 #define READ_CAPS(x) do { \ 2727 sc->x = htobe16(caps.x); \ 2728 } while (0) 2729 READ_CAPS(linkcaps); 2730 READ_CAPS(niccaps); 2731 READ_CAPS(toecaps); 2732 READ_CAPS(rdmacaps); 2733 READ_CAPS(iscsicaps); 2734 READ_CAPS(fcoecaps); 2735 2736 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { 2737 param[0] = FW_PARAM_PFVF(ETHOFLD_START); 2738 param[1] = FW_PARAM_PFVF(ETHOFLD_END); 2739 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2740 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); 2741 if (rc != 0) { 2742 device_printf(sc->dev, 2743 "failed to query NIC parameters: %d.\n", rc); 2744 return (rc); 2745 } 2746 sc->tids.etid_base = val[0]; 2747 sc->params.etid_min = val[0]; 2748 sc->tids.netids = val[1] - val[0] + 1; 2749 sc->params.netids = sc->tids.netids; 2750 sc->params.eo_wr_cred = val[2]; 2751 sc->params.ethoffload = 1; 2752 } 2753 2754 if (sc->toecaps) { 2755 /* query offload-related parameters */ 2756 param[0] = FW_PARAM_DEV(NTID); 2757 param[1] = FW_PARAM_PFVF(SERVER_START); 2758 param[2] = FW_PARAM_PFVF(SERVER_END); 2759 param[3] = FW_PARAM_PFVF(TDDP_START); 2760 param[4] = FW_PARAM_PFVF(TDDP_END); 2761 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2762 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2763 if (rc != 0) { 2764 device_printf(sc->dev, 2765 "failed to query TOE parameters: %d.\n", rc); 2766 return (rc); 2767 } 2768 sc->tids.ntids = val[0]; 2769 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 2770 sc->tids.stid_base = val[1]; 2771 sc->tids.nstids = val[2] - val[1] + 1; 2772 sc->vres.ddp.start = val[3]; 2773 sc->vres.ddp.size = val[4] - val[3] + 1; 2774 sc->params.ofldq_wr_cred = val[5]; 2775 sc->params.offload = 1; 2776 } 2777 if (sc->rdmacaps) { 2778 param[0] = FW_PARAM_PFVF(STAG_START); 2779 param[1] = FW_PARAM_PFVF(STAG_END); 2780 param[2] = FW_PARAM_PFVF(RQ_START); 2781 param[3] = FW_PARAM_PFVF(RQ_END); 2782 param[4] = FW_PARAM_PFVF(PBL_START); 2783 param[5] = FW_PARAM_PFVF(PBL_END); 2784 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2785 if (rc != 0) { 2786 device_printf(sc->dev, 2787 "failed to query RDMA parameters(1): %d.\n", rc); 2788 return (rc); 2789 } 2790 sc->vres.stag.start = val[0]; 2791 sc->vres.stag.size = val[1] - val[0] + 1; 2792 sc->vres.rq.start = val[2]; 2793 sc->vres.rq.size = val[3] - val[2] + 1; 2794 sc->vres.pbl.start = val[4]; 2795 sc->vres.pbl.size = val[5] - val[4] + 1; 2796 2797 param[0] = FW_PARAM_PFVF(SQRQ_START); 2798 param[1] = FW_PARAM_PFVF(SQRQ_END); 2799 param[2] = FW_PARAM_PFVF(CQ_START); 2800 param[3] = FW_PARAM_PFVF(CQ_END); 2801 param[4] = FW_PARAM_PFVF(OCQ_START); 2802 param[5] = FW_PARAM_PFVF(OCQ_END); 2803 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2804 if (rc != 0) { 2805 device_printf(sc->dev, 2806 "failed to query RDMA parameters(2): %d.\n", rc); 2807 return (rc); 2808 } 2809 sc->vres.qp.start = val[0]; 2810 sc->vres.qp.size = val[1] - val[0] + 1; 2811 sc->vres.cq.start = val[2]; 2812 sc->vres.cq.size = val[3] - val[2] + 1; 2813 sc->vres.ocq.start = val[4]; 2814 sc->vres.ocq.size = val[5] - val[4] + 1; 2815 } 2816 if (sc->iscsicaps) { 2817 param[0] = FW_PARAM_PFVF(ISCSI_START); 2818 param[1] = FW_PARAM_PFVF(ISCSI_END); 2819 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2820 if (rc != 0) { 2821 device_printf(sc->dev, 2822 "failed to query iSCSI parameters: %d.\n", rc); 2823 return (rc); 2824 } 2825 sc->vres.iscsi.start = val[0]; 2826 sc->vres.iscsi.size = val[1] - val[0] + 1; 2827 } 2828 2829 /* 2830 * We've got the params we wanted to query via the firmware. Now grab 2831 * some others directly from the chip. 2832 */ 2833 rc = t4_read_chip_settings(sc); 2834 2835 return (rc); 2836 } 2837 2838 static int 2839 set_params__post_init(struct adapter *sc) 2840 { 2841 uint32_t param, val; 2842 2843 /* ask for encapsulated CPLs */ 2844 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 2845 val = 1; 2846 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2847 2848 return (0); 2849 } 2850 2851 #undef FW_PARAM_PFVF 2852 #undef FW_PARAM_DEV 2853 2854 static void 2855 t4_set_desc(struct adapter *sc) 2856 { 2857 char buf[128]; 2858 struct adapter_params *p = &sc->params; 2859 2860 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, " 2861 "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "", 2862 chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec); 2863 2864 device_set_desc_copy(sc->dev, buf); 2865 } 2866 2867 static void 2868 build_medialist(struct port_info *pi, struct ifmedia *media) 2869 { 2870 int m; 2871 2872 PORT_LOCK(pi); 2873 2874 ifmedia_removeall(media); 2875 2876 m = IFM_ETHER | IFM_FDX; 2877 2878 switch(pi->port_type) { 2879 case FW_PORT_TYPE_BT_XFI: 2880 case FW_PORT_TYPE_BT_XAUI: 2881 ifmedia_add(media, m | IFM_10G_T, 0, NULL); 2882 /* fall through */ 2883 2884 case FW_PORT_TYPE_BT_SGMII: 2885 ifmedia_add(media, m | IFM_1000_T, 0, NULL); 2886 ifmedia_add(media, m | IFM_100_TX, 0, NULL); 2887 ifmedia_add(media, IFM_ETHER | IFM_AUTO, 0, NULL); 2888 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2889 break; 2890 2891 case FW_PORT_TYPE_CX4: 2892 ifmedia_add(media, m | IFM_10G_CX4, 0, NULL); 2893 ifmedia_set(media, m | IFM_10G_CX4); 2894 break; 2895 2896 case FW_PORT_TYPE_QSFP_10G: 2897 case FW_PORT_TYPE_SFP: 2898 case FW_PORT_TYPE_FIBER_XFI: 2899 case FW_PORT_TYPE_FIBER_XAUI: 2900 switch (pi->mod_type) { 2901 2902 case FW_PORT_MOD_TYPE_LR: 2903 ifmedia_add(media, m | IFM_10G_LR, 0, NULL); 2904 ifmedia_set(media, m | IFM_10G_LR); 2905 break; 2906 2907 case FW_PORT_MOD_TYPE_SR: 2908 ifmedia_add(media, m | IFM_10G_SR, 0, NULL); 2909 ifmedia_set(media, m | IFM_10G_SR); 2910 break; 2911 2912 case FW_PORT_MOD_TYPE_LRM: 2913 ifmedia_add(media, m | IFM_10G_LRM, 0, NULL); 2914 ifmedia_set(media, m | IFM_10G_LRM); 2915 break; 2916 2917 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2918 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2919 ifmedia_add(media, m | IFM_10G_TWINAX, 0, NULL); 2920 ifmedia_set(media, m | IFM_10G_TWINAX); 2921 break; 2922 2923 case FW_PORT_MOD_TYPE_NONE: 2924 m &= ~IFM_FDX; 2925 ifmedia_add(media, m | IFM_NONE, 0, NULL); 2926 ifmedia_set(media, m | IFM_NONE); 2927 break; 2928 2929 case FW_PORT_MOD_TYPE_NA: 2930 case FW_PORT_MOD_TYPE_ER: 2931 default: 2932 device_printf(pi->dev, 2933 "unknown port_type (%d), mod_type (%d)\n", 2934 pi->port_type, pi->mod_type); 2935 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 2936 ifmedia_set(media, m | IFM_UNKNOWN); 2937 break; 2938 } 2939 break; 2940 2941 case FW_PORT_TYPE_QSFP: 2942 switch (pi->mod_type) { 2943 2944 case FW_PORT_MOD_TYPE_LR: 2945 ifmedia_add(media, m | IFM_40G_LR4, 0, NULL); 2946 ifmedia_set(media, m | IFM_40G_LR4); 2947 break; 2948 2949 case FW_PORT_MOD_TYPE_SR: 2950 ifmedia_add(media, m | IFM_40G_SR4, 0, NULL); 2951 ifmedia_set(media, m | IFM_40G_SR4); 2952 break; 2953 2954 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2955 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2956 ifmedia_add(media, m | IFM_40G_CR4, 0, NULL); 2957 ifmedia_set(media, m | IFM_40G_CR4); 2958 break; 2959 2960 case FW_PORT_MOD_TYPE_NONE: 2961 m &= ~IFM_FDX; 2962 ifmedia_add(media, m | IFM_NONE, 0, NULL); 2963 ifmedia_set(media, m | IFM_NONE); 2964 break; 2965 2966 default: 2967 device_printf(pi->dev, 2968 "unknown port_type (%d), mod_type (%d)\n", 2969 pi->port_type, pi->mod_type); 2970 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 2971 ifmedia_set(media, m | IFM_UNKNOWN); 2972 break; 2973 } 2974 break; 2975 2976 default: 2977 device_printf(pi->dev, 2978 "unknown port_type (%d), mod_type (%d)\n", pi->port_type, 2979 pi->mod_type); 2980 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 2981 ifmedia_set(media, m | IFM_UNKNOWN); 2982 break; 2983 } 2984 2985 PORT_UNLOCK(pi); 2986 } 2987 2988 #define FW_MAC_EXACT_CHUNK 7 2989 2990 /* 2991 * Program the port's XGMAC based on parameters in ifnet. The caller also 2992 * indicates which parameters should be programmed (the rest are left alone). 2993 */ 2994 int 2995 update_mac_settings(struct ifnet *ifp, int flags) 2996 { 2997 int rc = 0; 2998 struct port_info *pi = ifp->if_softc; 2999 struct adapter *sc = pi->adapter; 3000 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 3001 uint16_t viid = 0xffff; 3002 int16_t *xact_addr_filt = NULL; 3003 3004 ASSERT_SYNCHRONIZED_OP(sc); 3005 KASSERT(flags, ("%s: not told what to update.", __func__)); 3006 3007 if (ifp == pi->ifp) { 3008 viid = pi->viid; 3009 xact_addr_filt = &pi->xact_addr_filt; 3010 } 3011 #ifdef DEV_NETMAP 3012 else if (ifp == pi->nm_ifp) { 3013 viid = pi->nm_viid; 3014 xact_addr_filt = &pi->nm_xact_addr_filt; 3015 } 3016 #endif 3017 if (flags & XGMAC_MTU) 3018 mtu = ifp->if_mtu; 3019 3020 if (flags & XGMAC_PROMISC) 3021 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 3022 3023 if (flags & XGMAC_ALLMULTI) 3024 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 3025 3026 if (flags & XGMAC_VLANEX) 3027 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 3028 3029 if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) { 3030 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti, 3031 1, vlanex, false); 3032 if (rc) { 3033 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, 3034 rc); 3035 return (rc); 3036 } 3037 } 3038 3039 if (flags & XGMAC_UCADDR) { 3040 uint8_t ucaddr[ETHER_ADDR_LEN]; 3041 3042 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 3043 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr, 3044 true, true); 3045 if (rc < 0) { 3046 rc = -rc; 3047 if_printf(ifp, "change_mac failed: %d\n", rc); 3048 return (rc); 3049 } else { 3050 *xact_addr_filt = rc; 3051 rc = 0; 3052 } 3053 } 3054 3055 if (flags & XGMAC_MCADDRS) { 3056 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 3057 int del = 1; 3058 uint64_t hash = 0; 3059 struct ifmultiaddr *ifma; 3060 int i = 0, j; 3061 3062 if_maddr_rlock(ifp); 3063 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3064 if (ifma->ifma_addr->sa_family != AF_LINK) 3065 continue; 3066 mcaddr[i] = 3067 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 3068 MPASS(ETHER_IS_MULTICAST(mcaddr[i])); 3069 i++; 3070 3071 if (i == FW_MAC_EXACT_CHUNK) { 3072 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, 3073 i, mcaddr, NULL, &hash, 0); 3074 if (rc < 0) { 3075 rc = -rc; 3076 for (j = 0; j < i; j++) { 3077 if_printf(ifp, 3078 "failed to add mc address" 3079 " %02x:%02x:%02x:" 3080 "%02x:%02x:%02x rc=%d\n", 3081 mcaddr[j][0], mcaddr[j][1], 3082 mcaddr[j][2], mcaddr[j][3], 3083 mcaddr[j][4], mcaddr[j][5], 3084 rc); 3085 } 3086 goto mcfail; 3087 } 3088 del = 0; 3089 i = 0; 3090 } 3091 } 3092 if (i > 0) { 3093 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i, 3094 mcaddr, NULL, &hash, 0); 3095 if (rc < 0) { 3096 rc = -rc; 3097 for (j = 0; j < i; j++) { 3098 if_printf(ifp, 3099 "failed to add mc address" 3100 " %02x:%02x:%02x:" 3101 "%02x:%02x:%02x rc=%d\n", 3102 mcaddr[j][0], mcaddr[j][1], 3103 mcaddr[j][2], mcaddr[j][3], 3104 mcaddr[j][4], mcaddr[j][5], 3105 rc); 3106 } 3107 goto mcfail; 3108 } 3109 } 3110 3111 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0); 3112 if (rc != 0) 3113 if_printf(ifp, "failed to set mc address hash: %d", rc); 3114 mcfail: 3115 if_maddr_runlock(ifp); 3116 } 3117 3118 return (rc); 3119 } 3120 3121 /* 3122 * {begin|end}_synchronized_op must be called from the same thread. 3123 */ 3124 int 3125 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags, 3126 char *wmesg) 3127 { 3128 int rc, pri; 3129 3130 #ifdef WITNESS 3131 /* the caller thinks it's ok to sleep, but is it really? */ 3132 if (flags & SLEEP_OK) 3133 pause("t4slptst", 1); 3134 #endif 3135 3136 if (INTR_OK) 3137 pri = PCATCH; 3138 else 3139 pri = 0; 3140 3141 ADAPTER_LOCK(sc); 3142 for (;;) { 3143 3144 if (pi && IS_DOOMED(pi)) { 3145 rc = ENXIO; 3146 goto done; 3147 } 3148 3149 if (!IS_BUSY(sc)) { 3150 rc = 0; 3151 break; 3152 } 3153 3154 if (!(flags & SLEEP_OK)) { 3155 rc = EBUSY; 3156 goto done; 3157 } 3158 3159 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 3160 rc = EINTR; 3161 goto done; 3162 } 3163 } 3164 3165 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 3166 SET_BUSY(sc); 3167 #ifdef INVARIANTS 3168 sc->last_op = wmesg; 3169 sc->last_op_thr = curthread; 3170 #endif 3171 3172 done: 3173 if (!(flags & HOLD_LOCK) || rc) 3174 ADAPTER_UNLOCK(sc); 3175 3176 return (rc); 3177 } 3178 3179 /* 3180 * {begin|end}_synchronized_op must be called from the same thread. 3181 */ 3182 void 3183 end_synchronized_op(struct adapter *sc, int flags) 3184 { 3185 3186 if (flags & LOCK_HELD) 3187 ADAPTER_LOCK_ASSERT_OWNED(sc); 3188 else 3189 ADAPTER_LOCK(sc); 3190 3191 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 3192 CLR_BUSY(sc); 3193 wakeup(&sc->flags); 3194 ADAPTER_UNLOCK(sc); 3195 } 3196 3197 static int 3198 cxgbe_init_synchronized(struct port_info *pi) 3199 { 3200 struct adapter *sc = pi->adapter; 3201 struct ifnet *ifp = pi->ifp; 3202 int rc = 0, i; 3203 struct sge_txq *txq; 3204 3205 ASSERT_SYNCHRONIZED_OP(sc); 3206 3207 if (isset(&sc->open_device_map, pi->port_id)) { 3208 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 3209 ("mismatch between open_device_map and if_drv_flags")); 3210 return (0); /* already running */ 3211 } 3212 3213 if (!(sc->flags & FULL_INIT_DONE) && 3214 ((rc = adapter_full_init(sc)) != 0)) 3215 return (rc); /* error message displayed already */ 3216 3217 if (!(pi->flags & PORT_INIT_DONE) && 3218 ((rc = port_full_init(pi)) != 0)) 3219 return (rc); /* error message displayed already */ 3220 3221 rc = update_mac_settings(ifp, XGMAC_ALL); 3222 if (rc) 3223 goto done; /* error message displayed already */ 3224 3225 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 3226 if (rc != 0) { 3227 if_printf(ifp, "enable_vi failed: %d\n", rc); 3228 goto done; 3229 } 3230 3231 /* 3232 * Can't fail from this point onwards. Review cxgbe_uninit_synchronized 3233 * if this changes. 3234 */ 3235 3236 for_each_txq(pi, i, txq) { 3237 TXQ_LOCK(txq); 3238 txq->eq.flags |= EQ_ENABLED; 3239 TXQ_UNLOCK(txq); 3240 } 3241 3242 /* 3243 * The first iq of the first port to come up is used for tracing. 3244 */ 3245 if (sc->traceq < 0) { 3246 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id; 3247 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : 3248 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | 3249 V_QUEUENUMBER(sc->traceq)); 3250 pi->flags |= HAS_TRACEQ; 3251 } 3252 3253 /* all ok */ 3254 setbit(&sc->open_device_map, pi->port_id); 3255 PORT_LOCK(pi); 3256 ifp->if_drv_flags |= IFF_DRV_RUNNING; 3257 PORT_UNLOCK(pi); 3258 3259 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 3260 done: 3261 if (rc != 0) 3262 cxgbe_uninit_synchronized(pi); 3263 3264 return (rc); 3265 } 3266 3267 /* 3268 * Idempotent. 3269 */ 3270 static int 3271 cxgbe_uninit_synchronized(struct port_info *pi) 3272 { 3273 struct adapter *sc = pi->adapter; 3274 struct ifnet *ifp = pi->ifp; 3275 int rc, i; 3276 struct sge_txq *txq; 3277 3278 ASSERT_SYNCHRONIZED_OP(sc); 3279 3280 if (!(pi->flags & PORT_INIT_DONE)) { 3281 KASSERT(!(ifp->if_drv_flags & IFF_DRV_RUNNING), 3282 ("uninited port is running")); 3283 return (0); 3284 } 3285 3286 /* 3287 * Disable the VI so that all its data in either direction is discarded 3288 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 3289 * tick) intact as the TP can deliver negative advice or data that it's 3290 * holding in its RAM (for an offloaded connection) even after the VI is 3291 * disabled. 3292 */ 3293 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 3294 if (rc) { 3295 if_printf(ifp, "disable_vi failed: %d\n", rc); 3296 return (rc); 3297 } 3298 3299 for_each_txq(pi, i, txq) { 3300 TXQ_LOCK(txq); 3301 txq->eq.flags &= ~EQ_ENABLED; 3302 TXQ_UNLOCK(txq); 3303 } 3304 3305 clrbit(&sc->open_device_map, pi->port_id); 3306 PORT_LOCK(pi); 3307 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3308 PORT_UNLOCK(pi); 3309 3310 pi->link_cfg.link_ok = 0; 3311 pi->link_cfg.speed = 0; 3312 pi->linkdnrc = -1; 3313 t4_os_link_changed(sc, pi->port_id, 0, -1); 3314 3315 return (0); 3316 } 3317 3318 /* 3319 * It is ok for this function to fail midway and return right away. t4_detach 3320 * will walk the entire sc->irq list and clean up whatever is valid. 3321 */ 3322 static int 3323 setup_intr_handlers(struct adapter *sc) 3324 { 3325 int rc, rid, p, q; 3326 char s[8]; 3327 struct irq *irq; 3328 struct port_info *pi; 3329 struct sge_rxq *rxq; 3330 #ifdef TCP_OFFLOAD 3331 struct sge_ofld_rxq *ofld_rxq; 3332 #endif 3333 #ifdef DEV_NETMAP 3334 struct sge_nm_rxq *nm_rxq; 3335 #endif 3336 3337 /* 3338 * Setup interrupts. 3339 */ 3340 irq = &sc->irq[0]; 3341 rid = sc->intr_type == INTR_INTX ? 0 : 1; 3342 if (sc->intr_count == 1) 3343 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all")); 3344 3345 /* Multiple interrupts. */ 3346 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 3347 ("%s: too few intr.", __func__)); 3348 3349 /* The first one is always error intr */ 3350 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 3351 if (rc != 0) 3352 return (rc); 3353 irq++; 3354 rid++; 3355 3356 /* The second one is always the firmware event queue */ 3357 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt"); 3358 if (rc != 0) 3359 return (rc); 3360 irq++; 3361 rid++; 3362 3363 for_each_port(sc, p) { 3364 pi = sc->port[p]; 3365 3366 if (pi->flags & INTR_RXQ) { 3367 for_each_rxq(pi, q, rxq) { 3368 snprintf(s, sizeof(s), "%d.%d", p, q); 3369 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq, 3370 s); 3371 if (rc != 0) 3372 return (rc); 3373 irq++; 3374 rid++; 3375 } 3376 } 3377 #ifdef TCP_OFFLOAD 3378 if (pi->flags & INTR_OFLD_RXQ) { 3379 for_each_ofld_rxq(pi, q, ofld_rxq) { 3380 snprintf(s, sizeof(s), "%d,%d", p, q); 3381 rc = t4_alloc_irq(sc, irq, rid, t4_intr, 3382 ofld_rxq, s); 3383 if (rc != 0) 3384 return (rc); 3385 irq++; 3386 rid++; 3387 } 3388 } 3389 #endif 3390 #ifdef DEV_NETMAP 3391 if (pi->flags & INTR_NM_RXQ) { 3392 for_each_nm_rxq(pi, q, nm_rxq) { 3393 snprintf(s, sizeof(s), "%d-%d", p, q); 3394 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr, 3395 nm_rxq, s); 3396 if (rc != 0) 3397 return (rc); 3398 irq++; 3399 rid++; 3400 } 3401 } 3402 #endif 3403 } 3404 MPASS(irq == &sc->irq[sc->intr_count]); 3405 3406 return (0); 3407 } 3408 3409 int 3410 adapter_full_init(struct adapter *sc) 3411 { 3412 int rc, i; 3413 3414 ASSERT_SYNCHRONIZED_OP(sc); 3415 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3416 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 3417 ("%s: FULL_INIT_DONE already", __func__)); 3418 3419 /* 3420 * queues that belong to the adapter (not any particular port). 3421 */ 3422 rc = t4_setup_adapter_queues(sc); 3423 if (rc != 0) 3424 goto done; 3425 3426 for (i = 0; i < nitems(sc->tq); i++) { 3427 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 3428 taskqueue_thread_enqueue, &sc->tq[i]); 3429 if (sc->tq[i] == NULL) { 3430 device_printf(sc->dev, 3431 "failed to allocate task queue %d\n", i); 3432 rc = ENOMEM; 3433 goto done; 3434 } 3435 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3436 device_get_nameunit(sc->dev), i); 3437 } 3438 3439 t4_intr_enable(sc); 3440 sc->flags |= FULL_INIT_DONE; 3441 done: 3442 if (rc != 0) 3443 adapter_full_uninit(sc); 3444 3445 return (rc); 3446 } 3447 3448 int 3449 adapter_full_uninit(struct adapter *sc) 3450 { 3451 int i; 3452 3453 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3454 3455 t4_teardown_adapter_queues(sc); 3456 3457 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3458 taskqueue_free(sc->tq[i]); 3459 sc->tq[i] = NULL; 3460 } 3461 3462 sc->flags &= ~FULL_INIT_DONE; 3463 3464 return (0); 3465 } 3466 3467 int 3468 port_full_init(struct port_info *pi) 3469 { 3470 struct adapter *sc = pi->adapter; 3471 struct ifnet *ifp = pi->ifp; 3472 uint16_t *rss; 3473 struct sge_rxq *rxq; 3474 int rc, i, j; 3475 3476 ASSERT_SYNCHRONIZED_OP(sc); 3477 KASSERT((pi->flags & PORT_INIT_DONE) == 0, 3478 ("%s: PORT_INIT_DONE already", __func__)); 3479 3480 sysctl_ctx_init(&pi->ctx); 3481 pi->flags |= PORT_SYSCTL_CTX; 3482 3483 /* 3484 * Allocate tx/rx/fl queues for this port. 3485 */ 3486 rc = t4_setup_port_queues(pi); 3487 if (rc != 0) 3488 goto done; /* error message displayed already */ 3489 3490 /* 3491 * Setup RSS for this port. Save a copy of the RSS table for later use. 3492 */ 3493 rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); 3494 for (i = 0; i < pi->rss_size;) { 3495 for_each_rxq(pi, j, rxq) { 3496 rss[i++] = rxq->iq.abs_id; 3497 if (i == pi->rss_size) 3498 break; 3499 } 3500 } 3501 3502 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss, 3503 pi->rss_size); 3504 if (rc != 0) { 3505 if_printf(ifp, "rss_config failed: %d\n", rc); 3506 goto done; 3507 } 3508 3509 pi->rss = rss; 3510 pi->flags |= PORT_INIT_DONE; 3511 done: 3512 if (rc != 0) 3513 port_full_uninit(pi); 3514 3515 return (rc); 3516 } 3517 3518 /* 3519 * Idempotent. 3520 */ 3521 int 3522 port_full_uninit(struct port_info *pi) 3523 { 3524 struct adapter *sc = pi->adapter; 3525 int i; 3526 struct sge_rxq *rxq; 3527 struct sge_txq *txq; 3528 #ifdef TCP_OFFLOAD 3529 struct sge_ofld_rxq *ofld_rxq; 3530 struct sge_wrq *ofld_txq; 3531 #endif 3532 3533 if (pi->flags & PORT_INIT_DONE) { 3534 3535 /* Need to quiesce queues. */ 3536 3537 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]); 3538 3539 for_each_txq(pi, i, txq) { 3540 quiesce_txq(sc, txq); 3541 } 3542 3543 #ifdef TCP_OFFLOAD 3544 for_each_ofld_txq(pi, i, ofld_txq) { 3545 quiesce_wrq(sc, ofld_txq); 3546 } 3547 #endif 3548 3549 for_each_rxq(pi, i, rxq) { 3550 quiesce_iq(sc, &rxq->iq); 3551 quiesce_fl(sc, &rxq->fl); 3552 } 3553 3554 #ifdef TCP_OFFLOAD 3555 for_each_ofld_rxq(pi, i, ofld_rxq) { 3556 quiesce_iq(sc, &ofld_rxq->iq); 3557 quiesce_fl(sc, &ofld_rxq->fl); 3558 } 3559 #endif 3560 free(pi->rss, M_CXGBE); 3561 } 3562 3563 t4_teardown_port_queues(pi); 3564 pi->flags &= ~PORT_INIT_DONE; 3565 3566 return (0); 3567 } 3568 3569 static void 3570 quiesce_txq(struct adapter *sc, struct sge_txq *txq) 3571 { 3572 struct sge_eq *eq = &txq->eq; 3573 struct sge_qstat *spg = (void *)&eq->desc[eq->sidx]; 3574 3575 (void) sc; /* unused */ 3576 3577 #ifdef INVARIANTS 3578 TXQ_LOCK(txq); 3579 MPASS((eq->flags & EQ_ENABLED) == 0); 3580 TXQ_UNLOCK(txq); 3581 #endif 3582 3583 /* Wait for the mp_ring to empty. */ 3584 while (!mp_ring_is_idle(txq->r)) { 3585 mp_ring_check_drainage(txq->r, 0); 3586 pause("rquiesce", 1); 3587 } 3588 3589 /* Then wait for the hardware to finish. */ 3590 while (spg->cidx != htobe16(eq->pidx)) 3591 pause("equiesce", 1); 3592 3593 /* Finally, wait for the driver to reclaim all descriptors. */ 3594 while (eq->cidx != eq->pidx) 3595 pause("dquiesce", 1); 3596 } 3597 3598 static void 3599 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq) 3600 { 3601 3602 /* XXXTX */ 3603 } 3604 3605 static void 3606 quiesce_iq(struct adapter *sc, struct sge_iq *iq) 3607 { 3608 (void) sc; /* unused */ 3609 3610 /* Synchronize with the interrupt handler */ 3611 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 3612 pause("iqfree", 1); 3613 } 3614 3615 static void 3616 quiesce_fl(struct adapter *sc, struct sge_fl *fl) 3617 { 3618 mtx_lock(&sc->sfl_lock); 3619 FL_LOCK(fl); 3620 fl->flags |= FL_DOOMED; 3621 FL_UNLOCK(fl); 3622 mtx_unlock(&sc->sfl_lock); 3623 3624 callout_drain(&sc->sfl_callout); 3625 KASSERT((fl->flags & FL_STARVING) == 0, 3626 ("%s: still starving", __func__)); 3627 } 3628 3629 static int 3630 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 3631 driver_intr_t *handler, void *arg, char *name) 3632 { 3633 int rc; 3634 3635 irq->rid = rid; 3636 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 3637 RF_SHAREABLE | RF_ACTIVE); 3638 if (irq->res == NULL) { 3639 device_printf(sc->dev, 3640 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 3641 return (ENOMEM); 3642 } 3643 3644 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 3645 NULL, handler, arg, &irq->tag); 3646 if (rc != 0) { 3647 device_printf(sc->dev, 3648 "failed to setup interrupt for rid %d, name %s: %d\n", 3649 rid, name, rc); 3650 } else if (name) 3651 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 3652 3653 return (rc); 3654 } 3655 3656 static int 3657 t4_free_irq(struct adapter *sc, struct irq *irq) 3658 { 3659 if (irq->tag) 3660 bus_teardown_intr(sc->dev, irq->res, irq->tag); 3661 if (irq->res) 3662 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 3663 3664 bzero(irq, sizeof(*irq)); 3665 3666 return (0); 3667 } 3668 3669 static void 3670 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 3671 unsigned int end) 3672 { 3673 uint32_t *p = (uint32_t *)(buf + start); 3674 3675 for ( ; start <= end; start += sizeof(uint32_t)) 3676 *p++ = t4_read_reg(sc, start); 3677 } 3678 3679 static void 3680 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 3681 { 3682 int i, n; 3683 const unsigned int *reg_ranges; 3684 static const unsigned int t4_reg_ranges[] = { 3685 0x1008, 0x1108, 3686 0x1180, 0x11b4, 3687 0x11fc, 0x123c, 3688 0x1300, 0x173c, 3689 0x1800, 0x18fc, 3690 0x3000, 0x30d8, 3691 0x30e0, 0x5924, 3692 0x5960, 0x59d4, 3693 0x5a00, 0x5af8, 3694 0x6000, 0x6098, 3695 0x6100, 0x6150, 3696 0x6200, 0x6208, 3697 0x6240, 0x6248, 3698 0x6280, 0x6338, 3699 0x6370, 0x638c, 3700 0x6400, 0x643c, 3701 0x6500, 0x6524, 3702 0x6a00, 0x6a38, 3703 0x6a60, 0x6a78, 3704 0x6b00, 0x6b84, 3705 0x6bf0, 0x6c84, 3706 0x6cf0, 0x6d84, 3707 0x6df0, 0x6e84, 3708 0x6ef0, 0x6f84, 3709 0x6ff0, 0x7084, 3710 0x70f0, 0x7184, 3711 0x71f0, 0x7284, 3712 0x72f0, 0x7384, 3713 0x73f0, 0x7450, 3714 0x7500, 0x7530, 3715 0x7600, 0x761c, 3716 0x7680, 0x76cc, 3717 0x7700, 0x7798, 3718 0x77c0, 0x77fc, 3719 0x7900, 0x79fc, 3720 0x7b00, 0x7c38, 3721 0x7d00, 0x7efc, 3722 0x8dc0, 0x8e1c, 3723 0x8e30, 0x8e78, 3724 0x8ea0, 0x8f6c, 3725 0x8fc0, 0x9074, 3726 0x90fc, 0x90fc, 3727 0x9400, 0x9458, 3728 0x9600, 0x96bc, 3729 0x9800, 0x9808, 3730 0x9820, 0x983c, 3731 0x9850, 0x9864, 3732 0x9c00, 0x9c6c, 3733 0x9c80, 0x9cec, 3734 0x9d00, 0x9d6c, 3735 0x9d80, 0x9dec, 3736 0x9e00, 0x9e6c, 3737 0x9e80, 0x9eec, 3738 0x9f00, 0x9f6c, 3739 0x9f80, 0x9fec, 3740 0xd004, 0xd03c, 3741 0xdfc0, 0xdfe0, 3742 0xe000, 0xea7c, 3743 0xf000, 0x11110, 3744 0x11118, 0x11190, 3745 0x19040, 0x1906c, 3746 0x19078, 0x19080, 3747 0x1908c, 0x19124, 3748 0x19150, 0x191b0, 3749 0x191d0, 0x191e8, 3750 0x19238, 0x1924c, 3751 0x193f8, 0x19474, 3752 0x19490, 0x194f8, 3753 0x19800, 0x19f30, 3754 0x1a000, 0x1a06c, 3755 0x1a0b0, 0x1a120, 3756 0x1a128, 0x1a138, 3757 0x1a190, 0x1a1c4, 3758 0x1a1fc, 0x1a1fc, 3759 0x1e040, 0x1e04c, 3760 0x1e284, 0x1e28c, 3761 0x1e2c0, 0x1e2c0, 3762 0x1e2e0, 0x1e2e0, 3763 0x1e300, 0x1e384, 3764 0x1e3c0, 0x1e3c8, 3765 0x1e440, 0x1e44c, 3766 0x1e684, 0x1e68c, 3767 0x1e6c0, 0x1e6c0, 3768 0x1e6e0, 0x1e6e0, 3769 0x1e700, 0x1e784, 3770 0x1e7c0, 0x1e7c8, 3771 0x1e840, 0x1e84c, 3772 0x1ea84, 0x1ea8c, 3773 0x1eac0, 0x1eac0, 3774 0x1eae0, 0x1eae0, 3775 0x1eb00, 0x1eb84, 3776 0x1ebc0, 0x1ebc8, 3777 0x1ec40, 0x1ec4c, 3778 0x1ee84, 0x1ee8c, 3779 0x1eec0, 0x1eec0, 3780 0x1eee0, 0x1eee0, 3781 0x1ef00, 0x1ef84, 3782 0x1efc0, 0x1efc8, 3783 0x1f040, 0x1f04c, 3784 0x1f284, 0x1f28c, 3785 0x1f2c0, 0x1f2c0, 3786 0x1f2e0, 0x1f2e0, 3787 0x1f300, 0x1f384, 3788 0x1f3c0, 0x1f3c8, 3789 0x1f440, 0x1f44c, 3790 0x1f684, 0x1f68c, 3791 0x1f6c0, 0x1f6c0, 3792 0x1f6e0, 0x1f6e0, 3793 0x1f700, 0x1f784, 3794 0x1f7c0, 0x1f7c8, 3795 0x1f840, 0x1f84c, 3796 0x1fa84, 0x1fa8c, 3797 0x1fac0, 0x1fac0, 3798 0x1fae0, 0x1fae0, 3799 0x1fb00, 0x1fb84, 3800 0x1fbc0, 0x1fbc8, 3801 0x1fc40, 0x1fc4c, 3802 0x1fe84, 0x1fe8c, 3803 0x1fec0, 0x1fec0, 3804 0x1fee0, 0x1fee0, 3805 0x1ff00, 0x1ff84, 3806 0x1ffc0, 0x1ffc8, 3807 0x20000, 0x2002c, 3808 0x20100, 0x2013c, 3809 0x20190, 0x201c8, 3810 0x20200, 0x20318, 3811 0x20400, 0x20528, 3812 0x20540, 0x20614, 3813 0x21000, 0x21040, 3814 0x2104c, 0x21060, 3815 0x210c0, 0x210ec, 3816 0x21200, 0x21268, 3817 0x21270, 0x21284, 3818 0x212fc, 0x21388, 3819 0x21400, 0x21404, 3820 0x21500, 0x21518, 3821 0x2152c, 0x2153c, 3822 0x21550, 0x21554, 3823 0x21600, 0x21600, 3824 0x21608, 0x21628, 3825 0x21630, 0x2163c, 3826 0x21700, 0x2171c, 3827 0x21780, 0x2178c, 3828 0x21800, 0x21c38, 3829 0x21c80, 0x21d7c, 3830 0x21e00, 0x21e04, 3831 0x22000, 0x2202c, 3832 0x22100, 0x2213c, 3833 0x22190, 0x221c8, 3834 0x22200, 0x22318, 3835 0x22400, 0x22528, 3836 0x22540, 0x22614, 3837 0x23000, 0x23040, 3838 0x2304c, 0x23060, 3839 0x230c0, 0x230ec, 3840 0x23200, 0x23268, 3841 0x23270, 0x23284, 3842 0x232fc, 0x23388, 3843 0x23400, 0x23404, 3844 0x23500, 0x23518, 3845 0x2352c, 0x2353c, 3846 0x23550, 0x23554, 3847 0x23600, 0x23600, 3848 0x23608, 0x23628, 3849 0x23630, 0x2363c, 3850 0x23700, 0x2371c, 3851 0x23780, 0x2378c, 3852 0x23800, 0x23c38, 3853 0x23c80, 0x23d7c, 3854 0x23e00, 0x23e04, 3855 0x24000, 0x2402c, 3856 0x24100, 0x2413c, 3857 0x24190, 0x241c8, 3858 0x24200, 0x24318, 3859 0x24400, 0x24528, 3860 0x24540, 0x24614, 3861 0x25000, 0x25040, 3862 0x2504c, 0x25060, 3863 0x250c0, 0x250ec, 3864 0x25200, 0x25268, 3865 0x25270, 0x25284, 3866 0x252fc, 0x25388, 3867 0x25400, 0x25404, 3868 0x25500, 0x25518, 3869 0x2552c, 0x2553c, 3870 0x25550, 0x25554, 3871 0x25600, 0x25600, 3872 0x25608, 0x25628, 3873 0x25630, 0x2563c, 3874 0x25700, 0x2571c, 3875 0x25780, 0x2578c, 3876 0x25800, 0x25c38, 3877 0x25c80, 0x25d7c, 3878 0x25e00, 0x25e04, 3879 0x26000, 0x2602c, 3880 0x26100, 0x2613c, 3881 0x26190, 0x261c8, 3882 0x26200, 0x26318, 3883 0x26400, 0x26528, 3884 0x26540, 0x26614, 3885 0x27000, 0x27040, 3886 0x2704c, 0x27060, 3887 0x270c0, 0x270ec, 3888 0x27200, 0x27268, 3889 0x27270, 0x27284, 3890 0x272fc, 0x27388, 3891 0x27400, 0x27404, 3892 0x27500, 0x27518, 3893 0x2752c, 0x2753c, 3894 0x27550, 0x27554, 3895 0x27600, 0x27600, 3896 0x27608, 0x27628, 3897 0x27630, 0x2763c, 3898 0x27700, 0x2771c, 3899 0x27780, 0x2778c, 3900 0x27800, 0x27c38, 3901 0x27c80, 0x27d7c, 3902 0x27e00, 0x27e04 3903 }; 3904 static const unsigned int t5_reg_ranges[] = { 3905 0x1008, 0x1148, 3906 0x1180, 0x11b4, 3907 0x11fc, 0x123c, 3908 0x1280, 0x173c, 3909 0x1800, 0x18fc, 3910 0x3000, 0x3028, 3911 0x3060, 0x30d8, 3912 0x30e0, 0x30fc, 3913 0x3140, 0x357c, 3914 0x35a8, 0x35cc, 3915 0x35ec, 0x35ec, 3916 0x3600, 0x5624, 3917 0x56cc, 0x575c, 3918 0x580c, 0x5814, 3919 0x5890, 0x58bc, 3920 0x5940, 0x59dc, 3921 0x59fc, 0x5a18, 3922 0x5a60, 0x5a9c, 3923 0x5b94, 0x5bfc, 3924 0x6000, 0x6040, 3925 0x6058, 0x614c, 3926 0x7700, 0x7798, 3927 0x77c0, 0x78fc, 3928 0x7b00, 0x7c54, 3929 0x7d00, 0x7efc, 3930 0x8dc0, 0x8de0, 3931 0x8df8, 0x8e84, 3932 0x8ea0, 0x8f84, 3933 0x8fc0, 0x90f8, 3934 0x9400, 0x9470, 3935 0x9600, 0x96f4, 3936 0x9800, 0x9808, 3937 0x9820, 0x983c, 3938 0x9850, 0x9864, 3939 0x9c00, 0x9c6c, 3940 0x9c80, 0x9cec, 3941 0x9d00, 0x9d6c, 3942 0x9d80, 0x9dec, 3943 0x9e00, 0x9e6c, 3944 0x9e80, 0x9eec, 3945 0x9f00, 0x9f6c, 3946 0x9f80, 0xa020, 3947 0xd004, 0xd03c, 3948 0xdfc0, 0xdfe0, 3949 0xe000, 0x11088, 3950 0x1109c, 0x11110, 3951 0x11118, 0x1117c, 3952 0x11190, 0x11204, 3953 0x19040, 0x1906c, 3954 0x19078, 0x19080, 3955 0x1908c, 0x19124, 3956 0x19150, 0x191b0, 3957 0x191d0, 0x191e8, 3958 0x19238, 0x19290, 3959 0x193f8, 0x19474, 3960 0x19490, 0x194cc, 3961 0x194f0, 0x194f8, 3962 0x19c00, 0x19c60, 3963 0x19c94, 0x19e10, 3964 0x19e50, 0x19f34, 3965 0x19f40, 0x19f50, 3966 0x19f90, 0x19fe4, 3967 0x1a000, 0x1a06c, 3968 0x1a0b0, 0x1a120, 3969 0x1a128, 0x1a138, 3970 0x1a190, 0x1a1c4, 3971 0x1a1fc, 0x1a1fc, 3972 0x1e008, 0x1e00c, 3973 0x1e040, 0x1e04c, 3974 0x1e284, 0x1e290, 3975 0x1e2c0, 0x1e2c0, 3976 0x1e2e0, 0x1e2e0, 3977 0x1e300, 0x1e384, 3978 0x1e3c0, 0x1e3c8, 3979 0x1e408, 0x1e40c, 3980 0x1e440, 0x1e44c, 3981 0x1e684, 0x1e690, 3982 0x1e6c0, 0x1e6c0, 3983 0x1e6e0, 0x1e6e0, 3984 0x1e700, 0x1e784, 3985 0x1e7c0, 0x1e7c8, 3986 0x1e808, 0x1e80c, 3987 0x1e840, 0x1e84c, 3988 0x1ea84, 0x1ea90, 3989 0x1eac0, 0x1eac0, 3990 0x1eae0, 0x1eae0, 3991 0x1eb00, 0x1eb84, 3992 0x1ebc0, 0x1ebc8, 3993 0x1ec08, 0x1ec0c, 3994 0x1ec40, 0x1ec4c, 3995 0x1ee84, 0x1ee90, 3996 0x1eec0, 0x1eec0, 3997 0x1eee0, 0x1eee0, 3998 0x1ef00, 0x1ef84, 3999 0x1efc0, 0x1efc8, 4000 0x1f008, 0x1f00c, 4001 0x1f040, 0x1f04c, 4002 0x1f284, 0x1f290, 4003 0x1f2c0, 0x1f2c0, 4004 0x1f2e0, 0x1f2e0, 4005 0x1f300, 0x1f384, 4006 0x1f3c0, 0x1f3c8, 4007 0x1f408, 0x1f40c, 4008 0x1f440, 0x1f44c, 4009 0x1f684, 0x1f690, 4010 0x1f6c0, 0x1f6c0, 4011 0x1f6e0, 0x1f6e0, 4012 0x1f700, 0x1f784, 4013 0x1f7c0, 0x1f7c8, 4014 0x1f808, 0x1f80c, 4015 0x1f840, 0x1f84c, 4016 0x1fa84, 0x1fa90, 4017 0x1fac0, 0x1fac0, 4018 0x1fae0, 0x1fae0, 4019 0x1fb00, 0x1fb84, 4020 0x1fbc0, 0x1fbc8, 4021 0x1fc08, 0x1fc0c, 4022 0x1fc40, 0x1fc4c, 4023 0x1fe84, 0x1fe90, 4024 0x1fec0, 0x1fec0, 4025 0x1fee0, 0x1fee0, 4026 0x1ff00, 0x1ff84, 4027 0x1ffc0, 0x1ffc8, 4028 0x30000, 0x30030, 4029 0x30100, 0x30144, 4030 0x30190, 0x301d0, 4031 0x30200, 0x30318, 4032 0x30400, 0x3052c, 4033 0x30540, 0x3061c, 4034 0x30800, 0x30834, 4035 0x308c0, 0x30908, 4036 0x30910, 0x309ac, 4037 0x30a00, 0x30a2c, 4038 0x30a44, 0x30a50, 4039 0x30a74, 0x30c24, 4040 0x30d00, 0x30d00, 4041 0x30d08, 0x30d14, 4042 0x30d1c, 0x30d20, 4043 0x30d3c, 0x30d50, 4044 0x31200, 0x3120c, 4045 0x31220, 0x31220, 4046 0x31240, 0x31240, 4047 0x31600, 0x3160c, 4048 0x31a00, 0x31a1c, 4049 0x31e00, 0x31e20, 4050 0x31e38, 0x31e3c, 4051 0x31e80, 0x31e80, 4052 0x31e88, 0x31ea8, 4053 0x31eb0, 0x31eb4, 4054 0x31ec8, 0x31ed4, 4055 0x31fb8, 0x32004, 4056 0x32200, 0x32200, 4057 0x32208, 0x32240, 4058 0x32248, 0x32280, 4059 0x32288, 0x322c0, 4060 0x322c8, 0x322fc, 4061 0x32600, 0x32630, 4062 0x32a00, 0x32abc, 4063 0x32b00, 0x32b70, 4064 0x33000, 0x33048, 4065 0x33060, 0x3309c, 4066 0x330f0, 0x33148, 4067 0x33160, 0x3319c, 4068 0x331f0, 0x332e4, 4069 0x332f8, 0x333e4, 4070 0x333f8, 0x33448, 4071 0x33460, 0x3349c, 4072 0x334f0, 0x33548, 4073 0x33560, 0x3359c, 4074 0x335f0, 0x336e4, 4075 0x336f8, 0x337e4, 4076 0x337f8, 0x337fc, 4077 0x33814, 0x33814, 4078 0x3382c, 0x3382c, 4079 0x33880, 0x3388c, 4080 0x338e8, 0x338ec, 4081 0x33900, 0x33948, 4082 0x33960, 0x3399c, 4083 0x339f0, 0x33ae4, 4084 0x33af8, 0x33b10, 4085 0x33b28, 0x33b28, 4086 0x33b3c, 0x33b50, 4087 0x33bf0, 0x33c10, 4088 0x33c28, 0x33c28, 4089 0x33c3c, 0x33c50, 4090 0x33cf0, 0x33cfc, 4091 0x34000, 0x34030, 4092 0x34100, 0x34144, 4093 0x34190, 0x341d0, 4094 0x34200, 0x34318, 4095 0x34400, 0x3452c, 4096 0x34540, 0x3461c, 4097 0x34800, 0x34834, 4098 0x348c0, 0x34908, 4099 0x34910, 0x349ac, 4100 0x34a00, 0x34a2c, 4101 0x34a44, 0x34a50, 4102 0x34a74, 0x34c24, 4103 0x34d00, 0x34d00, 4104 0x34d08, 0x34d14, 4105 0x34d1c, 0x34d20, 4106 0x34d3c, 0x34d50, 4107 0x35200, 0x3520c, 4108 0x35220, 0x35220, 4109 0x35240, 0x35240, 4110 0x35600, 0x3560c, 4111 0x35a00, 0x35a1c, 4112 0x35e00, 0x35e20, 4113 0x35e38, 0x35e3c, 4114 0x35e80, 0x35e80, 4115 0x35e88, 0x35ea8, 4116 0x35eb0, 0x35eb4, 4117 0x35ec8, 0x35ed4, 4118 0x35fb8, 0x36004, 4119 0x36200, 0x36200, 4120 0x36208, 0x36240, 4121 0x36248, 0x36280, 4122 0x36288, 0x362c0, 4123 0x362c8, 0x362fc, 4124 0x36600, 0x36630, 4125 0x36a00, 0x36abc, 4126 0x36b00, 0x36b70, 4127 0x37000, 0x37048, 4128 0x37060, 0x3709c, 4129 0x370f0, 0x37148, 4130 0x37160, 0x3719c, 4131 0x371f0, 0x372e4, 4132 0x372f8, 0x373e4, 4133 0x373f8, 0x37448, 4134 0x37460, 0x3749c, 4135 0x374f0, 0x37548, 4136 0x37560, 0x3759c, 4137 0x375f0, 0x376e4, 4138 0x376f8, 0x377e4, 4139 0x377f8, 0x377fc, 4140 0x37814, 0x37814, 4141 0x3782c, 0x3782c, 4142 0x37880, 0x3788c, 4143 0x378e8, 0x378ec, 4144 0x37900, 0x37948, 4145 0x37960, 0x3799c, 4146 0x379f0, 0x37ae4, 4147 0x37af8, 0x37b10, 4148 0x37b28, 0x37b28, 4149 0x37b3c, 0x37b50, 4150 0x37bf0, 0x37c10, 4151 0x37c28, 0x37c28, 4152 0x37c3c, 0x37c50, 4153 0x37cf0, 0x37cfc, 4154 0x38000, 0x38030, 4155 0x38100, 0x38144, 4156 0x38190, 0x381d0, 4157 0x38200, 0x38318, 4158 0x38400, 0x3852c, 4159 0x38540, 0x3861c, 4160 0x38800, 0x38834, 4161 0x388c0, 0x38908, 4162 0x38910, 0x389ac, 4163 0x38a00, 0x38a2c, 4164 0x38a44, 0x38a50, 4165 0x38a74, 0x38c24, 4166 0x38d00, 0x38d00, 4167 0x38d08, 0x38d14, 4168 0x38d1c, 0x38d20, 4169 0x38d3c, 0x38d50, 4170 0x39200, 0x3920c, 4171 0x39220, 0x39220, 4172 0x39240, 0x39240, 4173 0x39600, 0x3960c, 4174 0x39a00, 0x39a1c, 4175 0x39e00, 0x39e20, 4176 0x39e38, 0x39e3c, 4177 0x39e80, 0x39e80, 4178 0x39e88, 0x39ea8, 4179 0x39eb0, 0x39eb4, 4180 0x39ec8, 0x39ed4, 4181 0x39fb8, 0x3a004, 4182 0x3a200, 0x3a200, 4183 0x3a208, 0x3a240, 4184 0x3a248, 0x3a280, 4185 0x3a288, 0x3a2c0, 4186 0x3a2c8, 0x3a2fc, 4187 0x3a600, 0x3a630, 4188 0x3aa00, 0x3aabc, 4189 0x3ab00, 0x3ab70, 4190 0x3b000, 0x3b048, 4191 0x3b060, 0x3b09c, 4192 0x3b0f0, 0x3b148, 4193 0x3b160, 0x3b19c, 4194 0x3b1f0, 0x3b2e4, 4195 0x3b2f8, 0x3b3e4, 4196 0x3b3f8, 0x3b448, 4197 0x3b460, 0x3b49c, 4198 0x3b4f0, 0x3b548, 4199 0x3b560, 0x3b59c, 4200 0x3b5f0, 0x3b6e4, 4201 0x3b6f8, 0x3b7e4, 4202 0x3b7f8, 0x3b7fc, 4203 0x3b814, 0x3b814, 4204 0x3b82c, 0x3b82c, 4205 0x3b880, 0x3b88c, 4206 0x3b8e8, 0x3b8ec, 4207 0x3b900, 0x3b948, 4208 0x3b960, 0x3b99c, 4209 0x3b9f0, 0x3bae4, 4210 0x3baf8, 0x3bb10, 4211 0x3bb28, 0x3bb28, 4212 0x3bb3c, 0x3bb50, 4213 0x3bbf0, 0x3bc10, 4214 0x3bc28, 0x3bc28, 4215 0x3bc3c, 0x3bc50, 4216 0x3bcf0, 0x3bcfc, 4217 0x3c000, 0x3c030, 4218 0x3c100, 0x3c144, 4219 0x3c190, 0x3c1d0, 4220 0x3c200, 0x3c318, 4221 0x3c400, 0x3c52c, 4222 0x3c540, 0x3c61c, 4223 0x3c800, 0x3c834, 4224 0x3c8c0, 0x3c908, 4225 0x3c910, 0x3c9ac, 4226 0x3ca00, 0x3ca2c, 4227 0x3ca44, 0x3ca50, 4228 0x3ca74, 0x3cc24, 4229 0x3cd00, 0x3cd00, 4230 0x3cd08, 0x3cd14, 4231 0x3cd1c, 0x3cd20, 4232 0x3cd3c, 0x3cd50, 4233 0x3d200, 0x3d20c, 4234 0x3d220, 0x3d220, 4235 0x3d240, 0x3d240, 4236 0x3d600, 0x3d60c, 4237 0x3da00, 0x3da1c, 4238 0x3de00, 0x3de20, 4239 0x3de38, 0x3de3c, 4240 0x3de80, 0x3de80, 4241 0x3de88, 0x3dea8, 4242 0x3deb0, 0x3deb4, 4243 0x3dec8, 0x3ded4, 4244 0x3dfb8, 0x3e004, 4245 0x3e200, 0x3e200, 4246 0x3e208, 0x3e240, 4247 0x3e248, 0x3e280, 4248 0x3e288, 0x3e2c0, 4249 0x3e2c8, 0x3e2fc, 4250 0x3e600, 0x3e630, 4251 0x3ea00, 0x3eabc, 4252 0x3eb00, 0x3eb70, 4253 0x3f000, 0x3f048, 4254 0x3f060, 0x3f09c, 4255 0x3f0f0, 0x3f148, 4256 0x3f160, 0x3f19c, 4257 0x3f1f0, 0x3f2e4, 4258 0x3f2f8, 0x3f3e4, 4259 0x3f3f8, 0x3f448, 4260 0x3f460, 0x3f49c, 4261 0x3f4f0, 0x3f548, 4262 0x3f560, 0x3f59c, 4263 0x3f5f0, 0x3f6e4, 4264 0x3f6f8, 0x3f7e4, 4265 0x3f7f8, 0x3f7fc, 4266 0x3f814, 0x3f814, 4267 0x3f82c, 0x3f82c, 4268 0x3f880, 0x3f88c, 4269 0x3f8e8, 0x3f8ec, 4270 0x3f900, 0x3f948, 4271 0x3f960, 0x3f99c, 4272 0x3f9f0, 0x3fae4, 4273 0x3faf8, 0x3fb10, 4274 0x3fb28, 0x3fb28, 4275 0x3fb3c, 0x3fb50, 4276 0x3fbf0, 0x3fc10, 4277 0x3fc28, 0x3fc28, 4278 0x3fc3c, 0x3fc50, 4279 0x3fcf0, 0x3fcfc, 4280 0x40000, 0x4000c, 4281 0x40040, 0x40068, 4282 0x4007c, 0x40144, 4283 0x40180, 0x4018c, 4284 0x40200, 0x40298, 4285 0x402ac, 0x4033c, 4286 0x403f8, 0x403fc, 4287 0x41304, 0x413c4, 4288 0x41400, 0x4141c, 4289 0x41480, 0x414d0, 4290 0x44000, 0x44078, 4291 0x440c0, 0x44278, 4292 0x442c0, 0x44478, 4293 0x444c0, 0x44678, 4294 0x446c0, 0x44878, 4295 0x448c0, 0x449fc, 4296 0x45000, 0x45068, 4297 0x45080, 0x45084, 4298 0x450a0, 0x450b0, 4299 0x45200, 0x45268, 4300 0x45280, 0x45284, 4301 0x452a0, 0x452b0, 4302 0x460c0, 0x460e4, 4303 0x47000, 0x4708c, 4304 0x47200, 0x47250, 4305 0x47400, 0x47420, 4306 0x47600, 0x47618, 4307 0x47800, 0x47814, 4308 0x48000, 0x4800c, 4309 0x48040, 0x48068, 4310 0x4807c, 0x48144, 4311 0x48180, 0x4818c, 4312 0x48200, 0x48298, 4313 0x482ac, 0x4833c, 4314 0x483f8, 0x483fc, 4315 0x49304, 0x493c4, 4316 0x49400, 0x4941c, 4317 0x49480, 0x494d0, 4318 0x4c000, 0x4c078, 4319 0x4c0c0, 0x4c278, 4320 0x4c2c0, 0x4c478, 4321 0x4c4c0, 0x4c678, 4322 0x4c6c0, 0x4c878, 4323 0x4c8c0, 0x4c9fc, 4324 0x4d000, 0x4d068, 4325 0x4d080, 0x4d084, 4326 0x4d0a0, 0x4d0b0, 4327 0x4d200, 0x4d268, 4328 0x4d280, 0x4d284, 4329 0x4d2a0, 0x4d2b0, 4330 0x4e0c0, 0x4e0e4, 4331 0x4f000, 0x4f08c, 4332 0x4f200, 0x4f250, 4333 0x4f400, 0x4f420, 4334 0x4f600, 0x4f618, 4335 0x4f800, 0x4f814, 4336 0x50000, 0x500cc, 4337 0x50400, 0x50400, 4338 0x50800, 0x508cc, 4339 0x50c00, 0x50c00, 4340 0x51000, 0x5101c, 4341 0x51300, 0x51308, 4342 }; 4343 4344 if (is_t4(sc)) { 4345 reg_ranges = &t4_reg_ranges[0]; 4346 n = nitems(t4_reg_ranges); 4347 } else { 4348 reg_ranges = &t5_reg_ranges[0]; 4349 n = nitems(t5_reg_ranges); 4350 } 4351 4352 regs->version = chip_id(sc) | chip_rev(sc) << 10; 4353 for (i = 0; i < n; i += 2) 4354 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 4355 } 4356 4357 static void 4358 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi) 4359 { 4360 int i; 4361 u_int v, tnl_cong_drops; 4362 struct timeval tv; 4363 const struct timeval interval = {0, 250000}; /* 250ms */ 4364 4365 getmicrotime(&tv); 4366 timevalsub(&tv, &interval); 4367 if (timevalcmp(&tv, &pi->last_refreshed, <)) 4368 return; 4369 4370 tnl_cong_drops = 0; 4371 t4_get_port_stats(sc, pi->tx_chan, &pi->stats); 4372 for (i = 0; i < NCHAN; i++) { 4373 if (pi->rx_chan_map & (1 << i)) { 4374 mtx_lock(&sc->regwin_lock); 4375 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 4376 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 4377 mtx_unlock(&sc->regwin_lock); 4378 tnl_cong_drops += v; 4379 } 4380 } 4381 pi->tnl_cong_drops = tnl_cong_drops; 4382 getmicrotime(&pi->last_refreshed); 4383 } 4384 4385 static void 4386 cxgbe_tick(void *arg) 4387 { 4388 struct port_info *pi = arg; 4389 struct adapter *sc = pi->adapter; 4390 struct ifnet *ifp = pi->ifp; 4391 4392 PORT_LOCK(pi); 4393 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 4394 PORT_UNLOCK(pi); 4395 return; /* without scheduling another callout */ 4396 } 4397 4398 cxgbe_refresh_stats(sc, pi); 4399 4400 callout_schedule(&pi->tick, hz); 4401 PORT_UNLOCK(pi); 4402 } 4403 4404 static void 4405 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 4406 { 4407 struct ifnet *vlan; 4408 4409 if (arg != ifp || ifp->if_type != IFT_ETHER) 4410 return; 4411 4412 vlan = VLAN_DEVAT(ifp, vid); 4413 VLAN_SETCOOKIE(vlan, ifp); 4414 } 4415 4416 static int 4417 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 4418 { 4419 4420 #ifdef INVARIANTS 4421 panic("%s: opcode 0x%02x on iq %p with payload %p", 4422 __func__, rss->opcode, iq, m); 4423 #else 4424 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 4425 __func__, rss->opcode, iq, m); 4426 m_freem(m); 4427 #endif 4428 return (EDOOFUS); 4429 } 4430 4431 int 4432 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 4433 { 4434 uintptr_t *loc, new; 4435 4436 if (opcode >= nitems(sc->cpl_handler)) 4437 return (EINVAL); 4438 4439 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 4440 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 4441 atomic_store_rel_ptr(loc, new); 4442 4443 return (0); 4444 } 4445 4446 static int 4447 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 4448 { 4449 4450 #ifdef INVARIANTS 4451 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 4452 #else 4453 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 4454 __func__, iq, ctrl); 4455 #endif 4456 return (EDOOFUS); 4457 } 4458 4459 int 4460 t4_register_an_handler(struct adapter *sc, an_handler_t h) 4461 { 4462 uintptr_t *loc, new; 4463 4464 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 4465 loc = (uintptr_t *) &sc->an_handler; 4466 atomic_store_rel_ptr(loc, new); 4467 4468 return (0); 4469 } 4470 4471 static int 4472 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 4473 { 4474 const struct cpl_fw6_msg *cpl = 4475 __containerof(rpl, struct cpl_fw6_msg, data[0]); 4476 4477 #ifdef INVARIANTS 4478 panic("%s: fw_msg type %d", __func__, cpl->type); 4479 #else 4480 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 4481 #endif 4482 return (EDOOFUS); 4483 } 4484 4485 int 4486 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 4487 { 4488 uintptr_t *loc, new; 4489 4490 if (type >= nitems(sc->fw_msg_handler)) 4491 return (EINVAL); 4492 4493 /* 4494 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 4495 * handler dispatch table. Reject any attempt to install a handler for 4496 * this subtype. 4497 */ 4498 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 4499 return (EINVAL); 4500 4501 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 4502 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 4503 atomic_store_rel_ptr(loc, new); 4504 4505 return (0); 4506 } 4507 4508 static int 4509 t4_sysctls(struct adapter *sc) 4510 { 4511 struct sysctl_ctx_list *ctx; 4512 struct sysctl_oid *oid; 4513 struct sysctl_oid_list *children, *c0; 4514 static char *caps[] = { 4515 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 4516 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL" /* caps[1] niccaps */ 4517 "\6HASHFILTER\7ETHOFLD", 4518 "\20\1TOE", /* caps[2] toecaps */ 4519 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 4520 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 4521 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 4522 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 4523 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 4524 "\4PO_INITIAOR\5PO_TARGET" 4525 }; 4526 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 4527 4528 ctx = device_get_sysctl_ctx(sc->dev); 4529 4530 /* 4531 * dev.t4nex.X. 4532 */ 4533 oid = device_get_sysctl_tree(sc->dev); 4534 c0 = children = SYSCTL_CHILDREN(oid); 4535 4536 sc->sc_do_rxcopy = 1; 4537 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, 4538 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); 4539 4540 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 4541 sc->params.nports, "# of ports"); 4542 4543 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 4544 NULL, chip_rev(sc), "chip hardware revision"); 4545 4546 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 4547 CTLFLAG_RD, sc->fw_version, 0, "firmware version"); 4548 4549 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 4550 CTLFLAG_RD, sc->cfg_file, 0, "configuration file"); 4551 4552 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 4553 sc->cfcsum, "config file checksum"); 4554 4555 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 4556 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 4557 sysctl_bitfield, "A", "available doorbells"); 4558 4559 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 4560 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 4561 sysctl_bitfield, "A", "available link capabilities"); 4562 4563 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 4564 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 4565 sysctl_bitfield, "A", "available NIC capabilities"); 4566 4567 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 4568 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 4569 sysctl_bitfield, "A", "available TCP offload capabilities"); 4570 4571 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 4572 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 4573 sysctl_bitfield, "A", "available RDMA capabilities"); 4574 4575 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 4576 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 4577 sysctl_bitfield, "A", "available iSCSI capabilities"); 4578 4579 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 4580 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 4581 sysctl_bitfield, "A", "available FCoE capabilities"); 4582 4583 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 4584 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 4585 4586 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 4587 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 4588 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 4589 "interrupt holdoff timer values (us)"); 4590 4591 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 4592 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 4593 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 4594 "interrupt holdoff packet counter values"); 4595 4596 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, 4597 NULL, sc->tids.nftids, "number of filters"); 4598 4599 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | 4600 CTLFLAG_RD, sc, 0, sysctl_temperature, "I", 4601 "chip temperature (in Celsius)"); 4602 4603 t4_sge_sysctls(sc, ctx, children); 4604 4605 sc->lro_timeout = 100; 4606 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, 4607 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); 4608 4609 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "debug_flags", CTLFLAG_RW, 4610 &sc->debug_flags, 0, "flags to enable runtime debugging"); 4611 4612 #ifdef SBUF_DRAIN 4613 /* 4614 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 4615 */ 4616 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 4617 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 4618 "logs and miscellaneous information"); 4619 children = SYSCTL_CHILDREN(oid); 4620 4621 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 4622 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4623 sysctl_cctrl, "A", "congestion control"); 4624 4625 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 4626 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4627 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 4628 4629 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 4630 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 4631 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 4632 4633 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 4634 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 4635 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 4636 4637 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 4638 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 4639 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 4640 4641 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 4642 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 4643 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 4644 4645 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 4646 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 4647 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 4648 4649 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 4650 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4651 sysctl_cim_la, "A", "CIM logic analyzer"); 4652 4653 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", 4654 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4655 sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); 4656 4657 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 4658 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 4659 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 4660 4661 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 4662 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 4663 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 4664 4665 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 4666 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 4667 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 4668 4669 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 4670 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 4671 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 4672 4673 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 4674 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 4675 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 4676 4677 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 4678 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 4679 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 4680 4681 if (is_t5(sc)) { 4682 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 4683 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 4684 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 4685 4686 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 4687 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 4688 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 4689 } 4690 4691 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", 4692 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4693 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); 4694 4695 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 4696 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4697 sysctl_cim_qcfg, "A", "CIM queue configuration"); 4698 4699 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 4700 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4701 sysctl_cpl_stats, "A", "CPL statistics"); 4702 4703 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 4704 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4705 sysctl_ddp_stats, "A", "non-TCP DDP statistics"); 4706 4707 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 4708 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4709 sysctl_devlog, "A", "firmware's device log"); 4710 4711 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 4712 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4713 sysctl_fcoe_stats, "A", "FCoE statistics"); 4714 4715 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 4716 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4717 sysctl_hw_sched, "A", "hardware scheduler "); 4718 4719 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 4720 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4721 sysctl_l2t, "A", "hardware L2 table"); 4722 4723 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 4724 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4725 sysctl_lb_stats, "A", "loopback statistics"); 4726 4727 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 4728 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4729 sysctl_meminfo, "A", "memory regions"); 4730 4731 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", 4732 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4733 sysctl_mps_tcam, "A", "MPS TCAM entries"); 4734 4735 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 4736 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4737 sysctl_path_mtus, "A", "path MTUs"); 4738 4739 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 4740 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4741 sysctl_pm_stats, "A", "PM statistics"); 4742 4743 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 4744 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4745 sysctl_rdma_stats, "A", "RDMA statistics"); 4746 4747 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 4748 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4749 sysctl_tcp_stats, "A", "TCP statistics"); 4750 4751 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 4752 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4753 sysctl_tids, "A", "TID information"); 4754 4755 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 4756 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4757 sysctl_tp_err_stats, "A", "TP error statistics"); 4758 4759 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", 4760 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4761 sysctl_tp_la, "A", "TP logic analyzer"); 4762 4763 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 4764 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4765 sysctl_tx_rate, "A", "Tx rate"); 4766 4767 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", 4768 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4769 sysctl_ulprx_la, "A", "ULPRX logic analyzer"); 4770 4771 if (is_t5(sc)) { 4772 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 4773 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4774 sysctl_wcwr_stats, "A", "write combined work requests"); 4775 } 4776 #endif 4777 4778 #ifdef TCP_OFFLOAD 4779 if (is_offload(sc)) { 4780 /* 4781 * dev.t4nex.X.toe. 4782 */ 4783 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 4784 NULL, "TOE parameters"); 4785 children = SYSCTL_CHILDREN(oid); 4786 4787 sc->tt.sndbuf = 256 * 1024; 4788 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 4789 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 4790 4791 sc->tt.ddp = 0; 4792 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 4793 &sc->tt.ddp, 0, "DDP allowed"); 4794 4795 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 4796 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 4797 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 4798 4799 sc->tt.ddp_thres = 4800 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 4801 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 4802 &sc->tt.ddp_thres, 0, "DDP threshold"); 4803 4804 sc->tt.rx_coalesce = 1; 4805 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", 4806 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); 4807 4808 sc->tt.tx_align = 1; 4809 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align", 4810 CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload"); 4811 } 4812 #endif 4813 4814 4815 return (0); 4816 } 4817 4818 static int 4819 cxgbe_sysctls(struct port_info *pi) 4820 { 4821 struct sysctl_ctx_list *ctx; 4822 struct sysctl_oid *oid; 4823 struct sysctl_oid_list *children; 4824 struct adapter *sc = pi->adapter; 4825 4826 ctx = device_get_sysctl_ctx(pi->dev); 4827 4828 /* 4829 * dev.cxgbe.X. 4830 */ 4831 oid = device_get_sysctl_tree(pi->dev); 4832 children = SYSCTL_CHILDREN(oid); 4833 4834 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | 4835 CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); 4836 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { 4837 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 4838 CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I", 4839 "PHY temperature (in Celsius)"); 4840 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", 4841 CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I", 4842 "PHY firmware version"); 4843 } 4844 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 4845 &pi->nrxq, 0, "# of rx queues"); 4846 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 4847 &pi->ntxq, 0, "# of tx queues"); 4848 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 4849 &pi->first_rxq, 0, "index of first rx queue"); 4850 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 4851 &pi->first_txq, 0, "index of first tx queue"); 4852 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT | 4853 CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU", 4854 "Reserve queue 0 for non-flowid packets"); 4855 4856 #ifdef TCP_OFFLOAD 4857 if (is_offload(sc)) { 4858 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 4859 &pi->nofldrxq, 0, 4860 "# of rx queues for offloaded TCP connections"); 4861 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 4862 &pi->nofldtxq, 0, 4863 "# of tx queues for offloaded TCP connections"); 4864 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 4865 CTLFLAG_RD, &pi->first_ofld_rxq, 0, 4866 "index of first TOE rx queue"); 4867 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 4868 CTLFLAG_RD, &pi->first_ofld_txq, 0, 4869 "index of first TOE tx queue"); 4870 } 4871 #endif 4872 #ifdef DEV_NETMAP 4873 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD, 4874 &pi->nnmrxq, 0, "# of rx queues for netmap"); 4875 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD, 4876 &pi->nnmtxq, 0, "# of tx queues for netmap"); 4877 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq", 4878 CTLFLAG_RD, &pi->first_nm_rxq, 0, 4879 "index of first netmap rx queue"); 4880 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq", 4881 CTLFLAG_RD, &pi->first_nm_txq, 0, 4882 "index of first netmap tx queue"); 4883 #endif 4884 4885 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 4886 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 4887 "holdoff timer index"); 4888 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 4889 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 4890 "holdoff packet counter index"); 4891 4892 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 4893 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 4894 "rx queue size"); 4895 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 4896 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 4897 "tx queue size"); 4898 4899 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings", 4900 CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings, 4901 "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)"); 4902 4903 /* 4904 * dev.cxgbe.X.stats. 4905 */ 4906 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 4907 NULL, "port statistics"); 4908 children = SYSCTL_CHILDREN(oid); 4909 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD, 4910 &pi->tx_parse_error, 0, 4911 "# of tx packets with invalid length or # of segments"); 4912 4913 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 4914 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 4915 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ 4916 sysctl_handle_t4_reg64, "QU", desc) 4917 4918 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 4919 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 4920 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 4921 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 4922 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 4923 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 4924 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 4925 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 4926 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 4927 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 4928 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 4929 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 4930 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 4931 "# of tx frames in this range", 4932 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 4933 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 4934 "# of tx frames in this range", 4935 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 4936 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 4937 "# of tx frames in this range", 4938 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 4939 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 4940 "# of tx frames in this range", 4941 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 4942 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 4943 "# of tx frames in this range", 4944 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 4945 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 4946 "# of tx frames in this range", 4947 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 4948 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 4949 "# of tx frames in this range", 4950 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 4951 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 4952 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 4953 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 4954 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 4955 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 4956 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 4957 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 4958 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 4959 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 4960 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 4961 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 4962 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 4963 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 4964 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 4965 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 4966 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 4967 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 4968 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 4969 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 4970 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 4971 4972 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 4973 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 4974 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 4975 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 4976 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 4977 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 4978 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 4979 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 4980 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 4981 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 4982 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 4983 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 4984 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 4985 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 4986 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 4987 "# of frames received with bad FCS", 4988 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 4989 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 4990 "# of frames received with length error", 4991 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 4992 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 4993 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 4994 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 4995 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 4996 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 4997 "# of rx frames in this range", 4998 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 4999 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 5000 "# of rx frames in this range", 5001 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 5002 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 5003 "# of rx frames in this range", 5004 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 5005 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 5006 "# of rx frames in this range", 5007 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 5008 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 5009 "# of rx frames in this range", 5010 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 5011 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 5012 "# of rx frames in this range", 5013 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 5014 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 5015 "# of rx frames in this range", 5016 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 5017 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 5018 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 5019 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 5020 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 5021 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 5022 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 5023 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 5024 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 5025 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 5026 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 5027 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 5028 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 5029 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 5030 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 5031 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 5032 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 5033 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 5034 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 5035 5036 #undef SYSCTL_ADD_T4_REG64 5037 5038 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 5039 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 5040 &pi->stats.name, desc) 5041 5042 /* We get these from port_stats and they may be stale by upto 1s */ 5043 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 5044 "# drops due to buffer-group 0 overflows"); 5045 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 5046 "# drops due to buffer-group 1 overflows"); 5047 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 5048 "# drops due to buffer-group 2 overflows"); 5049 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 5050 "# drops due to buffer-group 3 overflows"); 5051 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 5052 "# of buffer-group 0 truncated packets"); 5053 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 5054 "# of buffer-group 1 truncated packets"); 5055 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 5056 "# of buffer-group 2 truncated packets"); 5057 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 5058 "# of buffer-group 3 truncated packets"); 5059 5060 #undef SYSCTL_ADD_T4_PORTSTAT 5061 5062 return (0); 5063 } 5064 5065 static int 5066 sysctl_int_array(SYSCTL_HANDLER_ARGS) 5067 { 5068 int rc, *i, space = 0; 5069 struct sbuf sb; 5070 5071 sbuf_new_for_sysctl(&sb, NULL, 64, req); 5072 for (i = arg1; arg2; arg2 -= sizeof(int), i++) { 5073 if (space) 5074 sbuf_printf(&sb, " "); 5075 sbuf_printf(&sb, "%d", *i); 5076 space = 1; 5077 } 5078 rc = sbuf_finish(&sb); 5079 sbuf_delete(&sb); 5080 return (rc); 5081 } 5082 5083 static int 5084 sysctl_bitfield(SYSCTL_HANDLER_ARGS) 5085 { 5086 int rc; 5087 struct sbuf *sb; 5088 5089 rc = sysctl_wire_old_buffer(req, 0); 5090 if (rc != 0) 5091 return(rc); 5092 5093 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5094 if (sb == NULL) 5095 return (ENOMEM); 5096 5097 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 5098 rc = sbuf_finish(sb); 5099 sbuf_delete(sb); 5100 5101 return (rc); 5102 } 5103 5104 static int 5105 sysctl_btphy(SYSCTL_HANDLER_ARGS) 5106 { 5107 struct port_info *pi = arg1; 5108 int op = arg2; 5109 struct adapter *sc = pi->adapter; 5110 u_int v; 5111 int rc; 5112 5113 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt"); 5114 if (rc) 5115 return (rc); 5116 /* XXX: magic numbers */ 5117 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, 5118 &v); 5119 end_synchronized_op(sc, 0); 5120 if (rc) 5121 return (rc); 5122 if (op == 0) 5123 v /= 256; 5124 5125 rc = sysctl_handle_int(oidp, &v, 0, req); 5126 return (rc); 5127 } 5128 5129 static int 5130 sysctl_noflowq(SYSCTL_HANDLER_ARGS) 5131 { 5132 struct port_info *pi = arg1; 5133 int rc, val; 5134 5135 val = pi->rsrv_noflowq; 5136 rc = sysctl_handle_int(oidp, &val, 0, req); 5137 if (rc != 0 || req->newptr == NULL) 5138 return (rc); 5139 5140 if ((val >= 1) && (pi->ntxq > 1)) 5141 pi->rsrv_noflowq = 1; 5142 else 5143 pi->rsrv_noflowq = 0; 5144 5145 return (rc); 5146 } 5147 5148 static int 5149 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 5150 { 5151 struct port_info *pi = arg1; 5152 struct adapter *sc = pi->adapter; 5153 int idx, rc, i; 5154 struct sge_rxq *rxq; 5155 #ifdef TCP_OFFLOAD 5156 struct sge_ofld_rxq *ofld_rxq; 5157 #endif 5158 uint8_t v; 5159 5160 idx = pi->tmr_idx; 5161 5162 rc = sysctl_handle_int(oidp, &idx, 0, req); 5163 if (rc != 0 || req->newptr == NULL) 5164 return (rc); 5165 5166 if (idx < 0 || idx >= SGE_NTIMERS) 5167 return (EINVAL); 5168 5169 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5170 "t4tmr"); 5171 if (rc) 5172 return (rc); 5173 5174 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 5175 for_each_rxq(pi, i, rxq) { 5176 #ifdef atomic_store_rel_8 5177 atomic_store_rel_8(&rxq->iq.intr_params, v); 5178 #else 5179 rxq->iq.intr_params = v; 5180 #endif 5181 } 5182 #ifdef TCP_OFFLOAD 5183 for_each_ofld_rxq(pi, i, ofld_rxq) { 5184 #ifdef atomic_store_rel_8 5185 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); 5186 #else 5187 ofld_rxq->iq.intr_params = v; 5188 #endif 5189 } 5190 #endif 5191 pi->tmr_idx = idx; 5192 5193 end_synchronized_op(sc, LOCK_HELD); 5194 return (0); 5195 } 5196 5197 static int 5198 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 5199 { 5200 struct port_info *pi = arg1; 5201 struct adapter *sc = pi->adapter; 5202 int idx, rc; 5203 5204 idx = pi->pktc_idx; 5205 5206 rc = sysctl_handle_int(oidp, &idx, 0, req); 5207 if (rc != 0 || req->newptr == NULL) 5208 return (rc); 5209 5210 if (idx < -1 || idx >= SGE_NCOUNTERS) 5211 return (EINVAL); 5212 5213 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5214 "t4pktc"); 5215 if (rc) 5216 return (rc); 5217 5218 if (pi->flags & PORT_INIT_DONE) 5219 rc = EBUSY; /* cannot be changed once the queues are created */ 5220 else 5221 pi->pktc_idx = idx; 5222 5223 end_synchronized_op(sc, LOCK_HELD); 5224 return (rc); 5225 } 5226 5227 static int 5228 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 5229 { 5230 struct port_info *pi = arg1; 5231 struct adapter *sc = pi->adapter; 5232 int qsize, rc; 5233 5234 qsize = pi->qsize_rxq; 5235 5236 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5237 if (rc != 0 || req->newptr == NULL) 5238 return (rc); 5239 5240 if (qsize < 128 || (qsize & 7)) 5241 return (EINVAL); 5242 5243 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5244 "t4rxqs"); 5245 if (rc) 5246 return (rc); 5247 5248 if (pi->flags & PORT_INIT_DONE) 5249 rc = EBUSY; /* cannot be changed once the queues are created */ 5250 else 5251 pi->qsize_rxq = qsize; 5252 5253 end_synchronized_op(sc, LOCK_HELD); 5254 return (rc); 5255 } 5256 5257 static int 5258 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 5259 { 5260 struct port_info *pi = arg1; 5261 struct adapter *sc = pi->adapter; 5262 int qsize, rc; 5263 5264 qsize = pi->qsize_txq; 5265 5266 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5267 if (rc != 0 || req->newptr == NULL) 5268 return (rc); 5269 5270 if (qsize < 128 || qsize > 65536) 5271 return (EINVAL); 5272 5273 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5274 "t4txqs"); 5275 if (rc) 5276 return (rc); 5277 5278 if (pi->flags & PORT_INIT_DONE) 5279 rc = EBUSY; /* cannot be changed once the queues are created */ 5280 else 5281 pi->qsize_txq = qsize; 5282 5283 end_synchronized_op(sc, LOCK_HELD); 5284 return (rc); 5285 } 5286 5287 static int 5288 sysctl_pause_settings(SYSCTL_HANDLER_ARGS) 5289 { 5290 struct port_info *pi = arg1; 5291 struct adapter *sc = pi->adapter; 5292 struct link_config *lc = &pi->link_cfg; 5293 int rc; 5294 5295 if (req->newptr == NULL) { 5296 struct sbuf *sb; 5297 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX"; 5298 5299 rc = sysctl_wire_old_buffer(req, 0); 5300 if (rc != 0) 5301 return(rc); 5302 5303 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5304 if (sb == NULL) 5305 return (ENOMEM); 5306 5307 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits); 5308 rc = sbuf_finish(sb); 5309 sbuf_delete(sb); 5310 } else { 5311 char s[2]; 5312 int n; 5313 5314 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX)); 5315 s[1] = 0; 5316 5317 rc = sysctl_handle_string(oidp, s, sizeof(s), req); 5318 if (rc != 0) 5319 return(rc); 5320 5321 if (s[1] != 0) 5322 return (EINVAL); 5323 if (s[0] < '0' || s[0] > '9') 5324 return (EINVAL); /* not a number */ 5325 n = s[0] - '0'; 5326 if (n & ~(PAUSE_TX | PAUSE_RX)) 5327 return (EINVAL); /* some other bit is set too */ 5328 5329 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE"); 5330 if (rc) 5331 return (rc); 5332 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) { 5333 int link_ok = lc->link_ok; 5334 5335 lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX); 5336 lc->requested_fc |= n; 5337 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc); 5338 lc->link_ok = link_ok; /* restore */ 5339 } 5340 end_synchronized_op(sc, 0); 5341 } 5342 5343 return (rc); 5344 } 5345 5346 static int 5347 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 5348 { 5349 struct adapter *sc = arg1; 5350 int reg = arg2; 5351 uint64_t val; 5352 5353 val = t4_read_reg64(sc, reg); 5354 5355 return (sysctl_handle_64(oidp, &val, 0, req)); 5356 } 5357 5358 static int 5359 sysctl_temperature(SYSCTL_HANDLER_ARGS) 5360 { 5361 struct adapter *sc = arg1; 5362 int rc, t; 5363 uint32_t param, val; 5364 5365 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); 5366 if (rc) 5367 return (rc); 5368 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 5369 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 5370 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); 5371 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 5372 end_synchronized_op(sc, 0); 5373 if (rc) 5374 return (rc); 5375 5376 /* unknown is returned as 0 but we display -1 in that case */ 5377 t = val == 0 ? -1 : val; 5378 5379 rc = sysctl_handle_int(oidp, &t, 0, req); 5380 return (rc); 5381 } 5382 5383 #ifdef SBUF_DRAIN 5384 static int 5385 sysctl_cctrl(SYSCTL_HANDLER_ARGS) 5386 { 5387 struct adapter *sc = arg1; 5388 struct sbuf *sb; 5389 int rc, i; 5390 uint16_t incr[NMTUS][NCCTRL_WIN]; 5391 static const char *dec_fac[] = { 5392 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 5393 "0.9375" 5394 }; 5395 5396 rc = sysctl_wire_old_buffer(req, 0); 5397 if (rc != 0) 5398 return (rc); 5399 5400 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5401 if (sb == NULL) 5402 return (ENOMEM); 5403 5404 t4_read_cong_tbl(sc, incr); 5405 5406 for (i = 0; i < NCCTRL_WIN; ++i) { 5407 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 5408 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 5409 incr[5][i], incr[6][i], incr[7][i]); 5410 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 5411 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 5412 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 5413 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 5414 } 5415 5416 rc = sbuf_finish(sb); 5417 sbuf_delete(sb); 5418 5419 return (rc); 5420 } 5421 5422 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 5423 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 5424 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 5425 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 5426 }; 5427 5428 static int 5429 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 5430 { 5431 struct adapter *sc = arg1; 5432 struct sbuf *sb; 5433 int rc, i, n, qid = arg2; 5434 uint32_t *buf, *p; 5435 char *qtype; 5436 u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5; 5437 5438 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 5439 ("%s: bad qid %d\n", __func__, qid)); 5440 5441 if (qid < CIM_NUM_IBQ) { 5442 /* inbound queue */ 5443 qtype = "IBQ"; 5444 n = 4 * CIM_IBQ_SIZE; 5445 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5446 rc = t4_read_cim_ibq(sc, qid, buf, n); 5447 } else { 5448 /* outbound queue */ 5449 qtype = "OBQ"; 5450 qid -= CIM_NUM_IBQ; 5451 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 5452 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5453 rc = t4_read_cim_obq(sc, qid, buf, n); 5454 } 5455 5456 if (rc < 0) { 5457 rc = -rc; 5458 goto done; 5459 } 5460 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 5461 5462 rc = sysctl_wire_old_buffer(req, 0); 5463 if (rc != 0) 5464 goto done; 5465 5466 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5467 if (sb == NULL) { 5468 rc = ENOMEM; 5469 goto done; 5470 } 5471 5472 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 5473 for (i = 0, p = buf; i < n; i += 16, p += 4) 5474 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 5475 p[2], p[3]); 5476 5477 rc = sbuf_finish(sb); 5478 sbuf_delete(sb); 5479 done: 5480 free(buf, M_CXGBE); 5481 return (rc); 5482 } 5483 5484 static int 5485 sysctl_cim_la(SYSCTL_HANDLER_ARGS) 5486 { 5487 struct adapter *sc = arg1; 5488 u_int cfg; 5489 struct sbuf *sb; 5490 uint32_t *buf, *p; 5491 int rc; 5492 5493 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 5494 if (rc != 0) 5495 return (rc); 5496 5497 rc = sysctl_wire_old_buffer(req, 0); 5498 if (rc != 0) 5499 return (rc); 5500 5501 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5502 if (sb == NULL) 5503 return (ENOMEM); 5504 5505 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 5506 M_ZERO | M_WAITOK); 5507 5508 rc = -t4_cim_read_la(sc, buf, NULL); 5509 if (rc != 0) 5510 goto done; 5511 5512 sbuf_printf(sb, "Status Data PC%s", 5513 cfg & F_UPDBGLACAPTPCONLY ? "" : 5514 " LS0Stat LS0Addr LS0Data"); 5515 5516 KASSERT((sc->params.cim_la_size & 7) == 0, 5517 ("%s: p will walk off the end of buf", __func__)); 5518 5519 for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) { 5520 if (cfg & F_UPDBGLACAPTPCONLY) { 5521 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 5522 p[6], p[7]); 5523 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 5524 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 5525 p[4] & 0xff, p[5] >> 8); 5526 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 5527 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5528 p[1] & 0xf, p[2] >> 4); 5529 } else { 5530 sbuf_printf(sb, 5531 "\n %02x %x%07x %x%07x %08x %08x " 5532 "%08x%08x%08x%08x", 5533 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5534 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 5535 p[6], p[7]); 5536 } 5537 } 5538 5539 rc = sbuf_finish(sb); 5540 sbuf_delete(sb); 5541 done: 5542 free(buf, M_CXGBE); 5543 return (rc); 5544 } 5545 5546 static int 5547 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) 5548 { 5549 struct adapter *sc = arg1; 5550 u_int i; 5551 struct sbuf *sb; 5552 uint32_t *buf, *p; 5553 int rc; 5554 5555 rc = sysctl_wire_old_buffer(req, 0); 5556 if (rc != 0) 5557 return (rc); 5558 5559 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5560 if (sb == NULL) 5561 return (ENOMEM); 5562 5563 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, 5564 M_ZERO | M_WAITOK); 5565 5566 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); 5567 p = buf; 5568 5569 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5570 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], 5571 p[1], p[0]); 5572 } 5573 5574 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); 5575 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5576 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", 5577 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, 5578 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, 5579 (p[1] >> 2) | ((p[2] & 3) << 30), 5580 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, 5581 p[0] & 1); 5582 } 5583 5584 rc = sbuf_finish(sb); 5585 sbuf_delete(sb); 5586 free(buf, M_CXGBE); 5587 return (rc); 5588 } 5589 5590 static int 5591 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) 5592 { 5593 struct adapter *sc = arg1; 5594 u_int i; 5595 struct sbuf *sb; 5596 uint32_t *buf, *p; 5597 int rc; 5598 5599 rc = sysctl_wire_old_buffer(req, 0); 5600 if (rc != 0) 5601 return (rc); 5602 5603 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5604 if (sb == NULL) 5605 return (ENOMEM); 5606 5607 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, 5608 M_ZERO | M_WAITOK); 5609 5610 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); 5611 p = buf; 5612 5613 sbuf_printf(sb, "Cntl ID DataBE Addr Data"); 5614 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5615 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", 5616 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, 5617 p[4], p[3], p[2], p[1], p[0]); 5618 } 5619 5620 sbuf_printf(sb, "\n\nCntl ID Data"); 5621 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5622 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", 5623 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); 5624 } 5625 5626 rc = sbuf_finish(sb); 5627 sbuf_delete(sb); 5628 free(buf, M_CXGBE); 5629 return (rc); 5630 } 5631 5632 static int 5633 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 5634 { 5635 struct adapter *sc = arg1; 5636 struct sbuf *sb; 5637 int rc, i; 5638 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5639 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5640 uint16_t thres[CIM_NUM_IBQ]; 5641 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 5642 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 5643 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 5644 5645 if (is_t4(sc)) { 5646 cim_num_obq = CIM_NUM_OBQ; 5647 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 5648 obq_rdaddr = A_UP_OBQ_0_REALADDR; 5649 } else { 5650 cim_num_obq = CIM_NUM_OBQ_T5; 5651 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 5652 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 5653 } 5654 nq = CIM_NUM_IBQ + cim_num_obq; 5655 5656 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 5657 if (rc == 0) 5658 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 5659 if (rc != 0) 5660 return (rc); 5661 5662 t4_read_cimq_cfg(sc, base, size, thres); 5663 5664 rc = sysctl_wire_old_buffer(req, 0); 5665 if (rc != 0) 5666 return (rc); 5667 5668 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5669 if (sb == NULL) 5670 return (ENOMEM); 5671 5672 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 5673 5674 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 5675 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 5676 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 5677 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5678 G_QUEREMFLITS(p[2]) * 16); 5679 for ( ; i < nq; i++, p += 4, wr += 2) 5680 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 5681 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 5682 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5683 G_QUEREMFLITS(p[2]) * 16); 5684 5685 rc = sbuf_finish(sb); 5686 sbuf_delete(sb); 5687 5688 return (rc); 5689 } 5690 5691 static int 5692 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 5693 { 5694 struct adapter *sc = arg1; 5695 struct sbuf *sb; 5696 int rc; 5697 struct tp_cpl_stats stats; 5698 5699 rc = sysctl_wire_old_buffer(req, 0); 5700 if (rc != 0) 5701 return (rc); 5702 5703 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5704 if (sb == NULL) 5705 return (ENOMEM); 5706 5707 t4_tp_get_cpl_stats(sc, &stats); 5708 5709 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 5710 "channel 3\n"); 5711 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 5712 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 5713 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 5714 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 5715 5716 rc = sbuf_finish(sb); 5717 sbuf_delete(sb); 5718 5719 return (rc); 5720 } 5721 5722 static int 5723 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 5724 { 5725 struct adapter *sc = arg1; 5726 struct sbuf *sb; 5727 int rc; 5728 struct tp_usm_stats stats; 5729 5730 rc = sysctl_wire_old_buffer(req, 0); 5731 if (rc != 0) 5732 return(rc); 5733 5734 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5735 if (sb == NULL) 5736 return (ENOMEM); 5737 5738 t4_get_usm_stats(sc, &stats); 5739 5740 sbuf_printf(sb, "Frames: %u\n", stats.frames); 5741 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 5742 sbuf_printf(sb, "Drops: %u", stats.drops); 5743 5744 rc = sbuf_finish(sb); 5745 sbuf_delete(sb); 5746 5747 return (rc); 5748 } 5749 5750 const char *devlog_level_strings[] = { 5751 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 5752 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 5753 [FW_DEVLOG_LEVEL_ERR] = "ERR", 5754 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 5755 [FW_DEVLOG_LEVEL_INFO] = "INFO", 5756 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 5757 }; 5758 5759 const char *devlog_facility_strings[] = { 5760 [FW_DEVLOG_FACILITY_CORE] = "CORE", 5761 [FW_DEVLOG_FACILITY_CF] = "CF", 5762 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 5763 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 5764 [FW_DEVLOG_FACILITY_RES] = "RES", 5765 [FW_DEVLOG_FACILITY_HW] = "HW", 5766 [FW_DEVLOG_FACILITY_FLR] = "FLR", 5767 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 5768 [FW_DEVLOG_FACILITY_PHY] = "PHY", 5769 [FW_DEVLOG_FACILITY_MAC] = "MAC", 5770 [FW_DEVLOG_FACILITY_PORT] = "PORT", 5771 [FW_DEVLOG_FACILITY_VI] = "VI", 5772 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 5773 [FW_DEVLOG_FACILITY_ACL] = "ACL", 5774 [FW_DEVLOG_FACILITY_TM] = "TM", 5775 [FW_DEVLOG_FACILITY_QFC] = "QFC", 5776 [FW_DEVLOG_FACILITY_DCB] = "DCB", 5777 [FW_DEVLOG_FACILITY_ETH] = "ETH", 5778 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 5779 [FW_DEVLOG_FACILITY_RI] = "RI", 5780 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 5781 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 5782 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 5783 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 5784 }; 5785 5786 static int 5787 sysctl_devlog(SYSCTL_HANDLER_ARGS) 5788 { 5789 struct adapter *sc = arg1; 5790 struct devlog_params *dparams = &sc->params.devlog; 5791 struct fw_devlog_e *buf, *e; 5792 int i, j, rc, nentries, first = 0, m; 5793 struct sbuf *sb; 5794 uint64_t ftstamp = UINT64_MAX; 5795 5796 if (dparams->start == 0) { 5797 dparams->memtype = FW_MEMTYPE_EDC0; 5798 dparams->start = 0x84000; 5799 dparams->size = 32768; 5800 } 5801 5802 nentries = dparams->size / sizeof(struct fw_devlog_e); 5803 5804 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 5805 if (buf == NULL) 5806 return (ENOMEM); 5807 5808 m = fwmtype_to_hwmtype(dparams->memtype); 5809 rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf); 5810 if (rc != 0) 5811 goto done; 5812 5813 for (i = 0; i < nentries; i++) { 5814 e = &buf[i]; 5815 5816 if (e->timestamp == 0) 5817 break; /* end */ 5818 5819 e->timestamp = be64toh(e->timestamp); 5820 e->seqno = be32toh(e->seqno); 5821 for (j = 0; j < 8; j++) 5822 e->params[j] = be32toh(e->params[j]); 5823 5824 if (e->timestamp < ftstamp) { 5825 ftstamp = e->timestamp; 5826 first = i; 5827 } 5828 } 5829 5830 if (buf[first].timestamp == 0) 5831 goto done; /* nothing in the log */ 5832 5833 rc = sysctl_wire_old_buffer(req, 0); 5834 if (rc != 0) 5835 goto done; 5836 5837 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5838 if (sb == NULL) { 5839 rc = ENOMEM; 5840 goto done; 5841 } 5842 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 5843 "Seq#", "Tstamp", "Level", "Facility", "Message"); 5844 5845 i = first; 5846 do { 5847 e = &buf[i]; 5848 if (e->timestamp == 0) 5849 break; /* end */ 5850 5851 sbuf_printf(sb, "%10d %15ju %8s %8s ", 5852 e->seqno, e->timestamp, 5853 (e->level < nitems(devlog_level_strings) ? 5854 devlog_level_strings[e->level] : "UNKNOWN"), 5855 (e->facility < nitems(devlog_facility_strings) ? 5856 devlog_facility_strings[e->facility] : "UNKNOWN")); 5857 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 5858 e->params[2], e->params[3], e->params[4], 5859 e->params[5], e->params[6], e->params[7]); 5860 5861 if (++i == nentries) 5862 i = 0; 5863 } while (i != first); 5864 5865 rc = sbuf_finish(sb); 5866 sbuf_delete(sb); 5867 done: 5868 free(buf, M_CXGBE); 5869 return (rc); 5870 } 5871 5872 static int 5873 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 5874 { 5875 struct adapter *sc = arg1; 5876 struct sbuf *sb; 5877 int rc; 5878 struct tp_fcoe_stats stats[4]; 5879 5880 rc = sysctl_wire_old_buffer(req, 0); 5881 if (rc != 0) 5882 return (rc); 5883 5884 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5885 if (sb == NULL) 5886 return (ENOMEM); 5887 5888 t4_get_fcoe_stats(sc, 0, &stats[0]); 5889 t4_get_fcoe_stats(sc, 1, &stats[1]); 5890 t4_get_fcoe_stats(sc, 2, &stats[2]); 5891 t4_get_fcoe_stats(sc, 3, &stats[3]); 5892 5893 sbuf_printf(sb, " channel 0 channel 1 " 5894 "channel 2 channel 3\n"); 5895 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 5896 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 5897 stats[3].octetsDDP); 5898 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 5899 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 5900 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 5901 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 5902 stats[3].framesDrop); 5903 5904 rc = sbuf_finish(sb); 5905 sbuf_delete(sb); 5906 5907 return (rc); 5908 } 5909 5910 static int 5911 sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 5912 { 5913 struct adapter *sc = arg1; 5914 struct sbuf *sb; 5915 int rc, i; 5916 unsigned int map, kbps, ipg, mode; 5917 unsigned int pace_tab[NTX_SCHED]; 5918 5919 rc = sysctl_wire_old_buffer(req, 0); 5920 if (rc != 0) 5921 return (rc); 5922 5923 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5924 if (sb == NULL) 5925 return (ENOMEM); 5926 5927 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 5928 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 5929 t4_read_pace_tbl(sc, pace_tab); 5930 5931 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 5932 "Class IPG (0.1 ns) Flow IPG (us)"); 5933 5934 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 5935 t4_get_tx_sched(sc, i, &kbps, &ipg); 5936 sbuf_printf(sb, "\n %u %-5s %u ", i, 5937 (mode & (1 << i)) ? "flow" : "class", map & 3); 5938 if (kbps) 5939 sbuf_printf(sb, "%9u ", kbps); 5940 else 5941 sbuf_printf(sb, " disabled "); 5942 5943 if (ipg) 5944 sbuf_printf(sb, "%13u ", ipg); 5945 else 5946 sbuf_printf(sb, " disabled "); 5947 5948 if (pace_tab[i]) 5949 sbuf_printf(sb, "%10u", pace_tab[i]); 5950 else 5951 sbuf_printf(sb, " disabled"); 5952 } 5953 5954 rc = sbuf_finish(sb); 5955 sbuf_delete(sb); 5956 5957 return (rc); 5958 } 5959 5960 static int 5961 sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 5962 { 5963 struct adapter *sc = arg1; 5964 struct sbuf *sb; 5965 int rc, i, j; 5966 uint64_t *p0, *p1; 5967 struct lb_port_stats s[2]; 5968 static const char *stat_name[] = { 5969 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 5970 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 5971 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 5972 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 5973 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 5974 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 5975 "BG2FramesTrunc:", "BG3FramesTrunc:" 5976 }; 5977 5978 rc = sysctl_wire_old_buffer(req, 0); 5979 if (rc != 0) 5980 return (rc); 5981 5982 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5983 if (sb == NULL) 5984 return (ENOMEM); 5985 5986 memset(s, 0, sizeof(s)); 5987 5988 for (i = 0; i < 4; i += 2) { 5989 t4_get_lb_stats(sc, i, &s[0]); 5990 t4_get_lb_stats(sc, i + 1, &s[1]); 5991 5992 p0 = &s[0].octets; 5993 p1 = &s[1].octets; 5994 sbuf_printf(sb, "%s Loopback %u" 5995 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 5996 5997 for (j = 0; j < nitems(stat_name); j++) 5998 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 5999 *p0++, *p1++); 6000 } 6001 6002 rc = sbuf_finish(sb); 6003 sbuf_delete(sb); 6004 6005 return (rc); 6006 } 6007 6008 static int 6009 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) 6010 { 6011 int rc = 0; 6012 struct port_info *pi = arg1; 6013 struct sbuf *sb; 6014 static const char *linkdnreasons[] = { 6015 "non-specific", "remote fault", "autoneg failed", "reserved3", 6016 "PHY overheated", "unknown", "rx los", "reserved7" 6017 }; 6018 6019 rc = sysctl_wire_old_buffer(req, 0); 6020 if (rc != 0) 6021 return(rc); 6022 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); 6023 if (sb == NULL) 6024 return (ENOMEM); 6025 6026 if (pi->linkdnrc < 0) 6027 sbuf_printf(sb, "n/a"); 6028 else if (pi->linkdnrc < nitems(linkdnreasons)) 6029 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]); 6030 else 6031 sbuf_printf(sb, "%d", pi->linkdnrc); 6032 6033 rc = sbuf_finish(sb); 6034 sbuf_delete(sb); 6035 6036 return (rc); 6037 } 6038 6039 struct mem_desc { 6040 unsigned int base; 6041 unsigned int limit; 6042 unsigned int idx; 6043 }; 6044 6045 static int 6046 mem_desc_cmp(const void *a, const void *b) 6047 { 6048 return ((const struct mem_desc *)a)->base - 6049 ((const struct mem_desc *)b)->base; 6050 } 6051 6052 static void 6053 mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 6054 unsigned int to) 6055 { 6056 unsigned int size; 6057 6058 size = to - from + 1; 6059 if (size == 0) 6060 return; 6061 6062 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 6063 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 6064 } 6065 6066 static int 6067 sysctl_meminfo(SYSCTL_HANDLER_ARGS) 6068 { 6069 struct adapter *sc = arg1; 6070 struct sbuf *sb; 6071 int rc, i, n; 6072 uint32_t lo, hi, used, alloc; 6073 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 6074 static const char *region[] = { 6075 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 6076 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 6077 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 6078 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 6079 "RQUDP region:", "PBL region:", "TXPBL region:", 6080 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 6081 "On-chip queues:" 6082 }; 6083 struct mem_desc avail[4]; 6084 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 6085 struct mem_desc *md = mem; 6086 6087 rc = sysctl_wire_old_buffer(req, 0); 6088 if (rc != 0) 6089 return (rc); 6090 6091 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6092 if (sb == NULL) 6093 return (ENOMEM); 6094 6095 for (i = 0; i < nitems(mem); i++) { 6096 mem[i].limit = 0; 6097 mem[i].idx = i; 6098 } 6099 6100 /* Find and sort the populated memory ranges */ 6101 i = 0; 6102 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 6103 if (lo & F_EDRAM0_ENABLE) { 6104 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 6105 avail[i].base = G_EDRAM0_BASE(hi) << 20; 6106 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 6107 avail[i].idx = 0; 6108 i++; 6109 } 6110 if (lo & F_EDRAM1_ENABLE) { 6111 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 6112 avail[i].base = G_EDRAM1_BASE(hi) << 20; 6113 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 6114 avail[i].idx = 1; 6115 i++; 6116 } 6117 if (lo & F_EXT_MEM_ENABLE) { 6118 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 6119 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 6120 avail[i].limit = avail[i].base + 6121 (G_EXT_MEM_SIZE(hi) << 20); 6122 avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */ 6123 i++; 6124 } 6125 if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) { 6126 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 6127 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 6128 avail[i].limit = avail[i].base + 6129 (G_EXT_MEM1_SIZE(hi) << 20); 6130 avail[i].idx = 4; 6131 i++; 6132 } 6133 if (!i) /* no memory available */ 6134 return 0; 6135 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 6136 6137 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 6138 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 6139 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 6140 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 6141 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 6142 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 6143 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 6144 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 6145 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 6146 6147 /* the next few have explicit upper bounds */ 6148 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 6149 md->limit = md->base - 1 + 6150 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 6151 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 6152 md++; 6153 6154 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 6155 md->limit = md->base - 1 + 6156 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 6157 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 6158 md++; 6159 6160 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6161 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 6162 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 6163 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 6164 } else { 6165 md->base = 0; 6166 md->idx = nitems(region); /* hide it */ 6167 } 6168 md++; 6169 6170 #define ulp_region(reg) \ 6171 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 6172 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 6173 6174 ulp_region(RX_ISCSI); 6175 ulp_region(RX_TDDP); 6176 ulp_region(TX_TPT); 6177 ulp_region(RX_STAG); 6178 ulp_region(RX_RQ); 6179 ulp_region(RX_RQUDP); 6180 ulp_region(RX_PBL); 6181 ulp_region(TX_PBL); 6182 #undef ulp_region 6183 6184 md->base = 0; 6185 md->idx = nitems(region); 6186 if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) { 6187 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); 6188 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc, 6189 A_SGE_DBVFIFO_SIZE))) << 2) - 1; 6190 } 6191 md++; 6192 6193 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 6194 md->limit = md->base + sc->tids.ntids - 1; 6195 md++; 6196 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 6197 md->limit = md->base + sc->tids.ntids - 1; 6198 md++; 6199 6200 md->base = sc->vres.ocq.start; 6201 if (sc->vres.ocq.size) 6202 md->limit = md->base + sc->vres.ocq.size - 1; 6203 else 6204 md->idx = nitems(region); /* hide it */ 6205 md++; 6206 6207 /* add any address-space holes, there can be up to 3 */ 6208 for (n = 0; n < i - 1; n++) 6209 if (avail[n].limit < avail[n + 1].base) 6210 (md++)->base = avail[n].limit; 6211 if (avail[n].limit) 6212 (md++)->base = avail[n].limit; 6213 6214 n = md - mem; 6215 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 6216 6217 for (lo = 0; lo < i; lo++) 6218 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 6219 avail[lo].limit - 1); 6220 6221 sbuf_printf(sb, "\n"); 6222 for (i = 0; i < n; i++) { 6223 if (mem[i].idx >= nitems(region)) 6224 continue; /* skip holes */ 6225 if (!mem[i].limit) 6226 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 6227 mem_region_show(sb, region[mem[i].idx], mem[i].base, 6228 mem[i].limit); 6229 } 6230 6231 sbuf_printf(sb, "\n"); 6232 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 6233 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 6234 mem_region_show(sb, "uP RAM:", lo, hi); 6235 6236 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 6237 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 6238 mem_region_show(sb, "uP Extmem2:", lo, hi); 6239 6240 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 6241 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 6242 G_PMRXMAXPAGE(lo), 6243 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 6244 (lo & F_PMRXNUMCHN) ? 2 : 1); 6245 6246 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 6247 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 6248 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 6249 G_PMTXMAXPAGE(lo), 6250 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 6251 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 6252 sbuf_printf(sb, "%u p-structs\n", 6253 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 6254 6255 for (i = 0; i < 4; i++) { 6256 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 6257 if (is_t4(sc)) { 6258 used = G_USED(lo); 6259 alloc = G_ALLOC(lo); 6260 } else { 6261 used = G_T5_USED(lo); 6262 alloc = G_T5_ALLOC(lo); 6263 } 6264 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 6265 i, used, alloc); 6266 } 6267 for (i = 0; i < 4; i++) { 6268 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 6269 if (is_t4(sc)) { 6270 used = G_USED(lo); 6271 alloc = G_ALLOC(lo); 6272 } else { 6273 used = G_T5_USED(lo); 6274 alloc = G_T5_ALLOC(lo); 6275 } 6276 sbuf_printf(sb, 6277 "\nLoopback %d using %u pages out of %u allocated", 6278 i, used, alloc); 6279 } 6280 6281 rc = sbuf_finish(sb); 6282 sbuf_delete(sb); 6283 6284 return (rc); 6285 } 6286 6287 static inline void 6288 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) 6289 { 6290 *mask = x | y; 6291 y = htobe64(y); 6292 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); 6293 } 6294 6295 static int 6296 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) 6297 { 6298 struct adapter *sc = arg1; 6299 struct sbuf *sb; 6300 int rc, i, n; 6301 6302 rc = sysctl_wire_old_buffer(req, 0); 6303 if (rc != 0) 6304 return (rc); 6305 6306 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6307 if (sb == NULL) 6308 return (ENOMEM); 6309 6310 sbuf_printf(sb, 6311 "Idx Ethernet address Mask Vld Ports PF" 6312 " VF Replication P0 P1 P2 P3 ML"); 6313 n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES : 6314 NUM_MPS_T5_CLS_SRAM_L_INSTANCES; 6315 for (i = 0; i < n; i++) { 6316 uint64_t tcamx, tcamy, mask; 6317 uint32_t cls_lo, cls_hi; 6318 uint8_t addr[ETHER_ADDR_LEN]; 6319 6320 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); 6321 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); 6322 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 6323 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 6324 6325 if (tcamx & tcamy) 6326 continue; 6327 6328 tcamxy2valmask(tcamx, tcamy, addr, &mask); 6329 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" 6330 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], 6331 addr[3], addr[4], addr[5], (uintmax_t)mask, 6332 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', 6333 G_PORTMAP(cls_hi), G_PF(cls_lo), 6334 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); 6335 6336 if (cls_lo & F_REPLICATE) { 6337 struct fw_ldst_cmd ldst_cmd; 6338 6339 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 6340 ldst_cmd.op_to_addrspace = 6341 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 6342 F_FW_CMD_REQUEST | F_FW_CMD_READ | 6343 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 6344 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 6345 ldst_cmd.u.mps.fid_ctl = 6346 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 6347 V_FW_LDST_CMD_CTL(i)); 6348 6349 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 6350 "t4mps"); 6351 if (rc) 6352 break; 6353 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 6354 sizeof(ldst_cmd), &ldst_cmd); 6355 end_synchronized_op(sc, 0); 6356 6357 if (rc != 0) { 6358 sbuf_printf(sb, 6359 " ------------ error %3u ------------", rc); 6360 rc = 0; 6361 } else { 6362 sbuf_printf(sb, " %08x %08x %08x %08x", 6363 be32toh(ldst_cmd.u.mps.rplc127_96), 6364 be32toh(ldst_cmd.u.mps.rplc95_64), 6365 be32toh(ldst_cmd.u.mps.rplc63_32), 6366 be32toh(ldst_cmd.u.mps.rplc31_0)); 6367 } 6368 } else 6369 sbuf_printf(sb, "%36s", ""); 6370 6371 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), 6372 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), 6373 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); 6374 } 6375 6376 if (rc) 6377 (void) sbuf_finish(sb); 6378 else 6379 rc = sbuf_finish(sb); 6380 sbuf_delete(sb); 6381 6382 return (rc); 6383 } 6384 6385 static int 6386 sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 6387 { 6388 struct adapter *sc = arg1; 6389 struct sbuf *sb; 6390 int rc; 6391 uint16_t mtus[NMTUS]; 6392 6393 rc = sysctl_wire_old_buffer(req, 0); 6394 if (rc != 0) 6395 return (rc); 6396 6397 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6398 if (sb == NULL) 6399 return (ENOMEM); 6400 6401 t4_read_mtu_tbl(sc, mtus, NULL); 6402 6403 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 6404 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 6405 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 6406 mtus[14], mtus[15]); 6407 6408 rc = sbuf_finish(sb); 6409 sbuf_delete(sb); 6410 6411 return (rc); 6412 } 6413 6414 static int 6415 sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 6416 { 6417 struct adapter *sc = arg1; 6418 struct sbuf *sb; 6419 int rc, i; 6420 uint32_t cnt[PM_NSTATS]; 6421 uint64_t cyc[PM_NSTATS]; 6422 static const char *rx_stats[] = { 6423 "Read:", "Write bypass:", "Write mem:", "Flush:" 6424 }; 6425 static const char *tx_stats[] = { 6426 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:" 6427 }; 6428 6429 rc = sysctl_wire_old_buffer(req, 0); 6430 if (rc != 0) 6431 return (rc); 6432 6433 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6434 if (sb == NULL) 6435 return (ENOMEM); 6436 6437 t4_pmtx_get_stats(sc, cnt, cyc); 6438 sbuf_printf(sb, " Tx pcmds Tx bytes"); 6439 for (i = 0; i < ARRAY_SIZE(tx_stats); i++) 6440 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i], 6441 cyc[i]); 6442 6443 t4_pmrx_get_stats(sc, cnt, cyc); 6444 sbuf_printf(sb, "\n Rx pcmds Rx bytes"); 6445 for (i = 0; i < ARRAY_SIZE(rx_stats); i++) 6446 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i], 6447 cyc[i]); 6448 6449 rc = sbuf_finish(sb); 6450 sbuf_delete(sb); 6451 6452 return (rc); 6453 } 6454 6455 static int 6456 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 6457 { 6458 struct adapter *sc = arg1; 6459 struct sbuf *sb; 6460 int rc; 6461 struct tp_rdma_stats stats; 6462 6463 rc = sysctl_wire_old_buffer(req, 0); 6464 if (rc != 0) 6465 return (rc); 6466 6467 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6468 if (sb == NULL) 6469 return (ENOMEM); 6470 6471 t4_tp_get_rdma_stats(sc, &stats); 6472 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 6473 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 6474 6475 rc = sbuf_finish(sb); 6476 sbuf_delete(sb); 6477 6478 return (rc); 6479 } 6480 6481 static int 6482 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 6483 { 6484 struct adapter *sc = arg1; 6485 struct sbuf *sb; 6486 int rc; 6487 struct tp_tcp_stats v4, v6; 6488 6489 rc = sysctl_wire_old_buffer(req, 0); 6490 if (rc != 0) 6491 return (rc); 6492 6493 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6494 if (sb == NULL) 6495 return (ENOMEM); 6496 6497 t4_tp_get_tcp_stats(sc, &v4, &v6); 6498 sbuf_printf(sb, 6499 " IP IPv6\n"); 6500 sbuf_printf(sb, "OutRsts: %20u %20u\n", 6501 v4.tcpOutRsts, v6.tcpOutRsts); 6502 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 6503 v4.tcpInSegs, v6.tcpInSegs); 6504 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 6505 v4.tcpOutSegs, v6.tcpOutSegs); 6506 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 6507 v4.tcpRetransSegs, v6.tcpRetransSegs); 6508 6509 rc = sbuf_finish(sb); 6510 sbuf_delete(sb); 6511 6512 return (rc); 6513 } 6514 6515 static int 6516 sysctl_tids(SYSCTL_HANDLER_ARGS) 6517 { 6518 struct adapter *sc = arg1; 6519 struct sbuf *sb; 6520 int rc; 6521 struct tid_info *t = &sc->tids; 6522 6523 rc = sysctl_wire_old_buffer(req, 0); 6524 if (rc != 0) 6525 return (rc); 6526 6527 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6528 if (sb == NULL) 6529 return (ENOMEM); 6530 6531 if (t->natids) { 6532 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 6533 t->atids_in_use); 6534 } 6535 6536 if (t->ntids) { 6537 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6538 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 6539 6540 if (b) { 6541 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 6542 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6543 t->ntids - 1); 6544 } else { 6545 sbuf_printf(sb, "TID range: %u-%u", 6546 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6547 t->ntids - 1); 6548 } 6549 } else 6550 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 6551 sbuf_printf(sb, ", in use: %u\n", 6552 atomic_load_acq_int(&t->tids_in_use)); 6553 } 6554 6555 if (t->nstids) { 6556 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 6557 t->stid_base + t->nstids - 1, t->stids_in_use); 6558 } 6559 6560 if (t->nftids) { 6561 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 6562 t->ftid_base + t->nftids - 1); 6563 } 6564 6565 if (t->netids) { 6566 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base, 6567 t->etid_base + t->netids - 1); 6568 } 6569 6570 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 6571 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 6572 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 6573 6574 rc = sbuf_finish(sb); 6575 sbuf_delete(sb); 6576 6577 return (rc); 6578 } 6579 6580 static int 6581 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 6582 { 6583 struct adapter *sc = arg1; 6584 struct sbuf *sb; 6585 int rc; 6586 struct tp_err_stats stats; 6587 6588 rc = sysctl_wire_old_buffer(req, 0); 6589 if (rc != 0) 6590 return (rc); 6591 6592 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6593 if (sb == NULL) 6594 return (ENOMEM); 6595 6596 t4_tp_get_err_stats(sc, &stats); 6597 6598 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6599 "channel 3\n"); 6600 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 6601 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 6602 stats.macInErrs[3]); 6603 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 6604 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 6605 stats.hdrInErrs[3]); 6606 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 6607 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 6608 stats.tcpInErrs[3]); 6609 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 6610 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 6611 stats.tcp6InErrs[3]); 6612 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 6613 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 6614 stats.tnlCongDrops[3]); 6615 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 6616 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 6617 stats.tnlTxDrops[3]); 6618 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 6619 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 6620 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 6621 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 6622 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 6623 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 6624 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 6625 stats.ofldNoNeigh, stats.ofldCongDefer); 6626 6627 rc = sbuf_finish(sb); 6628 sbuf_delete(sb); 6629 6630 return (rc); 6631 } 6632 6633 struct field_desc { 6634 const char *name; 6635 u_int start; 6636 u_int width; 6637 }; 6638 6639 static void 6640 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) 6641 { 6642 char buf[32]; 6643 int line_size = 0; 6644 6645 while (f->name) { 6646 uint64_t mask = (1ULL << f->width) - 1; 6647 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, 6648 ((uintmax_t)v >> f->start) & mask); 6649 6650 if (line_size + len >= 79) { 6651 line_size = 8; 6652 sbuf_printf(sb, "\n "); 6653 } 6654 sbuf_printf(sb, "%s ", buf); 6655 line_size += len + 1; 6656 f++; 6657 } 6658 sbuf_printf(sb, "\n"); 6659 } 6660 6661 static struct field_desc tp_la0[] = { 6662 { "RcfOpCodeOut", 60, 4 }, 6663 { "State", 56, 4 }, 6664 { "WcfState", 52, 4 }, 6665 { "RcfOpcSrcOut", 50, 2 }, 6666 { "CRxError", 49, 1 }, 6667 { "ERxError", 48, 1 }, 6668 { "SanityFailed", 47, 1 }, 6669 { "SpuriousMsg", 46, 1 }, 6670 { "FlushInputMsg", 45, 1 }, 6671 { "FlushInputCpl", 44, 1 }, 6672 { "RssUpBit", 43, 1 }, 6673 { "RssFilterHit", 42, 1 }, 6674 { "Tid", 32, 10 }, 6675 { "InitTcb", 31, 1 }, 6676 { "LineNumber", 24, 7 }, 6677 { "Emsg", 23, 1 }, 6678 { "EdataOut", 22, 1 }, 6679 { "Cmsg", 21, 1 }, 6680 { "CdataOut", 20, 1 }, 6681 { "EreadPdu", 19, 1 }, 6682 { "CreadPdu", 18, 1 }, 6683 { "TunnelPkt", 17, 1 }, 6684 { "RcfPeerFin", 16, 1 }, 6685 { "RcfReasonOut", 12, 4 }, 6686 { "TxCchannel", 10, 2 }, 6687 { "RcfTxChannel", 8, 2 }, 6688 { "RxEchannel", 6, 2 }, 6689 { "RcfRxChannel", 5, 1 }, 6690 { "RcfDataOutSrdy", 4, 1 }, 6691 { "RxDvld", 3, 1 }, 6692 { "RxOoDvld", 2, 1 }, 6693 { "RxCongestion", 1, 1 }, 6694 { "TxCongestion", 0, 1 }, 6695 { NULL } 6696 }; 6697 6698 static struct field_desc tp_la1[] = { 6699 { "CplCmdIn", 56, 8 }, 6700 { "CplCmdOut", 48, 8 }, 6701 { "ESynOut", 47, 1 }, 6702 { "EAckOut", 46, 1 }, 6703 { "EFinOut", 45, 1 }, 6704 { "ERstOut", 44, 1 }, 6705 { "SynIn", 43, 1 }, 6706 { "AckIn", 42, 1 }, 6707 { "FinIn", 41, 1 }, 6708 { "RstIn", 40, 1 }, 6709 { "DataIn", 39, 1 }, 6710 { "DataInVld", 38, 1 }, 6711 { "PadIn", 37, 1 }, 6712 { "RxBufEmpty", 36, 1 }, 6713 { "RxDdp", 35, 1 }, 6714 { "RxFbCongestion", 34, 1 }, 6715 { "TxFbCongestion", 33, 1 }, 6716 { "TxPktSumSrdy", 32, 1 }, 6717 { "RcfUlpType", 28, 4 }, 6718 { "Eread", 27, 1 }, 6719 { "Ebypass", 26, 1 }, 6720 { "Esave", 25, 1 }, 6721 { "Static0", 24, 1 }, 6722 { "Cread", 23, 1 }, 6723 { "Cbypass", 22, 1 }, 6724 { "Csave", 21, 1 }, 6725 { "CPktOut", 20, 1 }, 6726 { "RxPagePoolFull", 18, 2 }, 6727 { "RxLpbkPkt", 17, 1 }, 6728 { "TxLpbkPkt", 16, 1 }, 6729 { "RxVfValid", 15, 1 }, 6730 { "SynLearned", 14, 1 }, 6731 { "SetDelEntry", 13, 1 }, 6732 { "SetInvEntry", 12, 1 }, 6733 { "CpcmdDvld", 11, 1 }, 6734 { "CpcmdSave", 10, 1 }, 6735 { "RxPstructsFull", 8, 2 }, 6736 { "EpcmdDvld", 7, 1 }, 6737 { "EpcmdFlush", 6, 1 }, 6738 { "EpcmdTrimPrefix", 5, 1 }, 6739 { "EpcmdTrimPostfix", 4, 1 }, 6740 { "ERssIp4Pkt", 3, 1 }, 6741 { "ERssIp6Pkt", 2, 1 }, 6742 { "ERssTcpUdpPkt", 1, 1 }, 6743 { "ERssFceFipPkt", 0, 1 }, 6744 { NULL } 6745 }; 6746 6747 static struct field_desc tp_la2[] = { 6748 { "CplCmdIn", 56, 8 }, 6749 { "MpsVfVld", 55, 1 }, 6750 { "MpsPf", 52, 3 }, 6751 { "MpsVf", 44, 8 }, 6752 { "SynIn", 43, 1 }, 6753 { "AckIn", 42, 1 }, 6754 { "FinIn", 41, 1 }, 6755 { "RstIn", 40, 1 }, 6756 { "DataIn", 39, 1 }, 6757 { "DataInVld", 38, 1 }, 6758 { "PadIn", 37, 1 }, 6759 { "RxBufEmpty", 36, 1 }, 6760 { "RxDdp", 35, 1 }, 6761 { "RxFbCongestion", 34, 1 }, 6762 { "TxFbCongestion", 33, 1 }, 6763 { "TxPktSumSrdy", 32, 1 }, 6764 { "RcfUlpType", 28, 4 }, 6765 { "Eread", 27, 1 }, 6766 { "Ebypass", 26, 1 }, 6767 { "Esave", 25, 1 }, 6768 { "Static0", 24, 1 }, 6769 { "Cread", 23, 1 }, 6770 { "Cbypass", 22, 1 }, 6771 { "Csave", 21, 1 }, 6772 { "CPktOut", 20, 1 }, 6773 { "RxPagePoolFull", 18, 2 }, 6774 { "RxLpbkPkt", 17, 1 }, 6775 { "TxLpbkPkt", 16, 1 }, 6776 { "RxVfValid", 15, 1 }, 6777 { "SynLearned", 14, 1 }, 6778 { "SetDelEntry", 13, 1 }, 6779 { "SetInvEntry", 12, 1 }, 6780 { "CpcmdDvld", 11, 1 }, 6781 { "CpcmdSave", 10, 1 }, 6782 { "RxPstructsFull", 8, 2 }, 6783 { "EpcmdDvld", 7, 1 }, 6784 { "EpcmdFlush", 6, 1 }, 6785 { "EpcmdTrimPrefix", 5, 1 }, 6786 { "EpcmdTrimPostfix", 4, 1 }, 6787 { "ERssIp4Pkt", 3, 1 }, 6788 { "ERssIp6Pkt", 2, 1 }, 6789 { "ERssTcpUdpPkt", 1, 1 }, 6790 { "ERssFceFipPkt", 0, 1 }, 6791 { NULL } 6792 }; 6793 6794 static void 6795 tp_la_show(struct sbuf *sb, uint64_t *p, int idx) 6796 { 6797 6798 field_desc_show(sb, *p, tp_la0); 6799 } 6800 6801 static void 6802 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) 6803 { 6804 6805 if (idx) 6806 sbuf_printf(sb, "\n"); 6807 field_desc_show(sb, p[0], tp_la0); 6808 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6809 field_desc_show(sb, p[1], tp_la0); 6810 } 6811 6812 static void 6813 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) 6814 { 6815 6816 if (idx) 6817 sbuf_printf(sb, "\n"); 6818 field_desc_show(sb, p[0], tp_la0); 6819 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6820 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); 6821 } 6822 6823 static int 6824 sysctl_tp_la(SYSCTL_HANDLER_ARGS) 6825 { 6826 struct adapter *sc = arg1; 6827 struct sbuf *sb; 6828 uint64_t *buf, *p; 6829 int rc; 6830 u_int i, inc; 6831 void (*show_func)(struct sbuf *, uint64_t *, int); 6832 6833 rc = sysctl_wire_old_buffer(req, 0); 6834 if (rc != 0) 6835 return (rc); 6836 6837 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6838 if (sb == NULL) 6839 return (ENOMEM); 6840 6841 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); 6842 6843 t4_tp_read_la(sc, buf, NULL); 6844 p = buf; 6845 6846 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { 6847 case 2: 6848 inc = 2; 6849 show_func = tp_la_show2; 6850 break; 6851 case 3: 6852 inc = 2; 6853 show_func = tp_la_show3; 6854 break; 6855 default: 6856 inc = 1; 6857 show_func = tp_la_show; 6858 } 6859 6860 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) 6861 (*show_func)(sb, p, i); 6862 6863 rc = sbuf_finish(sb); 6864 sbuf_delete(sb); 6865 free(buf, M_CXGBE); 6866 return (rc); 6867 } 6868 6869 static int 6870 sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 6871 { 6872 struct adapter *sc = arg1; 6873 struct sbuf *sb; 6874 int rc; 6875 u64 nrate[NCHAN], orate[NCHAN]; 6876 6877 rc = sysctl_wire_old_buffer(req, 0); 6878 if (rc != 0) 6879 return (rc); 6880 6881 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6882 if (sb == NULL) 6883 return (ENOMEM); 6884 6885 t4_get_chan_txrate(sc, nrate, orate); 6886 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6887 "channel 3\n"); 6888 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 6889 nrate[0], nrate[1], nrate[2], nrate[3]); 6890 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 6891 orate[0], orate[1], orate[2], orate[3]); 6892 6893 rc = sbuf_finish(sb); 6894 sbuf_delete(sb); 6895 6896 return (rc); 6897 } 6898 6899 static int 6900 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) 6901 { 6902 struct adapter *sc = arg1; 6903 struct sbuf *sb; 6904 uint32_t *buf, *p; 6905 int rc, i; 6906 6907 rc = sysctl_wire_old_buffer(req, 0); 6908 if (rc != 0) 6909 return (rc); 6910 6911 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6912 if (sb == NULL) 6913 return (ENOMEM); 6914 6915 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, 6916 M_ZERO | M_WAITOK); 6917 6918 t4_ulprx_read_la(sc, buf); 6919 p = buf; 6920 6921 sbuf_printf(sb, " Pcmd Type Message" 6922 " Data"); 6923 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { 6924 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", 6925 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); 6926 } 6927 6928 rc = sbuf_finish(sb); 6929 sbuf_delete(sb); 6930 free(buf, M_CXGBE); 6931 return (rc); 6932 } 6933 6934 static int 6935 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 6936 { 6937 struct adapter *sc = arg1; 6938 struct sbuf *sb; 6939 int rc, v; 6940 6941 rc = sysctl_wire_old_buffer(req, 0); 6942 if (rc != 0) 6943 return (rc); 6944 6945 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6946 if (sb == NULL) 6947 return (ENOMEM); 6948 6949 v = t4_read_reg(sc, A_SGE_STAT_CFG); 6950 if (G_STATSOURCE_T5(v) == 7) { 6951 if (G_STATMODE(v) == 0) { 6952 sbuf_printf(sb, "total %d, incomplete %d", 6953 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6954 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6955 } else if (G_STATMODE(v) == 1) { 6956 sbuf_printf(sb, "total %d, data overflow %d", 6957 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6958 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6959 } 6960 } 6961 rc = sbuf_finish(sb); 6962 sbuf_delete(sb); 6963 6964 return (rc); 6965 } 6966 #endif 6967 6968 static uint32_t 6969 fconf_to_mode(uint32_t fconf) 6970 { 6971 uint32_t mode; 6972 6973 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 6974 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 6975 6976 if (fconf & F_FRAGMENTATION) 6977 mode |= T4_FILTER_IP_FRAGMENT; 6978 6979 if (fconf & F_MPSHITTYPE) 6980 mode |= T4_FILTER_MPS_HIT_TYPE; 6981 6982 if (fconf & F_MACMATCH) 6983 mode |= T4_FILTER_MAC_IDX; 6984 6985 if (fconf & F_ETHERTYPE) 6986 mode |= T4_FILTER_ETH_TYPE; 6987 6988 if (fconf & F_PROTOCOL) 6989 mode |= T4_FILTER_IP_PROTO; 6990 6991 if (fconf & F_TOS) 6992 mode |= T4_FILTER_IP_TOS; 6993 6994 if (fconf & F_VLAN) 6995 mode |= T4_FILTER_VLAN; 6996 6997 if (fconf & F_VNIC_ID) 6998 mode |= T4_FILTER_VNIC; 6999 7000 if (fconf & F_PORT) 7001 mode |= T4_FILTER_PORT; 7002 7003 if (fconf & F_FCOE) 7004 mode |= T4_FILTER_FCoE; 7005 7006 return (mode); 7007 } 7008 7009 static uint32_t 7010 mode_to_fconf(uint32_t mode) 7011 { 7012 uint32_t fconf = 0; 7013 7014 if (mode & T4_FILTER_IP_FRAGMENT) 7015 fconf |= F_FRAGMENTATION; 7016 7017 if (mode & T4_FILTER_MPS_HIT_TYPE) 7018 fconf |= F_MPSHITTYPE; 7019 7020 if (mode & T4_FILTER_MAC_IDX) 7021 fconf |= F_MACMATCH; 7022 7023 if (mode & T4_FILTER_ETH_TYPE) 7024 fconf |= F_ETHERTYPE; 7025 7026 if (mode & T4_FILTER_IP_PROTO) 7027 fconf |= F_PROTOCOL; 7028 7029 if (mode & T4_FILTER_IP_TOS) 7030 fconf |= F_TOS; 7031 7032 if (mode & T4_FILTER_VLAN) 7033 fconf |= F_VLAN; 7034 7035 if (mode & T4_FILTER_VNIC) 7036 fconf |= F_VNIC_ID; 7037 7038 if (mode & T4_FILTER_PORT) 7039 fconf |= F_PORT; 7040 7041 if (mode & T4_FILTER_FCoE) 7042 fconf |= F_FCOE; 7043 7044 return (fconf); 7045 } 7046 7047 static uint32_t 7048 fspec_to_fconf(struct t4_filter_specification *fs) 7049 { 7050 uint32_t fconf = 0; 7051 7052 if (fs->val.frag || fs->mask.frag) 7053 fconf |= F_FRAGMENTATION; 7054 7055 if (fs->val.matchtype || fs->mask.matchtype) 7056 fconf |= F_MPSHITTYPE; 7057 7058 if (fs->val.macidx || fs->mask.macidx) 7059 fconf |= F_MACMATCH; 7060 7061 if (fs->val.ethtype || fs->mask.ethtype) 7062 fconf |= F_ETHERTYPE; 7063 7064 if (fs->val.proto || fs->mask.proto) 7065 fconf |= F_PROTOCOL; 7066 7067 if (fs->val.tos || fs->mask.tos) 7068 fconf |= F_TOS; 7069 7070 if (fs->val.vlan_vld || fs->mask.vlan_vld) 7071 fconf |= F_VLAN; 7072 7073 if (fs->val.vnic_vld || fs->mask.vnic_vld) 7074 fconf |= F_VNIC_ID; 7075 7076 if (fs->val.iport || fs->mask.iport) 7077 fconf |= F_PORT; 7078 7079 if (fs->val.fcoe || fs->mask.fcoe) 7080 fconf |= F_FCOE; 7081 7082 return (fconf); 7083 } 7084 7085 static int 7086 get_filter_mode(struct adapter *sc, uint32_t *mode) 7087 { 7088 int rc; 7089 uint32_t fconf; 7090 7091 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7092 "t4getfm"); 7093 if (rc) 7094 return (rc); 7095 7096 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 7097 A_TP_VLAN_PRI_MAP); 7098 7099 if (sc->params.tp.vlan_pri_map != fconf) { 7100 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 7101 device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map, 7102 fconf); 7103 } 7104 7105 *mode = fconf_to_mode(fconf); 7106 7107 end_synchronized_op(sc, LOCK_HELD); 7108 return (0); 7109 } 7110 7111 static int 7112 set_filter_mode(struct adapter *sc, uint32_t mode) 7113 { 7114 uint32_t fconf; 7115 int rc; 7116 7117 fconf = mode_to_fconf(mode); 7118 7119 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7120 "t4setfm"); 7121 if (rc) 7122 return (rc); 7123 7124 if (sc->tids.ftids_in_use > 0) { 7125 rc = EBUSY; 7126 goto done; 7127 } 7128 7129 #ifdef TCP_OFFLOAD 7130 if (uld_active(sc, ULD_TOM)) { 7131 rc = EBUSY; 7132 goto done; 7133 } 7134 #endif 7135 7136 rc = -t4_set_filter_mode(sc, fconf); 7137 done: 7138 end_synchronized_op(sc, LOCK_HELD); 7139 return (rc); 7140 } 7141 7142 static inline uint64_t 7143 get_filter_hits(struct adapter *sc, uint32_t fid) 7144 { 7145 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 7146 uint64_t hits; 7147 7148 memwin_info(sc, 0, &mw_base, NULL); 7149 off = position_memwin(sc, 0, 7150 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 7151 if (is_t4(sc)) { 7152 hits = t4_read_reg64(sc, mw_base + off + 16); 7153 hits = be64toh(hits); 7154 } else { 7155 hits = t4_read_reg(sc, mw_base + off + 24); 7156 hits = be32toh(hits); 7157 } 7158 7159 return (hits); 7160 } 7161 7162 static int 7163 get_filter(struct adapter *sc, struct t4_filter *t) 7164 { 7165 int i, rc, nfilters = sc->tids.nftids; 7166 struct filter_entry *f; 7167 7168 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7169 "t4getf"); 7170 if (rc) 7171 return (rc); 7172 7173 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 7174 t->idx >= nfilters) { 7175 t->idx = 0xffffffff; 7176 goto done; 7177 } 7178 7179 f = &sc->tids.ftid_tab[t->idx]; 7180 for (i = t->idx; i < nfilters; i++, f++) { 7181 if (f->valid) { 7182 t->idx = i; 7183 t->l2tidx = f->l2t ? f->l2t->idx : 0; 7184 t->smtidx = f->smtidx; 7185 if (f->fs.hitcnts) 7186 t->hits = get_filter_hits(sc, t->idx); 7187 else 7188 t->hits = UINT64_MAX; 7189 t->fs = f->fs; 7190 7191 goto done; 7192 } 7193 } 7194 7195 t->idx = 0xffffffff; 7196 done: 7197 end_synchronized_op(sc, LOCK_HELD); 7198 return (0); 7199 } 7200 7201 static int 7202 set_filter(struct adapter *sc, struct t4_filter *t) 7203 { 7204 unsigned int nfilters, nports; 7205 struct filter_entry *f; 7206 int i, rc; 7207 7208 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 7209 if (rc) 7210 return (rc); 7211 7212 nfilters = sc->tids.nftids; 7213 nports = sc->params.nports; 7214 7215 if (nfilters == 0) { 7216 rc = ENOTSUP; 7217 goto done; 7218 } 7219 7220 if (!(sc->flags & FULL_INIT_DONE)) { 7221 rc = EAGAIN; 7222 goto done; 7223 } 7224 7225 if (t->idx >= nfilters) { 7226 rc = EINVAL; 7227 goto done; 7228 } 7229 7230 /* Validate against the global filter mode */ 7231 if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) != 7232 sc->params.tp.vlan_pri_map) { 7233 rc = E2BIG; 7234 goto done; 7235 } 7236 7237 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 7238 rc = EINVAL; 7239 goto done; 7240 } 7241 7242 if (t->fs.val.iport >= nports) { 7243 rc = EINVAL; 7244 goto done; 7245 } 7246 7247 /* Can't specify an iq if not steering to it */ 7248 if (!t->fs.dirsteer && t->fs.iq) { 7249 rc = EINVAL; 7250 goto done; 7251 } 7252 7253 /* IPv6 filter idx must be 4 aligned */ 7254 if (t->fs.type == 1 && 7255 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 7256 rc = EINVAL; 7257 goto done; 7258 } 7259 7260 if (sc->tids.ftid_tab == NULL) { 7261 KASSERT(sc->tids.ftids_in_use == 0, 7262 ("%s: no memory allocated but filters_in_use > 0", 7263 __func__)); 7264 7265 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 7266 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 7267 if (sc->tids.ftid_tab == NULL) { 7268 rc = ENOMEM; 7269 goto done; 7270 } 7271 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 7272 } 7273 7274 for (i = 0; i < 4; i++) { 7275 f = &sc->tids.ftid_tab[t->idx + i]; 7276 7277 if (f->pending || f->valid) { 7278 rc = EBUSY; 7279 goto done; 7280 } 7281 if (f->locked) { 7282 rc = EPERM; 7283 goto done; 7284 } 7285 7286 if (t->fs.type == 0) 7287 break; 7288 } 7289 7290 f = &sc->tids.ftid_tab[t->idx]; 7291 f->fs = t->fs; 7292 7293 rc = set_filter_wr(sc, t->idx); 7294 done: 7295 end_synchronized_op(sc, 0); 7296 7297 if (rc == 0) { 7298 mtx_lock(&sc->tids.ftid_lock); 7299 for (;;) { 7300 if (f->pending == 0) { 7301 rc = f->valid ? 0 : EIO; 7302 break; 7303 } 7304 7305 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7306 PCATCH, "t4setfw", 0)) { 7307 rc = EINPROGRESS; 7308 break; 7309 } 7310 } 7311 mtx_unlock(&sc->tids.ftid_lock); 7312 } 7313 return (rc); 7314 } 7315 7316 static int 7317 del_filter(struct adapter *sc, struct t4_filter *t) 7318 { 7319 unsigned int nfilters; 7320 struct filter_entry *f; 7321 int rc; 7322 7323 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 7324 if (rc) 7325 return (rc); 7326 7327 nfilters = sc->tids.nftids; 7328 7329 if (nfilters == 0) { 7330 rc = ENOTSUP; 7331 goto done; 7332 } 7333 7334 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 7335 t->idx >= nfilters) { 7336 rc = EINVAL; 7337 goto done; 7338 } 7339 7340 if (!(sc->flags & FULL_INIT_DONE)) { 7341 rc = EAGAIN; 7342 goto done; 7343 } 7344 7345 f = &sc->tids.ftid_tab[t->idx]; 7346 7347 if (f->pending) { 7348 rc = EBUSY; 7349 goto done; 7350 } 7351 if (f->locked) { 7352 rc = EPERM; 7353 goto done; 7354 } 7355 7356 if (f->valid) { 7357 t->fs = f->fs; /* extra info for the caller */ 7358 rc = del_filter_wr(sc, t->idx); 7359 } 7360 7361 done: 7362 end_synchronized_op(sc, 0); 7363 7364 if (rc == 0) { 7365 mtx_lock(&sc->tids.ftid_lock); 7366 for (;;) { 7367 if (f->pending == 0) { 7368 rc = f->valid ? EIO : 0; 7369 break; 7370 } 7371 7372 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7373 PCATCH, "t4delfw", 0)) { 7374 rc = EINPROGRESS; 7375 break; 7376 } 7377 } 7378 mtx_unlock(&sc->tids.ftid_lock); 7379 } 7380 7381 return (rc); 7382 } 7383 7384 static void 7385 clear_filter(struct filter_entry *f) 7386 { 7387 if (f->l2t) 7388 t4_l2t_release(f->l2t); 7389 7390 bzero(f, sizeof (*f)); 7391 } 7392 7393 static int 7394 set_filter_wr(struct adapter *sc, int fidx) 7395 { 7396 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7397 struct fw_filter_wr *fwr; 7398 unsigned int ftid; 7399 struct wrq_cookie cookie; 7400 7401 ASSERT_SYNCHRONIZED_OP(sc); 7402 7403 if (f->fs.newdmac || f->fs.newvlan) { 7404 /* This filter needs an L2T entry; allocate one. */ 7405 f->l2t = t4_l2t_alloc_switching(sc->l2t); 7406 if (f->l2t == NULL) 7407 return (EAGAIN); 7408 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 7409 f->fs.dmac)) { 7410 t4_l2t_release(f->l2t); 7411 f->l2t = NULL; 7412 return (ENOMEM); 7413 } 7414 } 7415 7416 ftid = sc->tids.ftid_base + fidx; 7417 7418 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 7419 if (fwr == NULL) 7420 return (ENOMEM); 7421 bzero(fwr, sizeof(*fwr)); 7422 7423 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 7424 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 7425 fwr->tid_to_iq = 7426 htobe32(V_FW_FILTER_WR_TID(ftid) | 7427 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 7428 V_FW_FILTER_WR_NOREPLY(0) | 7429 V_FW_FILTER_WR_IQ(f->fs.iq)); 7430 fwr->del_filter_to_l2tix = 7431 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 7432 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 7433 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 7434 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 7435 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 7436 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 7437 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 7438 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 7439 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 7440 f->fs.newvlan == VLAN_REWRITE) | 7441 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 7442 f->fs.newvlan == VLAN_REWRITE) | 7443 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 7444 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 7445 V_FW_FILTER_WR_PRIO(f->fs.prio) | 7446 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 7447 fwr->ethtype = htobe16(f->fs.val.ethtype); 7448 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 7449 fwr->frag_to_ovlan_vldm = 7450 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 7451 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 7452 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 7453 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 7454 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 7455 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 7456 fwr->smac_sel = 0; 7457 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 7458 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 7459 fwr->maci_to_matchtypem = 7460 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 7461 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 7462 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 7463 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 7464 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 7465 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 7466 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 7467 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 7468 fwr->ptcl = f->fs.val.proto; 7469 fwr->ptclm = f->fs.mask.proto; 7470 fwr->ttyp = f->fs.val.tos; 7471 fwr->ttypm = f->fs.mask.tos; 7472 fwr->ivlan = htobe16(f->fs.val.vlan); 7473 fwr->ivlanm = htobe16(f->fs.mask.vlan); 7474 fwr->ovlan = htobe16(f->fs.val.vnic); 7475 fwr->ovlanm = htobe16(f->fs.mask.vnic); 7476 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 7477 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 7478 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 7479 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 7480 fwr->lp = htobe16(f->fs.val.dport); 7481 fwr->lpm = htobe16(f->fs.mask.dport); 7482 fwr->fp = htobe16(f->fs.val.sport); 7483 fwr->fpm = htobe16(f->fs.mask.sport); 7484 if (f->fs.newsmac) 7485 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 7486 7487 f->pending = 1; 7488 sc->tids.ftids_in_use++; 7489 7490 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 7491 return (0); 7492 } 7493 7494 static int 7495 del_filter_wr(struct adapter *sc, int fidx) 7496 { 7497 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7498 struct fw_filter_wr *fwr; 7499 unsigned int ftid; 7500 struct wrq_cookie cookie; 7501 7502 ftid = sc->tids.ftid_base + fidx; 7503 7504 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 7505 if (fwr == NULL) 7506 return (ENOMEM); 7507 bzero(fwr, sizeof (*fwr)); 7508 7509 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 7510 7511 f->pending = 1; 7512 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 7513 return (0); 7514 } 7515 7516 int 7517 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 7518 { 7519 struct adapter *sc = iq->adapter; 7520 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 7521 unsigned int idx = GET_TID(rpl); 7522 unsigned int rc; 7523 struct filter_entry *f; 7524 7525 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 7526 rss->opcode)); 7527 7528 if (is_ftid(sc, idx)) { 7529 7530 idx -= sc->tids.ftid_base; 7531 f = &sc->tids.ftid_tab[idx]; 7532 rc = G_COOKIE(rpl->cookie); 7533 7534 mtx_lock(&sc->tids.ftid_lock); 7535 if (rc == FW_FILTER_WR_FLT_ADDED) { 7536 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 7537 __func__, idx)); 7538 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 7539 f->pending = 0; /* asynchronous setup completed */ 7540 f->valid = 1; 7541 } else { 7542 if (rc != FW_FILTER_WR_FLT_DELETED) { 7543 /* Add or delete failed, display an error */ 7544 log(LOG_ERR, 7545 "filter %u setup failed with error %u\n", 7546 idx, rc); 7547 } 7548 7549 clear_filter(f); 7550 sc->tids.ftids_in_use--; 7551 } 7552 wakeup(&sc->tids.ftid_tab); 7553 mtx_unlock(&sc->tids.ftid_lock); 7554 } 7555 7556 return (0); 7557 } 7558 7559 static int 7560 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 7561 { 7562 int rc; 7563 7564 if (cntxt->cid > M_CTXTQID) 7565 return (EINVAL); 7566 7567 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 7568 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 7569 return (EINVAL); 7570 7571 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 7572 if (rc) 7573 return (rc); 7574 7575 if (sc->flags & FW_OK) { 7576 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 7577 &cntxt->data[0]); 7578 if (rc == 0) 7579 goto done; 7580 } 7581 7582 /* 7583 * Read via firmware failed or wasn't even attempted. Read directly via 7584 * the backdoor. 7585 */ 7586 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 7587 done: 7588 end_synchronized_op(sc, 0); 7589 return (rc); 7590 } 7591 7592 static int 7593 load_fw(struct adapter *sc, struct t4_data *fw) 7594 { 7595 int rc; 7596 uint8_t *fw_data; 7597 7598 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 7599 if (rc) 7600 return (rc); 7601 7602 if (sc->flags & FULL_INIT_DONE) { 7603 rc = EBUSY; 7604 goto done; 7605 } 7606 7607 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 7608 if (fw_data == NULL) { 7609 rc = ENOMEM; 7610 goto done; 7611 } 7612 7613 rc = copyin(fw->data, fw_data, fw->len); 7614 if (rc == 0) 7615 rc = -t4_load_fw(sc, fw_data, fw->len); 7616 7617 free(fw_data, M_CXGBE); 7618 done: 7619 end_synchronized_op(sc, 0); 7620 return (rc); 7621 } 7622 7623 static int 7624 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 7625 { 7626 uint32_t addr, off, remaining, i, n; 7627 uint32_t *buf, *b; 7628 uint32_t mw_base, mw_aperture; 7629 int rc; 7630 uint8_t *dst; 7631 7632 rc = validate_mem_range(sc, mr->addr, mr->len); 7633 if (rc != 0) 7634 return (rc); 7635 7636 memwin_info(sc, win, &mw_base, &mw_aperture); 7637 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 7638 addr = mr->addr; 7639 remaining = mr->len; 7640 dst = (void *)mr->data; 7641 7642 while (remaining) { 7643 off = position_memwin(sc, win, addr); 7644 7645 /* number of bytes that we'll copy in the inner loop */ 7646 n = min(remaining, mw_aperture - off); 7647 for (i = 0; i < n; i += 4) 7648 *b++ = t4_read_reg(sc, mw_base + off + i); 7649 7650 rc = copyout(buf, dst, n); 7651 if (rc != 0) 7652 break; 7653 7654 b = buf; 7655 dst += n; 7656 remaining -= n; 7657 addr += n; 7658 } 7659 7660 free(buf, M_CXGBE); 7661 return (rc); 7662 } 7663 7664 static int 7665 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 7666 { 7667 int rc; 7668 7669 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 7670 return (EINVAL); 7671 7672 if (i2cd->len > sizeof(i2cd->data)) 7673 return (EFBIG); 7674 7675 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 7676 if (rc) 7677 return (rc); 7678 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 7679 i2cd->offset, i2cd->len, &i2cd->data[0]); 7680 end_synchronized_op(sc, 0); 7681 7682 return (rc); 7683 } 7684 7685 static int 7686 in_range(int val, int lo, int hi) 7687 { 7688 7689 return (val < 0 || (val <= hi && val >= lo)); 7690 } 7691 7692 static int 7693 set_sched_class(struct adapter *sc, struct t4_sched_params *p) 7694 { 7695 int fw_subcmd, fw_type, rc; 7696 7697 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc"); 7698 if (rc) 7699 return (rc); 7700 7701 if (!(sc->flags & FULL_INIT_DONE)) { 7702 rc = EAGAIN; 7703 goto done; 7704 } 7705 7706 /* 7707 * Translate the cxgbetool parameters into T4 firmware parameters. (The 7708 * sub-command and type are in common locations.) 7709 */ 7710 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG) 7711 fw_subcmd = FW_SCHED_SC_CONFIG; 7712 else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS) 7713 fw_subcmd = FW_SCHED_SC_PARAMS; 7714 else { 7715 rc = EINVAL; 7716 goto done; 7717 } 7718 if (p->type == SCHED_CLASS_TYPE_PACKET) 7719 fw_type = FW_SCHED_TYPE_PKTSCHED; 7720 else { 7721 rc = EINVAL; 7722 goto done; 7723 } 7724 7725 if (fw_subcmd == FW_SCHED_SC_CONFIG) { 7726 /* Vet our parameters ..*/ 7727 if (p->u.config.minmax < 0) { 7728 rc = EINVAL; 7729 goto done; 7730 } 7731 7732 /* And pass the request to the firmware ...*/ 7733 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1); 7734 goto done; 7735 } 7736 7737 if (fw_subcmd == FW_SCHED_SC_PARAMS) { 7738 int fw_level; 7739 int fw_mode; 7740 int fw_rateunit; 7741 int fw_ratemode; 7742 7743 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL) 7744 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL; 7745 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) 7746 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR; 7747 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) 7748 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL; 7749 else { 7750 rc = EINVAL; 7751 goto done; 7752 } 7753 7754 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS) 7755 fw_mode = FW_SCHED_PARAMS_MODE_CLASS; 7756 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW) 7757 fw_mode = FW_SCHED_PARAMS_MODE_FLOW; 7758 else { 7759 rc = EINVAL; 7760 goto done; 7761 } 7762 7763 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS) 7764 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 7765 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS) 7766 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE; 7767 else { 7768 rc = EINVAL; 7769 goto done; 7770 } 7771 7772 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL) 7773 fw_ratemode = FW_SCHED_PARAMS_RATE_REL; 7774 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS) 7775 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS; 7776 else { 7777 rc = EINVAL; 7778 goto done; 7779 } 7780 7781 /* Vet our parameters ... */ 7782 if (!in_range(p->u.params.channel, 0, 3) || 7783 !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) || 7784 !in_range(p->u.params.minrate, 0, 10000000) || 7785 !in_range(p->u.params.maxrate, 0, 10000000) || 7786 !in_range(p->u.params.weight, 0, 100)) { 7787 rc = ERANGE; 7788 goto done; 7789 } 7790 7791 /* 7792 * Translate any unset parameters into the firmware's 7793 * nomenclature and/or fail the call if the parameters 7794 * are required ... 7795 */ 7796 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 || 7797 p->u.params.channel < 0 || p->u.params.cl < 0) { 7798 rc = EINVAL; 7799 goto done; 7800 } 7801 if (p->u.params.minrate < 0) 7802 p->u.params.minrate = 0; 7803 if (p->u.params.maxrate < 0) { 7804 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7805 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7806 rc = EINVAL; 7807 goto done; 7808 } else 7809 p->u.params.maxrate = 0; 7810 } 7811 if (p->u.params.weight < 0) { 7812 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) { 7813 rc = EINVAL; 7814 goto done; 7815 } else 7816 p->u.params.weight = 0; 7817 } 7818 if (p->u.params.pktsize < 0) { 7819 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7820 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7821 rc = EINVAL; 7822 goto done; 7823 } else 7824 p->u.params.pktsize = 0; 7825 } 7826 7827 /* See what the firmware thinks of the request ... */ 7828 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode, 7829 fw_rateunit, fw_ratemode, p->u.params.channel, 7830 p->u.params.cl, p->u.params.minrate, p->u.params.maxrate, 7831 p->u.params.weight, p->u.params.pktsize, 1); 7832 goto done; 7833 } 7834 7835 rc = EINVAL; 7836 done: 7837 end_synchronized_op(sc, 0); 7838 return (rc); 7839 } 7840 7841 static int 7842 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p) 7843 { 7844 struct port_info *pi = NULL; 7845 struct sge_txq *txq; 7846 uint32_t fw_mnem, fw_queue, fw_class; 7847 int i, rc; 7848 7849 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq"); 7850 if (rc) 7851 return (rc); 7852 7853 if (!(sc->flags & FULL_INIT_DONE)) { 7854 rc = EAGAIN; 7855 goto done; 7856 } 7857 7858 if (p->port >= sc->params.nports) { 7859 rc = EINVAL; 7860 goto done; 7861 } 7862 7863 pi = sc->port[p->port]; 7864 if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) { 7865 rc = EINVAL; 7866 goto done; 7867 } 7868 7869 /* 7870 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX 7871 * Scheduling Class in this case). 7872 */ 7873 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 7874 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH)); 7875 fw_class = p->cl < 0 ? 0xffffffff : p->cl; 7876 7877 /* 7878 * If op.queue is non-negative, then we're only changing the scheduling 7879 * on a single specified TX queue. 7880 */ 7881 if (p->queue >= 0) { 7882 txq = &sc->sge.txq[pi->first_txq + p->queue]; 7883 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7884 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7885 &fw_class); 7886 goto done; 7887 } 7888 7889 /* 7890 * Change the scheduling on all the TX queues for the 7891 * interface. 7892 */ 7893 for_each_txq(pi, i, txq) { 7894 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7895 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7896 &fw_class); 7897 if (rc) 7898 goto done; 7899 } 7900 7901 rc = 0; 7902 done: 7903 end_synchronized_op(sc, 0); 7904 return (rc); 7905 } 7906 7907 int 7908 t4_os_find_pci_capability(struct adapter *sc, int cap) 7909 { 7910 int i; 7911 7912 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 7913 } 7914 7915 int 7916 t4_os_pci_save_state(struct adapter *sc) 7917 { 7918 device_t dev; 7919 struct pci_devinfo *dinfo; 7920 7921 dev = sc->dev; 7922 dinfo = device_get_ivars(dev); 7923 7924 pci_cfg_save(dev, dinfo, 0); 7925 return (0); 7926 } 7927 7928 int 7929 t4_os_pci_restore_state(struct adapter *sc) 7930 { 7931 device_t dev; 7932 struct pci_devinfo *dinfo; 7933 7934 dev = sc->dev; 7935 dinfo = device_get_ivars(dev); 7936 7937 pci_cfg_restore(dev, dinfo); 7938 return (0); 7939 } 7940 7941 void 7942 t4_os_portmod_changed(const struct adapter *sc, int idx) 7943 { 7944 struct port_info *pi = sc->port[idx]; 7945 static const char *mod_str[] = { 7946 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 7947 }; 7948 7949 build_medialist(pi, &pi->media); 7950 #ifdef DEV_NETMAP 7951 build_medialist(pi, &pi->nm_media); 7952 #endif 7953 7954 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 7955 if_printf(pi->ifp, "transceiver unplugged.\n"); 7956 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 7957 if_printf(pi->ifp, "unknown transceiver inserted.\n"); 7958 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 7959 if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 7960 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 7961 if_printf(pi->ifp, "%s transceiver inserted.\n", 7962 mod_str[pi->mod_type]); 7963 } else { 7964 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 7965 pi->mod_type); 7966 } 7967 } 7968 7969 void 7970 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason) 7971 { 7972 struct port_info *pi = sc->port[idx]; 7973 struct ifnet *ifp = pi->ifp; 7974 7975 if (link_stat) { 7976 pi->linkdnrc = -1; 7977 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 7978 if_link_state_change(ifp, LINK_STATE_UP); 7979 } else { 7980 if (reason >= 0) 7981 pi->linkdnrc = reason; 7982 if_link_state_change(ifp, LINK_STATE_DOWN); 7983 } 7984 } 7985 7986 void 7987 t4_iterate(void (*func)(struct adapter *, void *), void *arg) 7988 { 7989 struct adapter *sc; 7990 7991 sx_slock(&t4_list_lock); 7992 SLIST_FOREACH(sc, &t4_list, link) { 7993 /* 7994 * func should not make any assumptions about what state sc is 7995 * in - the only guarantee is that sc->sc_lock is a valid lock. 7996 */ 7997 func(sc, arg); 7998 } 7999 sx_sunlock(&t4_list_lock); 8000 } 8001 8002 static int 8003 t4_open(struct cdev *dev, int flags, int type, struct thread *td) 8004 { 8005 return (0); 8006 } 8007 8008 static int 8009 t4_close(struct cdev *dev, int flags, int type, struct thread *td) 8010 { 8011 return (0); 8012 } 8013 8014 static int 8015 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 8016 struct thread *td) 8017 { 8018 int rc; 8019 struct adapter *sc = dev->si_drv1; 8020 8021 rc = priv_check(td, PRIV_DRIVER); 8022 if (rc != 0) 8023 return (rc); 8024 8025 switch (cmd) { 8026 case CHELSIO_T4_GETREG: { 8027 struct t4_reg *edata = (struct t4_reg *)data; 8028 8029 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8030 return (EFAULT); 8031 8032 if (edata->size == 4) 8033 edata->val = t4_read_reg(sc, edata->addr); 8034 else if (edata->size == 8) 8035 edata->val = t4_read_reg64(sc, edata->addr); 8036 else 8037 return (EINVAL); 8038 8039 break; 8040 } 8041 case CHELSIO_T4_SETREG: { 8042 struct t4_reg *edata = (struct t4_reg *)data; 8043 8044 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8045 return (EFAULT); 8046 8047 if (edata->size == 4) { 8048 if (edata->val & 0xffffffff00000000) 8049 return (EINVAL); 8050 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 8051 } else if (edata->size == 8) 8052 t4_write_reg64(sc, edata->addr, edata->val); 8053 else 8054 return (EINVAL); 8055 break; 8056 } 8057 case CHELSIO_T4_REGDUMP: { 8058 struct t4_regdump *regs = (struct t4_regdump *)data; 8059 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 8060 uint8_t *buf; 8061 8062 if (regs->len < reglen) { 8063 regs->len = reglen; /* hint to the caller */ 8064 return (ENOBUFS); 8065 } 8066 8067 regs->len = reglen; 8068 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 8069 t4_get_regs(sc, regs, buf); 8070 rc = copyout(buf, regs->data, reglen); 8071 free(buf, M_CXGBE); 8072 break; 8073 } 8074 case CHELSIO_T4_GET_FILTER_MODE: 8075 rc = get_filter_mode(sc, (uint32_t *)data); 8076 break; 8077 case CHELSIO_T4_SET_FILTER_MODE: 8078 rc = set_filter_mode(sc, *(uint32_t *)data); 8079 break; 8080 case CHELSIO_T4_GET_FILTER: 8081 rc = get_filter(sc, (struct t4_filter *)data); 8082 break; 8083 case CHELSIO_T4_SET_FILTER: 8084 rc = set_filter(sc, (struct t4_filter *)data); 8085 break; 8086 case CHELSIO_T4_DEL_FILTER: 8087 rc = del_filter(sc, (struct t4_filter *)data); 8088 break; 8089 case CHELSIO_T4_GET_SGE_CONTEXT: 8090 rc = get_sge_context(sc, (struct t4_sge_context *)data); 8091 break; 8092 case CHELSIO_T4_LOAD_FW: 8093 rc = load_fw(sc, (struct t4_data *)data); 8094 break; 8095 case CHELSIO_T4_GET_MEM: 8096 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 8097 break; 8098 case CHELSIO_T4_GET_I2C: 8099 rc = read_i2c(sc, (struct t4_i2c_data *)data); 8100 break; 8101 case CHELSIO_T4_CLEAR_STATS: { 8102 int i; 8103 u_int port_id = *(uint32_t *)data; 8104 struct port_info *pi; 8105 8106 if (port_id >= sc->params.nports) 8107 return (EINVAL); 8108 pi = sc->port[port_id]; 8109 8110 /* MAC stats */ 8111 t4_clr_port_stats(sc, pi->tx_chan); 8112 pi->tx_parse_error = 0; 8113 8114 if (pi->flags & PORT_INIT_DONE) { 8115 struct sge_rxq *rxq; 8116 struct sge_txq *txq; 8117 struct sge_wrq *wrq; 8118 8119 for_each_rxq(pi, i, rxq) { 8120 #if defined(INET) || defined(INET6) 8121 rxq->lro.lro_queued = 0; 8122 rxq->lro.lro_flushed = 0; 8123 #endif 8124 rxq->rxcsum = 0; 8125 rxq->vlan_extraction = 0; 8126 } 8127 8128 for_each_txq(pi, i, txq) { 8129 txq->txcsum = 0; 8130 txq->tso_wrs = 0; 8131 txq->vlan_insertion = 0; 8132 txq->imm_wrs = 0; 8133 txq->sgl_wrs = 0; 8134 txq->txpkt_wrs = 0; 8135 txq->txpkts0_wrs = 0; 8136 txq->txpkts1_wrs = 0; 8137 txq->txpkts0_pkts = 0; 8138 txq->txpkts1_pkts = 0; 8139 mp_ring_reset_stats(txq->r); 8140 } 8141 8142 #ifdef TCP_OFFLOAD 8143 /* nothing to clear for each ofld_rxq */ 8144 8145 for_each_ofld_txq(pi, i, wrq) { 8146 wrq->tx_wrs_direct = 0; 8147 wrq->tx_wrs_copied = 0; 8148 } 8149 #endif 8150 wrq = &sc->sge.ctrlq[pi->port_id]; 8151 wrq->tx_wrs_direct = 0; 8152 wrq->tx_wrs_copied = 0; 8153 } 8154 break; 8155 } 8156 case CHELSIO_T4_SCHED_CLASS: 8157 rc = set_sched_class(sc, (struct t4_sched_params *)data); 8158 break; 8159 case CHELSIO_T4_SCHED_QUEUE: 8160 rc = set_sched_queue(sc, (struct t4_sched_queue *)data); 8161 break; 8162 case CHELSIO_T4_GET_TRACER: 8163 rc = t4_get_tracer(sc, (struct t4_tracer *)data); 8164 break; 8165 case CHELSIO_T4_SET_TRACER: 8166 rc = t4_set_tracer(sc, (struct t4_tracer *)data); 8167 break; 8168 default: 8169 rc = EINVAL; 8170 } 8171 8172 return (rc); 8173 } 8174 8175 #ifdef TCP_OFFLOAD 8176 void 8177 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask, 8178 const unsigned int *pgsz_order) 8179 { 8180 struct port_info *pi = ifp->if_softc; 8181 struct adapter *sc = pi->adapter; 8182 8183 t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask); 8184 t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) | 8185 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) | 8186 V_HPZ3(pgsz_order[3])); 8187 } 8188 8189 static int 8190 toe_capability(struct port_info *pi, int enable) 8191 { 8192 int rc; 8193 struct adapter *sc = pi->adapter; 8194 8195 ASSERT_SYNCHRONIZED_OP(sc); 8196 8197 if (!is_offload(sc)) 8198 return (ENODEV); 8199 8200 if (enable) { 8201 /* 8202 * We need the port's queues around so that we're able to send 8203 * and receive CPLs to/from the TOE even if the ifnet for this 8204 * port has never been UP'd administratively. 8205 */ 8206 if (!(pi->flags & PORT_INIT_DONE)) { 8207 rc = cxgbe_init_synchronized(pi); 8208 if (rc) 8209 return (rc); 8210 } 8211 8212 if (isset(&sc->offload_map, pi->port_id)) 8213 return (0); 8214 8215 if (!uld_active(sc, ULD_TOM)) { 8216 rc = t4_activate_uld(sc, ULD_TOM); 8217 if (rc == EAGAIN) { 8218 log(LOG_WARNING, 8219 "You must kldload t4_tom.ko before trying " 8220 "to enable TOE on a cxgbe interface.\n"); 8221 } 8222 if (rc != 0) 8223 return (rc); 8224 KASSERT(sc->tom_softc != NULL, 8225 ("%s: TOM activated but softc NULL", __func__)); 8226 KASSERT(uld_active(sc, ULD_TOM), 8227 ("%s: TOM activated but flag not set", __func__)); 8228 } 8229 8230 /* Activate iWARP and iSCSI too, if the modules are loaded. */ 8231 if (!uld_active(sc, ULD_IWARP)) 8232 (void) t4_activate_uld(sc, ULD_IWARP); 8233 if (!uld_active(sc, ULD_ISCSI)) 8234 (void) t4_activate_uld(sc, ULD_ISCSI); 8235 8236 setbit(&sc->offload_map, pi->port_id); 8237 } else { 8238 if (!isset(&sc->offload_map, pi->port_id)) 8239 return (0); 8240 8241 KASSERT(uld_active(sc, ULD_TOM), 8242 ("%s: TOM never initialized?", __func__)); 8243 clrbit(&sc->offload_map, pi->port_id); 8244 } 8245 8246 return (0); 8247 } 8248 8249 /* 8250 * Add an upper layer driver to the global list. 8251 */ 8252 int 8253 t4_register_uld(struct uld_info *ui) 8254 { 8255 int rc = 0; 8256 struct uld_info *u; 8257 8258 sx_xlock(&t4_uld_list_lock); 8259 SLIST_FOREACH(u, &t4_uld_list, link) { 8260 if (u->uld_id == ui->uld_id) { 8261 rc = EEXIST; 8262 goto done; 8263 } 8264 } 8265 8266 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 8267 ui->refcount = 0; 8268 done: 8269 sx_xunlock(&t4_uld_list_lock); 8270 return (rc); 8271 } 8272 8273 int 8274 t4_unregister_uld(struct uld_info *ui) 8275 { 8276 int rc = EINVAL; 8277 struct uld_info *u; 8278 8279 sx_xlock(&t4_uld_list_lock); 8280 8281 SLIST_FOREACH(u, &t4_uld_list, link) { 8282 if (u == ui) { 8283 if (ui->refcount > 0) { 8284 rc = EBUSY; 8285 goto done; 8286 } 8287 8288 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 8289 rc = 0; 8290 goto done; 8291 } 8292 } 8293 done: 8294 sx_xunlock(&t4_uld_list_lock); 8295 return (rc); 8296 } 8297 8298 int 8299 t4_activate_uld(struct adapter *sc, int id) 8300 { 8301 int rc; 8302 struct uld_info *ui; 8303 8304 ASSERT_SYNCHRONIZED_OP(sc); 8305 8306 if (id < 0 || id > ULD_MAX) 8307 return (EINVAL); 8308 rc = EAGAIN; /* kldoad the module with this ULD and try again. */ 8309 8310 sx_slock(&t4_uld_list_lock); 8311 8312 SLIST_FOREACH(ui, &t4_uld_list, link) { 8313 if (ui->uld_id == id) { 8314 if (!(sc->flags & FULL_INIT_DONE)) { 8315 rc = adapter_full_init(sc); 8316 if (rc != 0) 8317 break; 8318 } 8319 8320 rc = ui->activate(sc); 8321 if (rc == 0) { 8322 setbit(&sc->active_ulds, id); 8323 ui->refcount++; 8324 } 8325 break; 8326 } 8327 } 8328 8329 sx_sunlock(&t4_uld_list_lock); 8330 8331 return (rc); 8332 } 8333 8334 int 8335 t4_deactivate_uld(struct adapter *sc, int id) 8336 { 8337 int rc; 8338 struct uld_info *ui; 8339 8340 ASSERT_SYNCHRONIZED_OP(sc); 8341 8342 if (id < 0 || id > ULD_MAX) 8343 return (EINVAL); 8344 rc = ENXIO; 8345 8346 sx_slock(&t4_uld_list_lock); 8347 8348 SLIST_FOREACH(ui, &t4_uld_list, link) { 8349 if (ui->uld_id == id) { 8350 rc = ui->deactivate(sc); 8351 if (rc == 0) { 8352 clrbit(&sc->active_ulds, id); 8353 ui->refcount--; 8354 } 8355 break; 8356 } 8357 } 8358 8359 sx_sunlock(&t4_uld_list_lock); 8360 8361 return (rc); 8362 } 8363 8364 int 8365 uld_active(struct adapter *sc, int uld_id) 8366 { 8367 8368 MPASS(uld_id >= 0 && uld_id <= ULD_MAX); 8369 8370 return (isset(&sc->active_ulds, uld_id)); 8371 } 8372 #endif 8373 8374 /* 8375 * Come up with reasonable defaults for some of the tunables, provided they're 8376 * not set by the user (in which case we'll use the values as is). 8377 */ 8378 static void 8379 tweak_tunables(void) 8380 { 8381 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 8382 8383 if (t4_ntxq10g < 1) 8384 t4_ntxq10g = min(nc, NTXQ_10G); 8385 8386 if (t4_ntxq1g < 1) 8387 t4_ntxq1g = min(nc, NTXQ_1G); 8388 8389 if (t4_nrxq10g < 1) 8390 t4_nrxq10g = min(nc, NRXQ_10G); 8391 8392 if (t4_nrxq1g < 1) 8393 t4_nrxq1g = min(nc, NRXQ_1G); 8394 8395 #ifdef TCP_OFFLOAD 8396 if (t4_nofldtxq10g < 1) 8397 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 8398 8399 if (t4_nofldtxq1g < 1) 8400 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 8401 8402 if (t4_nofldrxq10g < 1) 8403 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 8404 8405 if (t4_nofldrxq1g < 1) 8406 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 8407 8408 if (t4_toecaps_allowed == -1) 8409 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 8410 #else 8411 if (t4_toecaps_allowed == -1) 8412 t4_toecaps_allowed = 0; 8413 #endif 8414 8415 #ifdef DEV_NETMAP 8416 if (t4_nnmtxq10g < 1) 8417 t4_nnmtxq10g = min(nc, NNMTXQ_10G); 8418 8419 if (t4_nnmtxq1g < 1) 8420 t4_nnmtxq1g = min(nc, NNMTXQ_1G); 8421 8422 if (t4_nnmrxq10g < 1) 8423 t4_nnmrxq10g = min(nc, NNMRXQ_10G); 8424 8425 if (t4_nnmrxq1g < 1) 8426 t4_nnmrxq1g = min(nc, NNMRXQ_1G); 8427 #endif 8428 8429 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 8430 t4_tmr_idx_10g = TMR_IDX_10G; 8431 8432 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 8433 t4_pktc_idx_10g = PKTC_IDX_10G; 8434 8435 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 8436 t4_tmr_idx_1g = TMR_IDX_1G; 8437 8438 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 8439 t4_pktc_idx_1g = PKTC_IDX_1G; 8440 8441 if (t4_qsize_txq < 128) 8442 t4_qsize_txq = 128; 8443 8444 if (t4_qsize_rxq < 128) 8445 t4_qsize_rxq = 128; 8446 while (t4_qsize_rxq & 7) 8447 t4_qsize_rxq++; 8448 8449 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 8450 } 8451 8452 static struct sx mlu; /* mod load unload */ 8453 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload"); 8454 8455 static int 8456 mod_event(module_t mod, int cmd, void *arg) 8457 { 8458 int rc = 0; 8459 static int loaded = 0; 8460 8461 switch (cmd) { 8462 case MOD_LOAD: 8463 sx_xlock(&mlu); 8464 if (loaded++ == 0) { 8465 t4_sge_modload(); 8466 sx_init(&t4_list_lock, "T4/T5 adapters"); 8467 SLIST_INIT(&t4_list); 8468 #ifdef TCP_OFFLOAD 8469 sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); 8470 SLIST_INIT(&t4_uld_list); 8471 #endif 8472 t4_tracer_modload(); 8473 tweak_tunables(); 8474 } 8475 sx_xunlock(&mlu); 8476 break; 8477 8478 case MOD_UNLOAD: 8479 sx_xlock(&mlu); 8480 if (--loaded == 0) { 8481 int tries; 8482 8483 sx_slock(&t4_list_lock); 8484 if (!SLIST_EMPTY(&t4_list)) { 8485 rc = EBUSY; 8486 sx_sunlock(&t4_list_lock); 8487 goto done_unload; 8488 } 8489 #ifdef TCP_OFFLOAD 8490 sx_slock(&t4_uld_list_lock); 8491 if (!SLIST_EMPTY(&t4_uld_list)) { 8492 rc = EBUSY; 8493 sx_sunlock(&t4_uld_list_lock); 8494 sx_sunlock(&t4_list_lock); 8495 goto done_unload; 8496 } 8497 #endif 8498 tries = 0; 8499 while (tries++ < 5 && t4_sge_extfree_refs() != 0) { 8500 uprintf("%ju clusters with custom free routine " 8501 "still is use.\n", t4_sge_extfree_refs()); 8502 pause("t4unload", 2 * hz); 8503 } 8504 #ifdef TCP_OFFLOAD 8505 sx_sunlock(&t4_uld_list_lock); 8506 #endif 8507 sx_sunlock(&t4_list_lock); 8508 8509 if (t4_sge_extfree_refs() == 0) { 8510 t4_tracer_modunload(); 8511 #ifdef TCP_OFFLOAD 8512 sx_destroy(&t4_uld_list_lock); 8513 #endif 8514 sx_destroy(&t4_list_lock); 8515 t4_sge_modunload(); 8516 loaded = 0; 8517 } else { 8518 rc = EBUSY; 8519 loaded++; /* undo earlier decrement */ 8520 } 8521 } 8522 done_unload: 8523 sx_xunlock(&mlu); 8524 break; 8525 } 8526 8527 return (rc); 8528 } 8529 8530 static devclass_t t4_devclass, t5_devclass; 8531 static devclass_t cxgbe_devclass, cxl_devclass; 8532 8533 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 8534 MODULE_VERSION(t4nex, 1); 8535 MODULE_DEPEND(t4nex, firmware, 1, 1, 1); 8536 8537 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 8538 MODULE_VERSION(t5nex, 1); 8539 MODULE_DEPEND(t5nex, firmware, 1, 1, 1); 8540 8541 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 8542 MODULE_VERSION(cxgbe, 1); 8543 8544 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 8545 MODULE_VERSION(cxl, 1); 8546