1 /*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_inet.h" 32 #include "opt_inet6.h" 33 34 #include <sys/param.h> 35 #include <sys/conf.h> 36 #include <sys/priv.h> 37 #include <sys/kernel.h> 38 #include <sys/bus.h> 39 #include <sys/module.h> 40 #include <sys/malloc.h> 41 #include <sys/queue.h> 42 #include <sys/taskqueue.h> 43 #include <sys/pciio.h> 44 #include <dev/pci/pcireg.h> 45 #include <dev/pci/pcivar.h> 46 #include <dev/pci/pci_private.h> 47 #include <sys/firmware.h> 48 #include <sys/sbuf.h> 49 #include <sys/smp.h> 50 #include <sys/socket.h> 51 #include <sys/sockio.h> 52 #include <sys/sysctl.h> 53 #include <net/ethernet.h> 54 #include <net/if.h> 55 #include <net/if_types.h> 56 #include <net/if_dl.h> 57 #include <net/if_vlan_var.h> 58 #if defined(__i386__) || defined(__amd64__) 59 #include <vm/vm.h> 60 #include <vm/pmap.h> 61 #endif 62 63 #include "common/common.h" 64 #include "common/t4_msg.h" 65 #include "common/t4_regs.h" 66 #include "common/t4_regs_values.h" 67 #include "t4_ioctl.h" 68 #include "t4_l2t.h" 69 #include "t4_mp_ring.h" 70 71 /* T4 bus driver interface */ 72 static int t4_probe(device_t); 73 static int t4_attach(device_t); 74 static int t4_detach(device_t); 75 static device_method_t t4_methods[] = { 76 DEVMETHOD(device_probe, t4_probe), 77 DEVMETHOD(device_attach, t4_attach), 78 DEVMETHOD(device_detach, t4_detach), 79 80 DEVMETHOD_END 81 }; 82 static driver_t t4_driver = { 83 "t4nex", 84 t4_methods, 85 sizeof(struct adapter) 86 }; 87 88 89 /* T4 port (cxgbe) interface */ 90 static int cxgbe_probe(device_t); 91 static int cxgbe_attach(device_t); 92 static int cxgbe_detach(device_t); 93 static device_method_t cxgbe_methods[] = { 94 DEVMETHOD(device_probe, cxgbe_probe), 95 DEVMETHOD(device_attach, cxgbe_attach), 96 DEVMETHOD(device_detach, cxgbe_detach), 97 { 0, 0 } 98 }; 99 static driver_t cxgbe_driver = { 100 "cxgbe", 101 cxgbe_methods, 102 sizeof(struct port_info) 103 }; 104 105 static d_ioctl_t t4_ioctl; 106 static d_open_t t4_open; 107 static d_close_t t4_close; 108 109 static struct cdevsw t4_cdevsw = { 110 .d_version = D_VERSION, 111 .d_flags = 0, 112 .d_open = t4_open, 113 .d_close = t4_close, 114 .d_ioctl = t4_ioctl, 115 .d_name = "t4nex", 116 }; 117 118 /* T5 bus driver interface */ 119 static int t5_probe(device_t); 120 static device_method_t t5_methods[] = { 121 DEVMETHOD(device_probe, t5_probe), 122 DEVMETHOD(device_attach, t4_attach), 123 DEVMETHOD(device_detach, t4_detach), 124 125 DEVMETHOD_END 126 }; 127 static driver_t t5_driver = { 128 "t5nex", 129 t5_methods, 130 sizeof(struct adapter) 131 }; 132 133 134 /* T5 port (cxl) interface */ 135 static driver_t cxl_driver = { 136 "cxl", 137 cxgbe_methods, 138 sizeof(struct port_info) 139 }; 140 141 static struct cdevsw t5_cdevsw = { 142 .d_version = D_VERSION, 143 .d_flags = 0, 144 .d_open = t4_open, 145 .d_close = t4_close, 146 .d_ioctl = t4_ioctl, 147 .d_name = "t5nex", 148 }; 149 150 /* ifnet + media interface */ 151 static void cxgbe_init(void *); 152 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 153 static int cxgbe_transmit(struct ifnet *, struct mbuf *); 154 static void cxgbe_qflush(struct ifnet *); 155 static uint64_t cxgbe_get_counter(struct ifnet *, ift_counter); 156 static int cxgbe_media_change(struct ifnet *); 157 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 158 159 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 160 161 /* 162 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 163 * then ADAPTER_LOCK, then t4_uld_list_lock. 164 */ 165 static struct sx t4_list_lock; 166 SLIST_HEAD(, adapter) t4_list; 167 #ifdef TCP_OFFLOAD 168 static struct sx t4_uld_list_lock; 169 SLIST_HEAD(, uld_info) t4_uld_list; 170 #endif 171 172 /* 173 * Tunables. See tweak_tunables() too. 174 * 175 * Each tunable is set to a default value here if it's known at compile-time. 176 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 177 * provide a reasonable default when the driver is loaded. 178 * 179 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 180 * T5 are under hw.cxl. 181 */ 182 183 /* 184 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 185 */ 186 #define NTXQ_10G 16 187 static int t4_ntxq10g = -1; 188 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 189 190 #define NRXQ_10G 8 191 static int t4_nrxq10g = -1; 192 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 193 194 #define NTXQ_1G 4 195 static int t4_ntxq1g = -1; 196 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 197 198 #define NRXQ_1G 2 199 static int t4_nrxq1g = -1; 200 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 201 202 static int t4_rsrv_noflowq = 0; 203 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq); 204 205 #ifdef TCP_OFFLOAD 206 #define NOFLDTXQ_10G 8 207 static int t4_nofldtxq10g = -1; 208 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 209 210 #define NOFLDRXQ_10G 2 211 static int t4_nofldrxq10g = -1; 212 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 213 214 #define NOFLDTXQ_1G 2 215 static int t4_nofldtxq1g = -1; 216 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 217 218 #define NOFLDRXQ_1G 1 219 static int t4_nofldrxq1g = -1; 220 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 221 #endif 222 223 #ifdef DEV_NETMAP 224 #define NNMTXQ_10G 2 225 static int t4_nnmtxq10g = -1; 226 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g); 227 228 #define NNMRXQ_10G 2 229 static int t4_nnmrxq10g = -1; 230 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g); 231 232 #define NNMTXQ_1G 1 233 static int t4_nnmtxq1g = -1; 234 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g); 235 236 #define NNMRXQ_1G 1 237 static int t4_nnmrxq1g = -1; 238 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g); 239 #endif 240 241 /* 242 * Holdoff parameters for 10G and 1G ports. 243 */ 244 #define TMR_IDX_10G 1 245 static int t4_tmr_idx_10g = TMR_IDX_10G; 246 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 247 248 #define PKTC_IDX_10G (-1) 249 static int t4_pktc_idx_10g = PKTC_IDX_10G; 250 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 251 252 #define TMR_IDX_1G 1 253 static int t4_tmr_idx_1g = TMR_IDX_1G; 254 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 255 256 #define PKTC_IDX_1G (-1) 257 static int t4_pktc_idx_1g = PKTC_IDX_1G; 258 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 259 260 /* 261 * Size (# of entries) of each tx and rx queue. 262 */ 263 static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 264 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 265 266 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 267 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 268 269 /* 270 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 271 */ 272 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 273 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 274 275 /* 276 * Configuration file. 277 */ 278 #define DEFAULT_CF "default" 279 #define FLASH_CF "flash" 280 #define UWIRE_CF "uwire" 281 #define FPGA_CF "fpga" 282 static char t4_cfg_file[32] = DEFAULT_CF; 283 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 284 285 /* 286 * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively). 287 * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them. 288 * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water 289 * mark or when signalled to do so, 0 to never emit PAUSE. 290 */ 291 static int t4_pause_settings = PAUSE_TX | PAUSE_RX; 292 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings); 293 294 /* 295 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 296 * encouraged respectively). 297 */ 298 static unsigned int t4_fw_install = 1; 299 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 300 301 /* 302 * ASIC features that will be used. Disable the ones you don't want so that the 303 * chip resources aren't wasted on features that will not be used. 304 */ 305 static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 306 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 307 308 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 309 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 310 311 static int t4_toecaps_allowed = -1; 312 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 313 314 static int t4_rdmacaps_allowed = 0; 315 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 316 317 static int t4_iscsicaps_allowed = 0; 318 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 319 320 static int t4_fcoecaps_allowed = 0; 321 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 322 323 static int t5_write_combine = 0; 324 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 325 326 struct intrs_and_queues { 327 uint16_t intr_type; /* INTx, MSI, or MSI-X */ 328 uint16_t nirq; /* Total # of vectors */ 329 uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */ 330 uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */ 331 uint16_t ntxq10g; /* # of NIC txq's for each 10G port */ 332 uint16_t nrxq10g; /* # of NIC rxq's for each 10G port */ 333 uint16_t ntxq1g; /* # of NIC txq's for each 1G port */ 334 uint16_t nrxq1g; /* # of NIC rxq's for each 1G port */ 335 uint16_t rsrv_noflowq; /* Flag whether to reserve queue 0 */ 336 #ifdef TCP_OFFLOAD 337 uint16_t nofldtxq10g; /* # of TOE txq's for each 10G port */ 338 uint16_t nofldrxq10g; /* # of TOE rxq's for each 10G port */ 339 uint16_t nofldtxq1g; /* # of TOE txq's for each 1G port */ 340 uint16_t nofldrxq1g; /* # of TOE rxq's for each 1G port */ 341 #endif 342 #ifdef DEV_NETMAP 343 uint16_t nnmtxq10g; /* # of netmap txq's for each 10G port */ 344 uint16_t nnmrxq10g; /* # of netmap rxq's for each 10G port */ 345 uint16_t nnmtxq1g; /* # of netmap txq's for each 1G port */ 346 uint16_t nnmrxq1g; /* # of netmap rxq's for each 1G port */ 347 #endif 348 }; 349 350 struct filter_entry { 351 uint32_t valid:1; /* filter allocated and valid */ 352 uint32_t locked:1; /* filter is administratively locked */ 353 uint32_t pending:1; /* filter action is pending firmware reply */ 354 uint32_t smtidx:8; /* Source MAC Table index for smac */ 355 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 356 357 struct t4_filter_specification fs; 358 }; 359 360 static int map_bars_0_and_4(struct adapter *); 361 static int map_bar_2(struct adapter *); 362 static void setup_memwin(struct adapter *); 363 static int validate_mem_range(struct adapter *, uint32_t, int); 364 static int fwmtype_to_hwmtype(int); 365 static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 366 uint32_t *); 367 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 368 static uint32_t position_memwin(struct adapter *, int, uint32_t); 369 static int cfg_itype_and_nqueues(struct adapter *, int, int, 370 struct intrs_and_queues *); 371 static int prep_firmware(struct adapter *); 372 static int partition_resources(struct adapter *, const struct firmware *, 373 const char *); 374 static int get_params__pre_init(struct adapter *); 375 static int get_params__post_init(struct adapter *); 376 static int set_params__post_init(struct adapter *); 377 static void t4_set_desc(struct adapter *); 378 static void build_medialist(struct port_info *, struct ifmedia *); 379 static int cxgbe_init_synchronized(struct port_info *); 380 static int cxgbe_uninit_synchronized(struct port_info *); 381 static int setup_intr_handlers(struct adapter *); 382 static void quiesce_txq(struct adapter *, struct sge_txq *); 383 static void quiesce_wrq(struct adapter *, struct sge_wrq *); 384 static void quiesce_iq(struct adapter *, struct sge_iq *); 385 static void quiesce_fl(struct adapter *, struct sge_fl *); 386 static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 387 driver_intr_t *, void *, char *); 388 static int t4_free_irq(struct adapter *, struct irq *); 389 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 390 unsigned int); 391 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 392 static void cxgbe_refresh_stats(struct adapter *, struct port_info *); 393 static void cxgbe_tick(void *); 394 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 395 static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 396 struct mbuf *); 397 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 398 static int fw_msg_not_handled(struct adapter *, const __be64 *); 399 static int t4_sysctls(struct adapter *); 400 static int cxgbe_sysctls(struct port_info *); 401 static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 402 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 403 static int sysctl_btphy(SYSCTL_HANDLER_ARGS); 404 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); 405 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 406 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 407 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 408 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 409 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS); 410 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 411 static int sysctl_temperature(SYSCTL_HANDLER_ARGS); 412 #ifdef SBUF_DRAIN 413 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 414 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 415 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 416 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); 417 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); 418 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 419 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 420 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 421 static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 422 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 423 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 424 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 425 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); 426 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 427 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); 428 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 429 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 430 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 431 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 432 static int sysctl_tids(SYSCTL_HANDLER_ARGS); 433 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 434 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 435 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 436 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 437 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 438 #endif 439 static uint32_t fconf_to_mode(uint32_t); 440 static uint32_t mode_to_fconf(uint32_t); 441 static uint32_t fspec_to_fconf(struct t4_filter_specification *); 442 static int get_filter_mode(struct adapter *, uint32_t *); 443 static int set_filter_mode(struct adapter *, uint32_t); 444 static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 445 static int get_filter(struct adapter *, struct t4_filter *); 446 static int set_filter(struct adapter *, struct t4_filter *); 447 static int del_filter(struct adapter *, struct t4_filter *); 448 static void clear_filter(struct filter_entry *); 449 static int set_filter_wr(struct adapter *, int); 450 static int del_filter_wr(struct adapter *, int); 451 static int get_sge_context(struct adapter *, struct t4_sge_context *); 452 static int load_fw(struct adapter *, struct t4_data *); 453 static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 454 static int read_i2c(struct adapter *, struct t4_i2c_data *); 455 static int set_sched_class(struct adapter *, struct t4_sched_params *); 456 static int set_sched_queue(struct adapter *, struct t4_sched_queue *); 457 #ifdef TCP_OFFLOAD 458 static int toe_capability(struct port_info *, int); 459 #endif 460 static int mod_event(module_t, int, void *); 461 462 struct { 463 uint16_t device; 464 char *desc; 465 } t4_pciids[] = { 466 {0xa000, "Chelsio Terminator 4 FPGA"}, 467 {0x4400, "Chelsio T440-dbg"}, 468 {0x4401, "Chelsio T420-CR"}, 469 {0x4402, "Chelsio T422-CR"}, 470 {0x4403, "Chelsio T440-CR"}, 471 {0x4404, "Chelsio T420-BCH"}, 472 {0x4405, "Chelsio T440-BCH"}, 473 {0x4406, "Chelsio T440-CH"}, 474 {0x4407, "Chelsio T420-SO"}, 475 {0x4408, "Chelsio T420-CX"}, 476 {0x4409, "Chelsio T420-BT"}, 477 {0x440a, "Chelsio T404-BT"}, 478 {0x440e, "Chelsio T440-LP-CR"}, 479 }, t5_pciids[] = { 480 {0xb000, "Chelsio Terminator 5 FPGA"}, 481 {0x5400, "Chelsio T580-dbg"}, 482 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ 483 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ 484 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ 485 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ 486 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ 487 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ 488 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ 489 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ 490 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 491 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ 492 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ 493 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ 494 {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */ 495 #ifdef notyet 496 {0x5404, "Chelsio T520-BCH"}, 497 {0x5405, "Chelsio T540-BCH"}, 498 {0x5406, "Chelsio T540-CH"}, 499 {0x5408, "Chelsio T520-CX"}, 500 {0x540b, "Chelsio B520-SR"}, 501 {0x540c, "Chelsio B504-BT"}, 502 {0x540f, "Chelsio Amsterdam"}, 503 {0x5413, "Chelsio T580-CHR"}, 504 #endif 505 }; 506 507 #ifdef TCP_OFFLOAD 508 /* 509 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 510 * exactly the same for both rxq and ofld_rxq. 511 */ 512 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 513 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 514 #endif 515 516 /* No easy way to include t4_msg.h before adapter.h so we check this way */ 517 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 518 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 519 520 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); 521 522 static int 523 t4_probe(device_t dev) 524 { 525 int i; 526 uint16_t v = pci_get_vendor(dev); 527 uint16_t d = pci_get_device(dev); 528 uint8_t f = pci_get_function(dev); 529 530 if (v != PCI_VENDOR_ID_CHELSIO) 531 return (ENXIO); 532 533 /* Attach only to PF0 of the FPGA */ 534 if (d == 0xa000 && f != 0) 535 return (ENXIO); 536 537 for (i = 0; i < nitems(t4_pciids); i++) { 538 if (d == t4_pciids[i].device) { 539 device_set_desc(dev, t4_pciids[i].desc); 540 return (BUS_PROBE_DEFAULT); 541 } 542 } 543 544 return (ENXIO); 545 } 546 547 static int 548 t5_probe(device_t dev) 549 { 550 int i; 551 uint16_t v = pci_get_vendor(dev); 552 uint16_t d = pci_get_device(dev); 553 uint8_t f = pci_get_function(dev); 554 555 if (v != PCI_VENDOR_ID_CHELSIO) 556 return (ENXIO); 557 558 /* Attach only to PF0 of the FPGA */ 559 if (d == 0xb000 && f != 0) 560 return (ENXIO); 561 562 for (i = 0; i < nitems(t5_pciids); i++) { 563 if (d == t5_pciids[i].device) { 564 device_set_desc(dev, t5_pciids[i].desc); 565 return (BUS_PROBE_DEFAULT); 566 } 567 } 568 569 return (ENXIO); 570 } 571 572 static int 573 t4_attach(device_t dev) 574 { 575 struct adapter *sc; 576 int rc = 0, i, n10g, n1g, rqidx, tqidx; 577 struct intrs_and_queues iaq; 578 struct sge *s; 579 #ifdef TCP_OFFLOAD 580 int ofld_rqidx, ofld_tqidx; 581 #endif 582 #ifdef DEV_NETMAP 583 int nm_rqidx, nm_tqidx; 584 #endif 585 586 sc = device_get_softc(dev); 587 sc->dev = dev; 588 TUNABLE_INT_FETCH("hw.cxgbe.debug_flags", &sc->debug_flags); 589 590 pci_enable_busmaster(dev); 591 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 592 uint32_t v; 593 594 pci_set_max_read_req(dev, 4096); 595 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 596 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 597 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 598 599 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5); 600 } 601 602 sc->traceq = -1; 603 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); 604 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", 605 device_get_nameunit(dev)); 606 607 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 608 device_get_nameunit(dev)); 609 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 610 sx_xlock(&t4_list_lock); 611 SLIST_INSERT_HEAD(&t4_list, sc, link); 612 sx_xunlock(&t4_list_lock); 613 614 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 615 TAILQ_INIT(&sc->sfl); 616 callout_init(&sc->sfl_callout, 1); 617 618 mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF); 619 620 rc = map_bars_0_and_4(sc); 621 if (rc != 0) 622 goto done; /* error message displayed already */ 623 624 /* 625 * This is the real PF# to which we're attaching. Works from within PCI 626 * passthrough environments too, where pci_get_function() could return a 627 * different PF# depending on the passthrough configuration. We need to 628 * use the real PF# in all our communication with the firmware. 629 */ 630 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 631 sc->mbox = sc->pf; 632 633 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 634 sc->an_handler = an_not_handled; 635 for (i = 0; i < nitems(sc->cpl_handler); i++) 636 sc->cpl_handler[i] = cpl_not_handled; 637 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 638 sc->fw_msg_handler[i] = fw_msg_not_handled; 639 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 640 t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt); 641 t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt); 642 t4_init_sge_cpl_handlers(sc); 643 644 /* Prepare the adapter for operation */ 645 rc = -t4_prep_adapter(sc); 646 if (rc != 0) { 647 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 648 goto done; 649 } 650 651 /* 652 * Do this really early, with the memory windows set up even before the 653 * character device. The userland tool's register i/o and mem read 654 * will work even in "recovery mode". 655 */ 656 setup_memwin(sc); 657 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 658 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 659 device_get_nameunit(dev)); 660 if (sc->cdev == NULL) 661 device_printf(dev, "failed to create nexus char device.\n"); 662 else 663 sc->cdev->si_drv1 = sc; 664 665 /* Go no further if recovery mode has been requested. */ 666 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 667 device_printf(dev, "recovery mode.\n"); 668 goto done; 669 } 670 671 #if defined(__i386__) 672 if ((cpu_feature & CPUID_CX8) == 0) { 673 device_printf(dev, "64 bit atomics not available.\n"); 674 rc = ENOTSUP; 675 goto done; 676 } 677 #endif 678 679 /* Prepare the firmware for operation */ 680 rc = prep_firmware(sc); 681 if (rc != 0) 682 goto done; /* error message displayed already */ 683 684 rc = get_params__post_init(sc); 685 if (rc != 0) 686 goto done; /* error message displayed already */ 687 688 rc = set_params__post_init(sc); 689 if (rc != 0) 690 goto done; /* error message displayed already */ 691 692 rc = map_bar_2(sc); 693 if (rc != 0) 694 goto done; /* error message displayed already */ 695 696 rc = t4_create_dma_tag(sc); 697 if (rc != 0) 698 goto done; /* error message displayed already */ 699 700 /* 701 * First pass over all the ports - allocate VIs and initialize some 702 * basic parameters like mac address, port type, etc. We also figure 703 * out whether a port is 10G or 1G and use that information when 704 * calculating how many interrupts to attempt to allocate. 705 */ 706 n10g = n1g = 0; 707 for_each_port(sc, i) { 708 struct port_info *pi; 709 710 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 711 sc->port[i] = pi; 712 713 /* These must be set before t4_port_init */ 714 pi->adapter = sc; 715 pi->port_id = i; 716 717 /* Allocate the vi and initialize parameters like mac addr */ 718 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 719 if (rc != 0) { 720 device_printf(dev, "unable to initialize port %d: %d\n", 721 i, rc); 722 free(pi, M_CXGBE); 723 sc->port[i] = NULL; 724 goto done; 725 } 726 727 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX); 728 pi->link_cfg.requested_fc |= t4_pause_settings; 729 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX); 730 pi->link_cfg.fc |= t4_pause_settings; 731 732 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 733 if (rc != 0) { 734 device_printf(dev, "port %d l1cfg failed: %d\n", i, rc); 735 free(pi, M_CXGBE); 736 sc->port[i] = NULL; 737 goto done; 738 } 739 740 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 741 device_get_nameunit(dev), i); 742 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 743 sc->chan_map[pi->tx_chan] = i; 744 745 if (is_10G_port(pi) || is_40G_port(pi)) { 746 n10g++; 747 pi->tmr_idx = t4_tmr_idx_10g; 748 pi->pktc_idx = t4_pktc_idx_10g; 749 } else { 750 n1g++; 751 pi->tmr_idx = t4_tmr_idx_1g; 752 pi->pktc_idx = t4_pktc_idx_1g; 753 } 754 755 pi->xact_addr_filt = -1; 756 pi->linkdnrc = -1; 757 758 pi->qsize_rxq = t4_qsize_rxq; 759 pi->qsize_txq = t4_qsize_txq; 760 761 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 762 if (pi->dev == NULL) { 763 device_printf(dev, 764 "failed to add device for port %d.\n", i); 765 rc = ENXIO; 766 goto done; 767 } 768 device_set_softc(pi->dev, pi); 769 } 770 771 /* 772 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 773 */ 774 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 775 if (rc != 0) 776 goto done; /* error message displayed already */ 777 778 sc->intr_type = iaq.intr_type; 779 sc->intr_count = iaq.nirq; 780 781 s = &sc->sge; 782 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 783 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 784 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 785 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 786 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 787 #ifdef TCP_OFFLOAD 788 if (is_offload(sc)) { 789 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 790 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 791 s->neq += s->nofldtxq + s->nofldrxq; 792 s->niq += s->nofldrxq; 793 794 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 795 M_CXGBE, M_ZERO | M_WAITOK); 796 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 797 M_CXGBE, M_ZERO | M_WAITOK); 798 } 799 #endif 800 #ifdef DEV_NETMAP 801 s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g; 802 s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g; 803 s->neq += s->nnmtxq + s->nnmrxq; 804 s->niq += s->nnmrxq; 805 806 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq), 807 M_CXGBE, M_ZERO | M_WAITOK); 808 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq), 809 M_CXGBE, M_ZERO | M_WAITOK); 810 #endif 811 812 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 813 M_ZERO | M_WAITOK); 814 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 815 M_ZERO | M_WAITOK); 816 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 817 M_ZERO | M_WAITOK); 818 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 819 M_ZERO | M_WAITOK); 820 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 821 M_ZERO | M_WAITOK); 822 823 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 824 M_ZERO | M_WAITOK); 825 826 t4_init_l2t(sc, M_WAITOK); 827 828 /* 829 * Second pass over the ports. This time we know the number of rx and 830 * tx queues that each port should get. 831 */ 832 rqidx = tqidx = 0; 833 #ifdef TCP_OFFLOAD 834 ofld_rqidx = ofld_tqidx = 0; 835 #endif 836 #ifdef DEV_NETMAP 837 nm_rqidx = nm_tqidx = 0; 838 #endif 839 for_each_port(sc, i) { 840 struct port_info *pi = sc->port[i]; 841 842 if (pi == NULL) 843 continue; 844 845 pi->first_rxq = rqidx; 846 pi->first_txq = tqidx; 847 if (is_10G_port(pi) || is_40G_port(pi)) { 848 pi->flags |= iaq.intr_flags_10g; 849 pi->nrxq = iaq.nrxq10g; 850 pi->ntxq = iaq.ntxq10g; 851 } else { 852 pi->flags |= iaq.intr_flags_1g; 853 pi->nrxq = iaq.nrxq1g; 854 pi->ntxq = iaq.ntxq1g; 855 } 856 857 if (pi->ntxq > 1) 858 pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0; 859 else 860 pi->rsrv_noflowq = 0; 861 862 rqidx += pi->nrxq; 863 tqidx += pi->ntxq; 864 #ifdef TCP_OFFLOAD 865 if (is_offload(sc)) { 866 pi->first_ofld_rxq = ofld_rqidx; 867 pi->first_ofld_txq = ofld_tqidx; 868 if (is_10G_port(pi) || is_40G_port(pi)) { 869 pi->nofldrxq = iaq.nofldrxq10g; 870 pi->nofldtxq = iaq.nofldtxq10g; 871 } else { 872 pi->nofldrxq = iaq.nofldrxq1g; 873 pi->nofldtxq = iaq.nofldtxq1g; 874 } 875 ofld_rqidx += pi->nofldrxq; 876 ofld_tqidx += pi->nofldtxq; 877 } 878 #endif 879 #ifdef DEV_NETMAP 880 pi->first_nm_rxq = nm_rqidx; 881 pi->first_nm_txq = nm_tqidx; 882 if (is_10G_port(pi) || is_40G_port(pi)) { 883 pi->nnmrxq = iaq.nnmrxq10g; 884 pi->nnmtxq = iaq.nnmtxq10g; 885 } else { 886 pi->nnmrxq = iaq.nnmrxq1g; 887 pi->nnmtxq = iaq.nnmtxq1g; 888 } 889 nm_rqidx += pi->nnmrxq; 890 nm_tqidx += pi->nnmtxq; 891 #endif 892 } 893 894 rc = setup_intr_handlers(sc); 895 if (rc != 0) { 896 device_printf(dev, 897 "failed to setup interrupt handlers: %d\n", rc); 898 goto done; 899 } 900 901 rc = bus_generic_attach(dev); 902 if (rc != 0) { 903 device_printf(dev, 904 "failed to attach all child ports: %d\n", rc); 905 goto done; 906 } 907 908 device_printf(dev, 909 "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 910 sc->params.pci.speed, sc->params.pci.width, sc->params.nports, 911 sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" : 912 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 913 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 914 915 t4_set_desc(sc); 916 917 done: 918 if (rc != 0 && sc->cdev) { 919 /* cdev was created and so cxgbetool works; recover that way. */ 920 device_printf(dev, 921 "error during attach, adapter is now in recovery mode.\n"); 922 rc = 0; 923 } 924 925 if (rc != 0) 926 t4_detach(dev); 927 else 928 t4_sysctls(sc); 929 930 return (rc); 931 } 932 933 /* 934 * Idempotent 935 */ 936 static int 937 t4_detach(device_t dev) 938 { 939 struct adapter *sc; 940 struct port_info *pi; 941 int i, rc; 942 943 sc = device_get_softc(dev); 944 945 if (sc->flags & FULL_INIT_DONE) 946 t4_intr_disable(sc); 947 948 if (sc->cdev) { 949 destroy_dev(sc->cdev); 950 sc->cdev = NULL; 951 } 952 953 rc = bus_generic_detach(dev); 954 if (rc) { 955 device_printf(dev, 956 "failed to detach child devices: %d\n", rc); 957 return (rc); 958 } 959 960 for (i = 0; i < sc->intr_count; i++) 961 t4_free_irq(sc, &sc->irq[i]); 962 963 for (i = 0; i < MAX_NPORTS; i++) { 964 pi = sc->port[i]; 965 if (pi) { 966 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid); 967 if (pi->dev) 968 device_delete_child(dev, pi->dev); 969 970 mtx_destroy(&pi->pi_lock); 971 free(pi, M_CXGBE); 972 } 973 } 974 975 if (sc->flags & FULL_INIT_DONE) 976 adapter_full_uninit(sc); 977 978 if (sc->flags & FW_OK) 979 t4_fw_bye(sc, sc->mbox); 980 981 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 982 pci_release_msi(dev); 983 984 if (sc->regs_res) 985 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 986 sc->regs_res); 987 988 if (sc->udbs_res) 989 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 990 sc->udbs_res); 991 992 if (sc->msix_res) 993 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 994 sc->msix_res); 995 996 if (sc->l2t) 997 t4_free_l2t(sc->l2t); 998 999 #ifdef TCP_OFFLOAD 1000 free(sc->sge.ofld_rxq, M_CXGBE); 1001 free(sc->sge.ofld_txq, M_CXGBE); 1002 #endif 1003 #ifdef DEV_NETMAP 1004 free(sc->sge.nm_rxq, M_CXGBE); 1005 free(sc->sge.nm_txq, M_CXGBE); 1006 #endif 1007 free(sc->irq, M_CXGBE); 1008 free(sc->sge.rxq, M_CXGBE); 1009 free(sc->sge.txq, M_CXGBE); 1010 free(sc->sge.ctrlq, M_CXGBE); 1011 free(sc->sge.iqmap, M_CXGBE); 1012 free(sc->sge.eqmap, M_CXGBE); 1013 free(sc->tids.ftid_tab, M_CXGBE); 1014 t4_destroy_dma_tag(sc); 1015 if (mtx_initialized(&sc->sc_lock)) { 1016 sx_xlock(&t4_list_lock); 1017 SLIST_REMOVE(&t4_list, sc, adapter, link); 1018 sx_xunlock(&t4_list_lock); 1019 mtx_destroy(&sc->sc_lock); 1020 } 1021 1022 if (mtx_initialized(&sc->tids.ftid_lock)) 1023 mtx_destroy(&sc->tids.ftid_lock); 1024 if (mtx_initialized(&sc->sfl_lock)) 1025 mtx_destroy(&sc->sfl_lock); 1026 if (mtx_initialized(&sc->ifp_lock)) 1027 mtx_destroy(&sc->ifp_lock); 1028 if (mtx_initialized(&sc->regwin_lock)) 1029 mtx_destroy(&sc->regwin_lock); 1030 1031 bzero(sc, sizeof(*sc)); 1032 1033 return (0); 1034 } 1035 1036 static int 1037 cxgbe_probe(device_t dev) 1038 { 1039 char buf[128]; 1040 struct port_info *pi = device_get_softc(dev); 1041 1042 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 1043 device_set_desc_copy(dev, buf); 1044 1045 return (BUS_PROBE_DEFAULT); 1046 } 1047 1048 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 1049 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 1050 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS) 1051 #define T4_CAP_ENABLE (T4_CAP) 1052 1053 static int 1054 cxgbe_attach(device_t dev) 1055 { 1056 struct port_info *pi = device_get_softc(dev); 1057 struct ifnet *ifp; 1058 char *s; 1059 int n, o; 1060 1061 /* Allocate an ifnet and set it up */ 1062 ifp = if_alloc(IFT_ETHER); 1063 if (ifp == NULL) { 1064 device_printf(dev, "Cannot allocate ifnet\n"); 1065 return (ENOMEM); 1066 } 1067 pi->ifp = ifp; 1068 ifp->if_softc = pi; 1069 1070 callout_init(&pi->tick, 1); 1071 1072 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1073 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1074 1075 ifp->if_init = cxgbe_init; 1076 ifp->if_ioctl = cxgbe_ioctl; 1077 ifp->if_transmit = cxgbe_transmit; 1078 ifp->if_qflush = cxgbe_qflush; 1079 ifp->if_get_counter = cxgbe_get_counter; 1080 1081 ifp->if_capabilities = T4_CAP; 1082 #ifdef TCP_OFFLOAD 1083 if (is_offload(pi->adapter)) 1084 ifp->if_capabilities |= IFCAP_TOE; 1085 #endif 1086 ifp->if_capenable = T4_CAP_ENABLE; 1087 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1088 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1089 1090 ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1091 ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS; 1092 ifp->if_hw_tsomaxsegsize = 65536; 1093 1094 /* Initialize ifmedia for this port */ 1095 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 1096 cxgbe_media_status); 1097 build_medialist(pi, &pi->media); 1098 1099 pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 1100 EVENTHANDLER_PRI_ANY); 1101 1102 ether_ifattach(ifp, pi->hw_addr); 1103 1104 n = 128; 1105 s = malloc(n, M_CXGBE, M_WAITOK); 1106 o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq); 1107 MPASS(n > o); 1108 #ifdef TCP_OFFLOAD 1109 if (is_offload(pi->adapter)) { 1110 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)", 1111 pi->nofldtxq, pi->nofldrxq); 1112 MPASS(n > o); 1113 } 1114 #endif 1115 #ifdef DEV_NETMAP 1116 o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq, 1117 pi->nnmrxq); 1118 MPASS(n > o); 1119 #endif 1120 device_printf(dev, "%s\n", s); 1121 free(s, M_CXGBE); 1122 1123 #ifdef DEV_NETMAP 1124 /* nm_media handled here to keep implementation private to this file */ 1125 ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change, 1126 cxgbe_media_status); 1127 build_medialist(pi, &pi->nm_media); 1128 create_netmap_ifnet(pi); /* logs errors it something fails */ 1129 #endif 1130 cxgbe_sysctls(pi); 1131 1132 return (0); 1133 } 1134 1135 static int 1136 cxgbe_detach(device_t dev) 1137 { 1138 struct port_info *pi = device_get_softc(dev); 1139 struct adapter *sc = pi->adapter; 1140 struct ifnet *ifp = pi->ifp; 1141 1142 /* Tell if_ioctl and if_init that the port is going away */ 1143 ADAPTER_LOCK(sc); 1144 SET_DOOMED(pi); 1145 wakeup(&sc->flags); 1146 while (IS_BUSY(sc)) 1147 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 1148 SET_BUSY(sc); 1149 #ifdef INVARIANTS 1150 sc->last_op = "t4detach"; 1151 sc->last_op_thr = curthread; 1152 #endif 1153 ADAPTER_UNLOCK(sc); 1154 1155 if (pi->flags & HAS_TRACEQ) { 1156 sc->traceq = -1; /* cloner should not create ifnet */ 1157 t4_tracer_port_detach(sc); 1158 } 1159 1160 if (pi->vlan_c) 1161 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c); 1162 1163 PORT_LOCK(pi); 1164 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1165 callout_stop(&pi->tick); 1166 PORT_UNLOCK(pi); 1167 callout_drain(&pi->tick); 1168 1169 /* Let detach proceed even if these fail. */ 1170 cxgbe_uninit_synchronized(pi); 1171 port_full_uninit(pi); 1172 1173 ifmedia_removeall(&pi->media); 1174 ether_ifdetach(pi->ifp); 1175 if_free(pi->ifp); 1176 1177 #ifdef DEV_NETMAP 1178 /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */ 1179 destroy_netmap_ifnet(pi); 1180 #endif 1181 1182 ADAPTER_LOCK(sc); 1183 CLR_BUSY(sc); 1184 wakeup(&sc->flags); 1185 ADAPTER_UNLOCK(sc); 1186 1187 return (0); 1188 } 1189 1190 static void 1191 cxgbe_init(void *arg) 1192 { 1193 struct port_info *pi = arg; 1194 struct adapter *sc = pi->adapter; 1195 1196 if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0) 1197 return; 1198 cxgbe_init_synchronized(pi); 1199 end_synchronized_op(sc, 0); 1200 } 1201 1202 static int 1203 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1204 { 1205 int rc = 0, mtu, flags, can_sleep; 1206 struct port_info *pi = ifp->if_softc; 1207 struct adapter *sc = pi->adapter; 1208 struct ifreq *ifr = (struct ifreq *)data; 1209 uint32_t mask; 1210 1211 switch (cmd) { 1212 case SIOCSIFMTU: 1213 mtu = ifr->ifr_mtu; 1214 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1215 return (EINVAL); 1216 1217 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu"); 1218 if (rc) 1219 return (rc); 1220 ifp->if_mtu = mtu; 1221 if (pi->flags & PORT_INIT_DONE) { 1222 t4_update_fl_bufsize(ifp); 1223 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1224 rc = update_mac_settings(ifp, XGMAC_MTU); 1225 } 1226 end_synchronized_op(sc, 0); 1227 break; 1228 1229 case SIOCSIFFLAGS: 1230 can_sleep = 0; 1231 redo_sifflags: 1232 rc = begin_synchronized_op(sc, pi, 1233 can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg"); 1234 if (rc) 1235 return (rc); 1236 1237 if (ifp->if_flags & IFF_UP) { 1238 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1239 flags = pi->if_flags; 1240 if ((ifp->if_flags ^ flags) & 1241 (IFF_PROMISC | IFF_ALLMULTI)) { 1242 if (can_sleep == 1) { 1243 end_synchronized_op(sc, 0); 1244 can_sleep = 0; 1245 goto redo_sifflags; 1246 } 1247 rc = update_mac_settings(ifp, 1248 XGMAC_PROMISC | XGMAC_ALLMULTI); 1249 } 1250 } else { 1251 if (can_sleep == 0) { 1252 end_synchronized_op(sc, LOCK_HELD); 1253 can_sleep = 1; 1254 goto redo_sifflags; 1255 } 1256 rc = cxgbe_init_synchronized(pi); 1257 } 1258 pi->if_flags = ifp->if_flags; 1259 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1260 if (can_sleep == 0) { 1261 end_synchronized_op(sc, LOCK_HELD); 1262 can_sleep = 1; 1263 goto redo_sifflags; 1264 } 1265 rc = cxgbe_uninit_synchronized(pi); 1266 } 1267 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD); 1268 break; 1269 1270 case SIOCADDMULTI: 1271 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1272 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi"); 1273 if (rc) 1274 return (rc); 1275 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1276 rc = update_mac_settings(ifp, XGMAC_MCADDRS); 1277 end_synchronized_op(sc, LOCK_HELD); 1278 break; 1279 1280 case SIOCSIFCAP: 1281 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap"); 1282 if (rc) 1283 return (rc); 1284 1285 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1286 if (mask & IFCAP_TXCSUM) { 1287 ifp->if_capenable ^= IFCAP_TXCSUM; 1288 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1289 1290 if (IFCAP_TSO4 & ifp->if_capenable && 1291 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1292 ifp->if_capenable &= ~IFCAP_TSO4; 1293 if_printf(ifp, 1294 "tso4 disabled due to -txcsum.\n"); 1295 } 1296 } 1297 if (mask & IFCAP_TXCSUM_IPV6) { 1298 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1299 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1300 1301 if (IFCAP_TSO6 & ifp->if_capenable && 1302 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1303 ifp->if_capenable &= ~IFCAP_TSO6; 1304 if_printf(ifp, 1305 "tso6 disabled due to -txcsum6.\n"); 1306 } 1307 } 1308 if (mask & IFCAP_RXCSUM) 1309 ifp->if_capenable ^= IFCAP_RXCSUM; 1310 if (mask & IFCAP_RXCSUM_IPV6) 1311 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1312 1313 /* 1314 * Note that we leave CSUM_TSO alone (it is always set). The 1315 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1316 * sending a TSO request our way, so it's sufficient to toggle 1317 * IFCAP_TSOx only. 1318 */ 1319 if (mask & IFCAP_TSO4) { 1320 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1321 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1322 if_printf(ifp, "enable txcsum first.\n"); 1323 rc = EAGAIN; 1324 goto fail; 1325 } 1326 ifp->if_capenable ^= IFCAP_TSO4; 1327 } 1328 if (mask & IFCAP_TSO6) { 1329 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1330 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1331 if_printf(ifp, "enable txcsum6 first.\n"); 1332 rc = EAGAIN; 1333 goto fail; 1334 } 1335 ifp->if_capenable ^= IFCAP_TSO6; 1336 } 1337 if (mask & IFCAP_LRO) { 1338 #if defined(INET) || defined(INET6) 1339 int i; 1340 struct sge_rxq *rxq; 1341 1342 ifp->if_capenable ^= IFCAP_LRO; 1343 for_each_rxq(pi, i, rxq) { 1344 if (ifp->if_capenable & IFCAP_LRO) 1345 rxq->iq.flags |= IQ_LRO_ENABLED; 1346 else 1347 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1348 } 1349 #endif 1350 } 1351 #ifdef TCP_OFFLOAD 1352 if (mask & IFCAP_TOE) { 1353 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1354 1355 rc = toe_capability(pi, enable); 1356 if (rc != 0) 1357 goto fail; 1358 1359 ifp->if_capenable ^= mask; 1360 } 1361 #endif 1362 if (mask & IFCAP_VLAN_HWTAGGING) { 1363 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1364 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1365 rc = update_mac_settings(ifp, XGMAC_VLANEX); 1366 } 1367 if (mask & IFCAP_VLAN_MTU) { 1368 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1369 1370 /* Need to find out how to disable auto-mtu-inflation */ 1371 } 1372 if (mask & IFCAP_VLAN_HWTSO) 1373 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1374 if (mask & IFCAP_VLAN_HWCSUM) 1375 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1376 1377 #ifdef VLAN_CAPABILITIES 1378 VLAN_CAPABILITIES(ifp); 1379 #endif 1380 fail: 1381 end_synchronized_op(sc, 0); 1382 break; 1383 1384 case SIOCSIFMEDIA: 1385 case SIOCGIFMEDIA: 1386 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1387 break; 1388 1389 case SIOCGI2C: { 1390 struct ifi2creq i2c; 1391 1392 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); 1393 if (rc != 0) 1394 break; 1395 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) { 1396 rc = EPERM; 1397 break; 1398 } 1399 if (i2c.len > sizeof(i2c.data)) { 1400 rc = EINVAL; 1401 break; 1402 } 1403 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4i2c"); 1404 if (rc) 1405 return (rc); 1406 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr, 1407 i2c.offset, i2c.len, &i2c.data[0]); 1408 end_synchronized_op(sc, 0); 1409 if (rc == 0) 1410 rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); 1411 break; 1412 } 1413 1414 default: 1415 rc = ether_ioctl(ifp, cmd, data); 1416 } 1417 1418 return (rc); 1419 } 1420 1421 static int 1422 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1423 { 1424 struct port_info *pi = ifp->if_softc; 1425 struct adapter *sc = pi->adapter; 1426 struct sge_txq *txq; 1427 void *items[1]; 1428 int rc; 1429 1430 M_ASSERTPKTHDR(m); 1431 MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */ 1432 1433 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1434 m_freem(m); 1435 return (ENETDOWN); 1436 } 1437 1438 rc = parse_pkt(&m); 1439 if (__predict_false(rc != 0)) { 1440 MPASS(m == NULL); /* was freed already */ 1441 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */ 1442 return (rc); 1443 } 1444 1445 /* Select a txq. */ 1446 txq = &sc->sge.txq[pi->first_txq]; 1447 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 1448 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) + 1449 pi->rsrv_noflowq); 1450 1451 items[0] = m; 1452 rc = mp_ring_enqueue(txq->r, items, 1, 4096); 1453 if (__predict_false(rc != 0)) 1454 m_freem(m); 1455 1456 return (rc); 1457 } 1458 1459 static void 1460 cxgbe_qflush(struct ifnet *ifp) 1461 { 1462 struct port_info *pi = ifp->if_softc; 1463 struct sge_txq *txq; 1464 int i; 1465 1466 /* queues do not exist if !PORT_INIT_DONE. */ 1467 if (pi->flags & PORT_INIT_DONE) { 1468 for_each_txq(pi, i, txq) { 1469 TXQ_LOCK(txq); 1470 txq->eq.flags &= ~EQ_ENABLED; 1471 TXQ_UNLOCK(txq); 1472 while (!mp_ring_is_idle(txq->r)) { 1473 mp_ring_check_drainage(txq->r, 0); 1474 pause("qflush", 1); 1475 } 1476 } 1477 } 1478 if_qflush(ifp); 1479 } 1480 1481 static uint64_t 1482 cxgbe_get_counter(struct ifnet *ifp, ift_counter c) 1483 { 1484 struct port_info *pi = ifp->if_softc; 1485 struct adapter *sc = pi->adapter; 1486 struct port_stats *s = &pi->stats; 1487 1488 cxgbe_refresh_stats(sc, pi); 1489 1490 switch (c) { 1491 case IFCOUNTER_IPACKETS: 1492 return (s->rx_frames - s->rx_pause); 1493 1494 case IFCOUNTER_IERRORS: 1495 return (s->rx_jabber + s->rx_runt + s->rx_too_long + 1496 s->rx_fcs_err + s->rx_len_err); 1497 1498 case IFCOUNTER_OPACKETS: 1499 return (s->tx_frames - s->tx_pause); 1500 1501 case IFCOUNTER_OERRORS: 1502 return (s->tx_error_frames); 1503 1504 case IFCOUNTER_IBYTES: 1505 return (s->rx_octets - s->rx_pause * 64); 1506 1507 case IFCOUNTER_OBYTES: 1508 return (s->tx_octets - s->tx_pause * 64); 1509 1510 case IFCOUNTER_IMCASTS: 1511 return (s->rx_mcast_frames - s->rx_pause); 1512 1513 case IFCOUNTER_OMCASTS: 1514 return (s->tx_mcast_frames - s->tx_pause); 1515 1516 case IFCOUNTER_IQDROPS: 1517 return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 1518 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 1519 s->rx_trunc3 + pi->tnl_cong_drops); 1520 1521 case IFCOUNTER_OQDROPS: { 1522 uint64_t drops; 1523 1524 drops = s->tx_drop; 1525 if (pi->flags & PORT_INIT_DONE) { 1526 int i; 1527 struct sge_txq *txq; 1528 1529 for_each_txq(pi, i, txq) 1530 drops += counter_u64_fetch(txq->r->drops); 1531 } 1532 1533 return (drops); 1534 1535 } 1536 1537 default: 1538 return (if_get_counter_default(ifp, c)); 1539 } 1540 } 1541 1542 static int 1543 cxgbe_media_change(struct ifnet *ifp) 1544 { 1545 struct port_info *pi = ifp->if_softc; 1546 1547 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1548 1549 return (EOPNOTSUPP); 1550 } 1551 1552 static void 1553 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1554 { 1555 struct port_info *pi = ifp->if_softc; 1556 struct ifmedia *media = NULL; 1557 struct ifmedia_entry *cur; 1558 int speed = pi->link_cfg.speed; 1559 1560 if (ifp == pi->ifp) 1561 media = &pi->media; 1562 #ifdef DEV_NETMAP 1563 else if (ifp == pi->nm_ifp) 1564 media = &pi->nm_media; 1565 #endif 1566 MPASS(media != NULL); 1567 1568 cur = media->ifm_cur; 1569 1570 ifmr->ifm_status = IFM_AVALID; 1571 if (!pi->link_cfg.link_ok) 1572 return; 1573 1574 ifmr->ifm_status |= IFM_ACTIVE; 1575 1576 /* active and current will differ iff current media is autoselect. */ 1577 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1578 return; 1579 1580 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1581 if (speed == SPEED_10000) 1582 ifmr->ifm_active |= IFM_10G_T; 1583 else if (speed == SPEED_1000) 1584 ifmr->ifm_active |= IFM_1000_T; 1585 else if (speed == SPEED_100) 1586 ifmr->ifm_active |= IFM_100_TX; 1587 else if (speed == SPEED_10) 1588 ifmr->ifm_active |= IFM_10_T; 1589 else 1590 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1591 speed)); 1592 } 1593 1594 void 1595 t4_fatal_err(struct adapter *sc) 1596 { 1597 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1598 t4_intr_disable(sc); 1599 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1600 device_get_nameunit(sc->dev)); 1601 } 1602 1603 static int 1604 map_bars_0_and_4(struct adapter *sc) 1605 { 1606 sc->regs_rid = PCIR_BAR(0); 1607 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1608 &sc->regs_rid, RF_ACTIVE); 1609 if (sc->regs_res == NULL) { 1610 device_printf(sc->dev, "cannot map registers.\n"); 1611 return (ENXIO); 1612 } 1613 sc->bt = rman_get_bustag(sc->regs_res); 1614 sc->bh = rman_get_bushandle(sc->regs_res); 1615 sc->mmio_len = rman_get_size(sc->regs_res); 1616 setbit(&sc->doorbells, DOORBELL_KDB); 1617 1618 sc->msix_rid = PCIR_BAR(4); 1619 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1620 &sc->msix_rid, RF_ACTIVE); 1621 if (sc->msix_res == NULL) { 1622 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1623 return (ENXIO); 1624 } 1625 1626 return (0); 1627 } 1628 1629 static int 1630 map_bar_2(struct adapter *sc) 1631 { 1632 1633 /* 1634 * T4: only iWARP driver uses the userspace doorbells. There is no need 1635 * to map it if RDMA is disabled. 1636 */ 1637 if (is_t4(sc) && sc->rdmacaps == 0) 1638 return (0); 1639 1640 sc->udbs_rid = PCIR_BAR(2); 1641 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1642 &sc->udbs_rid, RF_ACTIVE); 1643 if (sc->udbs_res == NULL) { 1644 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1645 return (ENXIO); 1646 } 1647 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1648 1649 if (is_t5(sc)) { 1650 setbit(&sc->doorbells, DOORBELL_UDB); 1651 #if defined(__i386__) || defined(__amd64__) 1652 if (t5_write_combine) { 1653 int rc; 1654 1655 /* 1656 * Enable write combining on BAR2. This is the 1657 * userspace doorbell BAR and is split into 128B 1658 * (UDBS_SEG_SIZE) doorbell regions, each associated 1659 * with an egress queue. The first 64B has the doorbell 1660 * and the second 64B can be used to submit a tx work 1661 * request with an implicit doorbell. 1662 */ 1663 1664 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1665 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1666 if (rc == 0) { 1667 clrbit(&sc->doorbells, DOORBELL_UDB); 1668 setbit(&sc->doorbells, DOORBELL_WCWR); 1669 setbit(&sc->doorbells, DOORBELL_UDBWC); 1670 } else { 1671 device_printf(sc->dev, 1672 "couldn't enable write combining: %d\n", 1673 rc); 1674 } 1675 1676 t4_write_reg(sc, A_SGE_STAT_CFG, 1677 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1678 } 1679 #endif 1680 } 1681 1682 return (0); 1683 } 1684 1685 static const struct memwin t4_memwin[] = { 1686 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1687 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1688 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1689 }; 1690 1691 static const struct memwin t5_memwin[] = { 1692 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1693 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1694 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1695 }; 1696 1697 static void 1698 setup_memwin(struct adapter *sc) 1699 { 1700 const struct memwin *mw; 1701 int i, n; 1702 uint32_t bar0; 1703 1704 if (is_t4(sc)) { 1705 /* 1706 * Read low 32b of bar0 indirectly via the hardware backdoor 1707 * mechanism. Works from within PCI passthrough environments 1708 * too, where rman_get_start() can return a different value. We 1709 * need to program the T4 memory window decoders with the actual 1710 * addresses that will be coming across the PCIe link. 1711 */ 1712 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1713 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1714 1715 mw = &t4_memwin[0]; 1716 n = nitems(t4_memwin); 1717 } else { 1718 /* T5 uses the relative offset inside the PCIe BAR */ 1719 bar0 = 0; 1720 1721 mw = &t5_memwin[0]; 1722 n = nitems(t5_memwin); 1723 } 1724 1725 for (i = 0; i < n; i++, mw++) { 1726 t4_write_reg(sc, 1727 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 1728 (mw->base + bar0) | V_BIR(0) | 1729 V_WINDOW(ilog2(mw->aperture) - 10)); 1730 } 1731 1732 /* flush */ 1733 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1734 } 1735 1736 /* 1737 * Verify that the memory range specified by the addr/len pair is valid and lies 1738 * entirely within a single region (EDCx or MCx). 1739 */ 1740 static int 1741 validate_mem_range(struct adapter *sc, uint32_t addr, int len) 1742 { 1743 uint32_t em, addr_len, maddr, mlen; 1744 1745 /* Memory can only be accessed in naturally aligned 4 byte units */ 1746 if (addr & 3 || len & 3 || len == 0) 1747 return (EINVAL); 1748 1749 /* Enabled memories */ 1750 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1751 if (em & F_EDRAM0_ENABLE) { 1752 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1753 maddr = G_EDRAM0_BASE(addr_len) << 20; 1754 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1755 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1756 addr + len <= maddr + mlen) 1757 return (0); 1758 } 1759 if (em & F_EDRAM1_ENABLE) { 1760 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1761 maddr = G_EDRAM1_BASE(addr_len) << 20; 1762 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1763 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1764 addr + len <= maddr + mlen) 1765 return (0); 1766 } 1767 if (em & F_EXT_MEM_ENABLE) { 1768 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1769 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1770 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1771 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1772 addr + len <= maddr + mlen) 1773 return (0); 1774 } 1775 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 1776 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1777 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1778 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1779 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1780 addr + len <= maddr + mlen) 1781 return (0); 1782 } 1783 1784 return (EFAULT); 1785 } 1786 1787 static int 1788 fwmtype_to_hwmtype(int mtype) 1789 { 1790 1791 switch (mtype) { 1792 case FW_MEMTYPE_EDC0: 1793 return (MEM_EDC0); 1794 case FW_MEMTYPE_EDC1: 1795 return (MEM_EDC1); 1796 case FW_MEMTYPE_EXTMEM: 1797 return (MEM_MC0); 1798 case FW_MEMTYPE_EXTMEM1: 1799 return (MEM_MC1); 1800 default: 1801 panic("%s: cannot translate fw mtype %d.", __func__, mtype); 1802 } 1803 } 1804 1805 /* 1806 * Verify that the memory range specified by the memtype/offset/len pair is 1807 * valid and lies entirely within the memtype specified. The global address of 1808 * the start of the range is returned in addr. 1809 */ 1810 static int 1811 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 1812 uint32_t *addr) 1813 { 1814 uint32_t em, addr_len, maddr, mlen; 1815 1816 /* Memory can only be accessed in naturally aligned 4 byte units */ 1817 if (off & 3 || len & 3 || len == 0) 1818 return (EINVAL); 1819 1820 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1821 switch (fwmtype_to_hwmtype(mtype)) { 1822 case MEM_EDC0: 1823 if (!(em & F_EDRAM0_ENABLE)) 1824 return (EINVAL); 1825 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1826 maddr = G_EDRAM0_BASE(addr_len) << 20; 1827 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1828 break; 1829 case MEM_EDC1: 1830 if (!(em & F_EDRAM1_ENABLE)) 1831 return (EINVAL); 1832 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1833 maddr = G_EDRAM1_BASE(addr_len) << 20; 1834 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1835 break; 1836 case MEM_MC: 1837 if (!(em & F_EXT_MEM_ENABLE)) 1838 return (EINVAL); 1839 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1840 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1841 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1842 break; 1843 case MEM_MC1: 1844 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 1845 return (EINVAL); 1846 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1847 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1848 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1849 break; 1850 default: 1851 return (EINVAL); 1852 } 1853 1854 if (mlen > 0 && off < mlen && off + len <= mlen) { 1855 *addr = maddr + off; /* global address */ 1856 return (0); 1857 } 1858 1859 return (EFAULT); 1860 } 1861 1862 static void 1863 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 1864 { 1865 const struct memwin *mw; 1866 1867 if (is_t4(sc)) { 1868 KASSERT(win >= 0 && win < nitems(t4_memwin), 1869 ("%s: incorrect memwin# (%d)", __func__, win)); 1870 mw = &t4_memwin[win]; 1871 } else { 1872 KASSERT(win >= 0 && win < nitems(t5_memwin), 1873 ("%s: incorrect memwin# (%d)", __func__, win)); 1874 mw = &t5_memwin[win]; 1875 } 1876 1877 if (base != NULL) 1878 *base = mw->base; 1879 if (aperture != NULL) 1880 *aperture = mw->aperture; 1881 } 1882 1883 /* 1884 * Positions the memory window such that it can be used to access the specified 1885 * address in the chip's address space. The return value is the offset of addr 1886 * from the start of the window. 1887 */ 1888 static uint32_t 1889 position_memwin(struct adapter *sc, int n, uint32_t addr) 1890 { 1891 uint32_t start, pf; 1892 uint32_t reg; 1893 1894 KASSERT(n >= 0 && n <= 3, 1895 ("%s: invalid window %d.", __func__, n)); 1896 KASSERT((addr & 3) == 0, 1897 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 1898 1899 if (is_t4(sc)) { 1900 pf = 0; 1901 start = addr & ~0xf; /* start must be 16B aligned */ 1902 } else { 1903 pf = V_PFNUM(sc->pf); 1904 start = addr & ~0x7f; /* start must be 128B aligned */ 1905 } 1906 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 1907 1908 t4_write_reg(sc, reg, start | pf); 1909 t4_read_reg(sc, reg); 1910 1911 return (addr - start); 1912 } 1913 1914 static int 1915 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1916 struct intrs_and_queues *iaq) 1917 { 1918 int rc, itype, navail, nrxq10g, nrxq1g, n; 1919 int nofldrxq10g = 0, nofldrxq1g = 0; 1920 int nnmrxq10g = 0, nnmrxq1g = 0; 1921 1922 bzero(iaq, sizeof(*iaq)); 1923 1924 iaq->ntxq10g = t4_ntxq10g; 1925 iaq->ntxq1g = t4_ntxq1g; 1926 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1927 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1928 iaq->rsrv_noflowq = t4_rsrv_noflowq; 1929 #ifdef TCP_OFFLOAD 1930 if (is_offload(sc)) { 1931 iaq->nofldtxq10g = t4_nofldtxq10g; 1932 iaq->nofldtxq1g = t4_nofldtxq1g; 1933 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1934 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1935 } 1936 #endif 1937 #ifdef DEV_NETMAP 1938 iaq->nnmtxq10g = t4_nnmtxq10g; 1939 iaq->nnmtxq1g = t4_nnmtxq1g; 1940 iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g; 1941 iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g; 1942 #endif 1943 1944 for (itype = INTR_MSIX; itype; itype >>= 1) { 1945 1946 if ((itype & t4_intr_types) == 0) 1947 continue; /* not allowed */ 1948 1949 if (itype == INTR_MSIX) 1950 navail = pci_msix_count(sc->dev); 1951 else if (itype == INTR_MSI) 1952 navail = pci_msi_count(sc->dev); 1953 else 1954 navail = 1; 1955 restart: 1956 if (navail == 0) 1957 continue; 1958 1959 iaq->intr_type = itype; 1960 iaq->intr_flags_10g = 0; 1961 iaq->intr_flags_1g = 0; 1962 1963 /* 1964 * Best option: an interrupt vector for errors, one for the 1965 * firmware event queue, and one for every rxq (NIC, TOE, and 1966 * netmap). 1967 */ 1968 iaq->nirq = T4_EXTRA_INTR; 1969 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g); 1970 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g); 1971 if (iaq->nirq <= navail && 1972 (itype != INTR_MSI || powerof2(iaq->nirq))) { 1973 iaq->intr_flags_10g = INTR_ALL; 1974 iaq->intr_flags_1g = INTR_ALL; 1975 goto allocate; 1976 } 1977 1978 /* 1979 * Second best option: a vector for errors, one for the firmware 1980 * event queue, and vectors for either all the NIC rx queues or 1981 * all the TOE rx queues. The queues that don't get vectors 1982 * will forward their interrupts to those that do. 1983 * 1984 * Note: netmap rx queues cannot be created early and so they 1985 * can't be setup to receive forwarded interrupts for others. 1986 */ 1987 iaq->nirq = T4_EXTRA_INTR; 1988 if (nrxq10g >= nofldrxq10g) { 1989 iaq->intr_flags_10g = INTR_RXQ; 1990 iaq->nirq += n10g * nrxq10g; 1991 #ifdef DEV_NETMAP 1992 iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g); 1993 #endif 1994 } else { 1995 iaq->intr_flags_10g = INTR_OFLD_RXQ; 1996 iaq->nirq += n10g * nofldrxq10g; 1997 #ifdef DEV_NETMAP 1998 iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g); 1999 #endif 2000 } 2001 if (nrxq1g >= nofldrxq1g) { 2002 iaq->intr_flags_1g = INTR_RXQ; 2003 iaq->nirq += n1g * nrxq1g; 2004 #ifdef DEV_NETMAP 2005 iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g); 2006 #endif 2007 } else { 2008 iaq->intr_flags_1g = INTR_OFLD_RXQ; 2009 iaq->nirq += n1g * nofldrxq1g; 2010 #ifdef DEV_NETMAP 2011 iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g); 2012 #endif 2013 } 2014 if (iaq->nirq <= navail && 2015 (itype != INTR_MSI || powerof2(iaq->nirq))) 2016 goto allocate; 2017 2018 /* 2019 * Next best option: an interrupt vector for errors, one for the 2020 * firmware event queue, and at least one per port. At this 2021 * point we know we'll have to downsize nrxq and/or nofldrxq 2022 * and/or nnmrxq to fit what's available to us. 2023 */ 2024 iaq->nirq = T4_EXTRA_INTR; 2025 iaq->nirq += n10g + n1g; 2026 if (iaq->nirq <= navail) { 2027 int leftover = navail - iaq->nirq; 2028 2029 if (n10g > 0) { 2030 int target = max(nrxq10g, nofldrxq10g); 2031 2032 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ? 2033 INTR_RXQ : INTR_OFLD_RXQ; 2034 2035 n = 1; 2036 while (n < target && leftover >= n10g) { 2037 leftover -= n10g; 2038 iaq->nirq += n10g; 2039 n++; 2040 } 2041 iaq->nrxq10g = min(n, nrxq10g); 2042 #ifdef TCP_OFFLOAD 2043 iaq->nofldrxq10g = min(n, nofldrxq10g); 2044 #endif 2045 #ifdef DEV_NETMAP 2046 iaq->nnmrxq10g = min(n, nnmrxq10g); 2047 #endif 2048 } 2049 2050 if (n1g > 0) { 2051 int target = max(nrxq1g, nofldrxq1g); 2052 2053 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ? 2054 INTR_RXQ : INTR_OFLD_RXQ; 2055 2056 n = 1; 2057 while (n < target && leftover >= n1g) { 2058 leftover -= n1g; 2059 iaq->nirq += n1g; 2060 n++; 2061 } 2062 iaq->nrxq1g = min(n, nrxq1g); 2063 #ifdef TCP_OFFLOAD 2064 iaq->nofldrxq1g = min(n, nofldrxq1g); 2065 #endif 2066 #ifdef DEV_NETMAP 2067 iaq->nnmrxq1g = min(n, nnmrxq1g); 2068 #endif 2069 } 2070 2071 if (itype != INTR_MSI || powerof2(iaq->nirq)) 2072 goto allocate; 2073 } 2074 2075 /* 2076 * Least desirable option: one interrupt vector for everything. 2077 */ 2078 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 2079 iaq->intr_flags_10g = iaq->intr_flags_1g = 0; 2080 #ifdef TCP_OFFLOAD 2081 if (is_offload(sc)) 2082 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 2083 #endif 2084 #ifdef DEV_NETMAP 2085 iaq->nnmrxq10g = iaq->nnmrxq1g = 1; 2086 #endif 2087 2088 allocate: 2089 navail = iaq->nirq; 2090 rc = 0; 2091 if (itype == INTR_MSIX) 2092 rc = pci_alloc_msix(sc->dev, &navail); 2093 else if (itype == INTR_MSI) 2094 rc = pci_alloc_msi(sc->dev, &navail); 2095 2096 if (rc == 0) { 2097 if (navail == iaq->nirq) 2098 return (0); 2099 2100 /* 2101 * Didn't get the number requested. Use whatever number 2102 * the kernel is willing to allocate (it's in navail). 2103 */ 2104 device_printf(sc->dev, "fewer vectors than requested, " 2105 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 2106 itype, iaq->nirq, navail); 2107 pci_release_msi(sc->dev); 2108 goto restart; 2109 } 2110 2111 device_printf(sc->dev, 2112 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 2113 itype, rc, iaq->nirq, navail); 2114 } 2115 2116 device_printf(sc->dev, 2117 "failed to find a usable interrupt type. " 2118 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 2119 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 2120 2121 return (ENXIO); 2122 } 2123 2124 #define FW_VERSION(chip) ( \ 2125 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ 2126 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ 2127 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ 2128 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) 2129 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) 2130 2131 struct fw_info { 2132 uint8_t chip; 2133 char *kld_name; 2134 char *fw_mod_name; 2135 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 2136 } fw_info[] = { 2137 { 2138 .chip = CHELSIO_T4, 2139 .kld_name = "t4fw_cfg", 2140 .fw_mod_name = "t4fw", 2141 .fw_hdr = { 2142 .chip = FW_HDR_CHIP_T4, 2143 .fw_ver = htobe32_const(FW_VERSION(T4)), 2144 .intfver_nic = FW_INTFVER(T4, NIC), 2145 .intfver_vnic = FW_INTFVER(T4, VNIC), 2146 .intfver_ofld = FW_INTFVER(T4, OFLD), 2147 .intfver_ri = FW_INTFVER(T4, RI), 2148 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 2149 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 2150 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 2151 .intfver_fcoe = FW_INTFVER(T4, FCOE), 2152 }, 2153 }, { 2154 .chip = CHELSIO_T5, 2155 .kld_name = "t5fw_cfg", 2156 .fw_mod_name = "t5fw", 2157 .fw_hdr = { 2158 .chip = FW_HDR_CHIP_T5, 2159 .fw_ver = htobe32_const(FW_VERSION(T5)), 2160 .intfver_nic = FW_INTFVER(T5, NIC), 2161 .intfver_vnic = FW_INTFVER(T5, VNIC), 2162 .intfver_ofld = FW_INTFVER(T5, OFLD), 2163 .intfver_ri = FW_INTFVER(T5, RI), 2164 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 2165 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 2166 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 2167 .intfver_fcoe = FW_INTFVER(T5, FCOE), 2168 }, 2169 } 2170 }; 2171 2172 static struct fw_info * 2173 find_fw_info(int chip) 2174 { 2175 int i; 2176 2177 for (i = 0; i < nitems(fw_info); i++) { 2178 if (fw_info[i].chip == chip) 2179 return (&fw_info[i]); 2180 } 2181 return (NULL); 2182 } 2183 2184 /* 2185 * Is the given firmware API compatible with the one the driver was compiled 2186 * with? 2187 */ 2188 static int 2189 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 2190 { 2191 2192 /* short circuit if it's the exact same firmware version */ 2193 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 2194 return (1); 2195 2196 /* 2197 * XXX: Is this too conservative? Perhaps I should limit this to the 2198 * features that are supported in the driver. 2199 */ 2200 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 2201 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 2202 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 2203 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 2204 return (1); 2205 #undef SAME_INTF 2206 2207 return (0); 2208 } 2209 2210 /* 2211 * The firmware in the KLD is usable, but should it be installed? This routine 2212 * explains itself in detail if it indicates the KLD firmware should be 2213 * installed. 2214 */ 2215 static int 2216 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 2217 { 2218 const char *reason; 2219 2220 if (!card_fw_usable) { 2221 reason = "incompatible or unusable"; 2222 goto install; 2223 } 2224 2225 if (k > c) { 2226 reason = "older than the version bundled with this driver"; 2227 goto install; 2228 } 2229 2230 if (t4_fw_install == 2 && k != c) { 2231 reason = "different than the version bundled with this driver"; 2232 goto install; 2233 } 2234 2235 return (0); 2236 2237 install: 2238 if (t4_fw_install == 0) { 2239 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2240 "but the driver is prohibited from installing a different " 2241 "firmware on the card.\n", 2242 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2243 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 2244 2245 return (0); 2246 } 2247 2248 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2249 "installing firmware %u.%u.%u.%u on card.\n", 2250 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2251 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 2252 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2253 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2254 2255 return (1); 2256 } 2257 /* 2258 * Establish contact with the firmware and determine if we are the master driver 2259 * or not, and whether we are responsible for chip initialization. 2260 */ 2261 static int 2262 prep_firmware(struct adapter *sc) 2263 { 2264 const struct firmware *fw = NULL, *default_cfg; 2265 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 2266 enum dev_state state; 2267 struct fw_info *fw_info; 2268 struct fw_hdr *card_fw; /* fw on the card */ 2269 const struct fw_hdr *kld_fw; /* fw in the KLD */ 2270 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 2271 against */ 2272 2273 /* Contact firmware. */ 2274 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 2275 if (rc < 0 || state == DEV_STATE_ERR) { 2276 rc = -rc; 2277 device_printf(sc->dev, 2278 "failed to connect to the firmware: %d, %d.\n", rc, state); 2279 return (rc); 2280 } 2281 pf = rc; 2282 if (pf == sc->mbox) 2283 sc->flags |= MASTER_PF; 2284 else if (state == DEV_STATE_UNINIT) { 2285 /* 2286 * We didn't get to be the master so we definitely won't be 2287 * configuring the chip. It's a bug if someone else hasn't 2288 * configured it already. 2289 */ 2290 device_printf(sc->dev, "couldn't be master(%d), " 2291 "device not already initialized either(%d).\n", rc, state); 2292 return (EDOOFUS); 2293 } 2294 2295 /* This is the firmware whose headers the driver was compiled against */ 2296 fw_info = find_fw_info(chip_id(sc)); 2297 if (fw_info == NULL) { 2298 device_printf(sc->dev, 2299 "unable to look up firmware information for chip %d.\n", 2300 chip_id(sc)); 2301 return (EINVAL); 2302 } 2303 drv_fw = &fw_info->fw_hdr; 2304 2305 /* 2306 * The firmware KLD contains many modules. The KLD name is also the 2307 * name of the module that contains the default config file. 2308 */ 2309 default_cfg = firmware_get(fw_info->kld_name); 2310 2311 /* Read the header of the firmware on the card */ 2312 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 2313 rc = -t4_read_flash(sc, FLASH_FW_START, 2314 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 2315 if (rc == 0) 2316 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 2317 else { 2318 device_printf(sc->dev, 2319 "Unable to read card's firmware header: %d\n", rc); 2320 card_fw_usable = 0; 2321 } 2322 2323 /* This is the firmware in the KLD */ 2324 fw = firmware_get(fw_info->fw_mod_name); 2325 if (fw != NULL) { 2326 kld_fw = (const void *)fw->data; 2327 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 2328 } else { 2329 kld_fw = NULL; 2330 kld_fw_usable = 0; 2331 } 2332 2333 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2334 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) { 2335 /* 2336 * Common case: the firmware on the card is an exact match and 2337 * the KLD is an exact match too, or the KLD is 2338 * absent/incompatible. Note that t4_fw_install = 2 is ignored 2339 * here -- use cxgbetool loadfw if you want to reinstall the 2340 * same firmware as the one on the card. 2341 */ 2342 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2343 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2344 be32toh(card_fw->fw_ver))) { 2345 2346 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 2347 if (rc != 0) { 2348 device_printf(sc->dev, 2349 "failed to install firmware: %d\n", rc); 2350 goto done; 2351 } 2352 2353 /* Installed successfully, update the cached header too. */ 2354 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2355 card_fw_usable = 1; 2356 need_fw_reset = 0; /* already reset as part of load_fw */ 2357 } 2358 2359 if (!card_fw_usable) { 2360 uint32_t d, c, k; 2361 2362 d = ntohl(drv_fw->fw_ver); 2363 c = ntohl(card_fw->fw_ver); 2364 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2365 2366 device_printf(sc->dev, "Cannot find a usable firmware: " 2367 "fw_install %d, chip state %d, " 2368 "driver compiled with %d.%d.%d.%d, " 2369 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2370 t4_fw_install, state, 2371 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2372 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2373 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2374 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2375 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2376 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2377 rc = EINVAL; 2378 goto done; 2379 } 2380 2381 /* We're using whatever's on the card and it's known to be good. */ 2382 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2383 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2384 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2385 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2386 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2387 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2388 t4_get_tp_version(sc, &sc->params.tp_vers); 2389 2390 /* Reset device */ 2391 if (need_fw_reset && 2392 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2393 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2394 if (rc != ETIMEDOUT && rc != EIO) 2395 t4_fw_bye(sc, sc->mbox); 2396 goto done; 2397 } 2398 sc->flags |= FW_OK; 2399 2400 rc = get_params__pre_init(sc); 2401 if (rc != 0) 2402 goto done; /* error message displayed already */ 2403 2404 /* Partition adapter resources as specified in the config file. */ 2405 if (state == DEV_STATE_UNINIT) { 2406 2407 KASSERT(sc->flags & MASTER_PF, 2408 ("%s: trying to change chip settings when not master.", 2409 __func__)); 2410 2411 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2412 if (rc != 0) 2413 goto done; /* error message displayed already */ 2414 2415 t4_tweak_chip_settings(sc); 2416 2417 /* get basic stuff going */ 2418 rc = -t4_fw_initialize(sc, sc->mbox); 2419 if (rc != 0) { 2420 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2421 goto done; 2422 } 2423 } else { 2424 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2425 sc->cfcsum = 0; 2426 } 2427 2428 done: 2429 free(card_fw, M_CXGBE); 2430 if (fw != NULL) 2431 firmware_put(fw, FIRMWARE_UNLOAD); 2432 if (default_cfg != NULL) 2433 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2434 2435 return (rc); 2436 } 2437 2438 #define FW_PARAM_DEV(param) \ 2439 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2440 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2441 #define FW_PARAM_PFVF(param) \ 2442 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2443 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2444 2445 /* 2446 * Partition chip resources for use between various PFs, VFs, etc. 2447 */ 2448 static int 2449 partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2450 const char *name_prefix) 2451 { 2452 const struct firmware *cfg = NULL; 2453 int rc = 0; 2454 struct fw_caps_config_cmd caps; 2455 uint32_t mtype, moff, finicsum, cfcsum; 2456 2457 /* 2458 * Figure out what configuration file to use. Pick the default config 2459 * file for the card if the user hasn't specified one explicitly. 2460 */ 2461 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2462 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2463 /* Card specific overrides go here. */ 2464 if (pci_get_device(sc->dev) == 0x440a) 2465 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2466 if (is_fpga(sc)) 2467 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2468 } 2469 2470 /* 2471 * We need to load another module if the profile is anything except 2472 * "default" or "flash". 2473 */ 2474 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2475 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2476 char s[32]; 2477 2478 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2479 cfg = firmware_get(s); 2480 if (cfg == NULL) { 2481 if (default_cfg != NULL) { 2482 device_printf(sc->dev, 2483 "unable to load module \"%s\" for " 2484 "configuration profile \"%s\", will use " 2485 "the default config file instead.\n", 2486 s, sc->cfg_file); 2487 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2488 "%s", DEFAULT_CF); 2489 } else { 2490 device_printf(sc->dev, 2491 "unable to load module \"%s\" for " 2492 "configuration profile \"%s\", will use " 2493 "the config file on the card's flash " 2494 "instead.\n", s, sc->cfg_file); 2495 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2496 "%s", FLASH_CF); 2497 } 2498 } 2499 } 2500 2501 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2502 default_cfg == NULL) { 2503 device_printf(sc->dev, 2504 "default config file not available, will use the config " 2505 "file on the card's flash instead.\n"); 2506 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2507 } 2508 2509 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2510 u_int cflen, i, n; 2511 const uint32_t *cfdata; 2512 uint32_t param, val, addr, off, mw_base, mw_aperture; 2513 2514 KASSERT(cfg != NULL || default_cfg != NULL, 2515 ("%s: no config to upload", __func__)); 2516 2517 /* 2518 * Ask the firmware where it wants us to upload the config file. 2519 */ 2520 param = FW_PARAM_DEV(CF); 2521 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2522 if (rc != 0) { 2523 /* No support for config file? Shouldn't happen. */ 2524 device_printf(sc->dev, 2525 "failed to query config file location: %d.\n", rc); 2526 goto done; 2527 } 2528 mtype = G_FW_PARAMS_PARAM_Y(val); 2529 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2530 2531 /* 2532 * XXX: sheer laziness. We deliberately added 4 bytes of 2533 * useless stuffing/comments at the end of the config file so 2534 * it's ok to simply throw away the last remaining bytes when 2535 * the config file is not an exact multiple of 4. This also 2536 * helps with the validate_mt_off_len check. 2537 */ 2538 if (cfg != NULL) { 2539 cflen = cfg->datasize & ~3; 2540 cfdata = cfg->data; 2541 } else { 2542 cflen = default_cfg->datasize & ~3; 2543 cfdata = default_cfg->data; 2544 } 2545 2546 if (cflen > FLASH_CFG_MAX_SIZE) { 2547 device_printf(sc->dev, 2548 "config file too long (%d, max allowed is %d). " 2549 "Will try to use the config on the card, if any.\n", 2550 cflen, FLASH_CFG_MAX_SIZE); 2551 goto use_config_on_flash; 2552 } 2553 2554 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2555 if (rc != 0) { 2556 device_printf(sc->dev, 2557 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2558 "Will try to use the config on the card, if any.\n", 2559 __func__, mtype, moff, cflen, rc); 2560 goto use_config_on_flash; 2561 } 2562 2563 memwin_info(sc, 2, &mw_base, &mw_aperture); 2564 while (cflen) { 2565 off = position_memwin(sc, 2, addr); 2566 n = min(cflen, mw_aperture - off); 2567 for (i = 0; i < n; i += 4) 2568 t4_write_reg(sc, mw_base + off + i, *cfdata++); 2569 cflen -= n; 2570 addr += n; 2571 } 2572 } else { 2573 use_config_on_flash: 2574 mtype = FW_MEMTYPE_FLASH; 2575 moff = t4_flash_cfg_addr(sc); 2576 } 2577 2578 bzero(&caps, sizeof(caps)); 2579 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2580 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2581 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2582 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2583 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2584 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2585 if (rc != 0) { 2586 device_printf(sc->dev, 2587 "failed to pre-process config file: %d " 2588 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2589 goto done; 2590 } 2591 2592 finicsum = be32toh(caps.finicsum); 2593 cfcsum = be32toh(caps.cfcsum); 2594 if (finicsum != cfcsum) { 2595 device_printf(sc->dev, 2596 "WARNING: config file checksum mismatch: %08x %08x\n", 2597 finicsum, cfcsum); 2598 } 2599 sc->cfcsum = cfcsum; 2600 2601 #define LIMIT_CAPS(x) do { \ 2602 caps.x &= htobe16(t4_##x##_allowed); \ 2603 } while (0) 2604 2605 /* 2606 * Let the firmware know what features will (not) be used so it can tune 2607 * things accordingly. 2608 */ 2609 LIMIT_CAPS(linkcaps); 2610 LIMIT_CAPS(niccaps); 2611 LIMIT_CAPS(toecaps); 2612 LIMIT_CAPS(rdmacaps); 2613 LIMIT_CAPS(iscsicaps); 2614 LIMIT_CAPS(fcoecaps); 2615 #undef LIMIT_CAPS 2616 2617 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2618 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2619 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2620 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2621 if (rc != 0) { 2622 device_printf(sc->dev, 2623 "failed to process config file: %d.\n", rc); 2624 } 2625 done: 2626 if (cfg != NULL) 2627 firmware_put(cfg, FIRMWARE_UNLOAD); 2628 return (rc); 2629 } 2630 2631 /* 2632 * Retrieve parameters that are needed (or nice to have) very early. 2633 */ 2634 static int 2635 get_params__pre_init(struct adapter *sc) 2636 { 2637 int rc; 2638 uint32_t param[2], val[2]; 2639 struct fw_devlog_cmd cmd; 2640 struct devlog_params *dlog = &sc->params.devlog; 2641 2642 param[0] = FW_PARAM_DEV(PORTVEC); 2643 param[1] = FW_PARAM_DEV(CCLK); 2644 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2645 if (rc != 0) { 2646 device_printf(sc->dev, 2647 "failed to query parameters (pre_init): %d.\n", rc); 2648 return (rc); 2649 } 2650 2651 sc->params.portvec = val[0]; 2652 sc->params.nports = bitcount32(val[0]); 2653 sc->params.vpd.cclk = val[1]; 2654 2655 /* Read device log parameters. */ 2656 bzero(&cmd, sizeof(cmd)); 2657 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2658 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2659 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2660 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2661 if (rc != 0) { 2662 device_printf(sc->dev, 2663 "failed to get devlog parameters: %d.\n", rc); 2664 bzero(dlog, sizeof (*dlog)); 2665 rc = 0; /* devlog isn't critical for device operation */ 2666 } else { 2667 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2668 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2669 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2670 dlog->size = be32toh(cmd.memsize_devlog); 2671 } 2672 2673 return (rc); 2674 } 2675 2676 /* 2677 * Retrieve various parameters that are of interest to the driver. The device 2678 * has been initialized by the firmware at this point. 2679 */ 2680 static int 2681 get_params__post_init(struct adapter *sc) 2682 { 2683 int rc; 2684 uint32_t param[7], val[7]; 2685 struct fw_caps_config_cmd caps; 2686 2687 param[0] = FW_PARAM_PFVF(IQFLINT_START); 2688 param[1] = FW_PARAM_PFVF(EQ_START); 2689 param[2] = FW_PARAM_PFVF(FILTER_START); 2690 param[3] = FW_PARAM_PFVF(FILTER_END); 2691 param[4] = FW_PARAM_PFVF(L2T_START); 2692 param[5] = FW_PARAM_PFVF(L2T_END); 2693 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2694 if (rc != 0) { 2695 device_printf(sc->dev, 2696 "failed to query parameters (post_init): %d.\n", rc); 2697 return (rc); 2698 } 2699 2700 sc->sge.iq_start = val[0]; 2701 sc->sge.eq_start = val[1]; 2702 sc->tids.ftid_base = val[2]; 2703 sc->tids.nftids = val[3] - val[2] + 1; 2704 sc->params.ftid_min = val[2]; 2705 sc->params.ftid_max = val[3]; 2706 sc->vres.l2t.start = val[4]; 2707 sc->vres.l2t.size = val[5] - val[4] + 1; 2708 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2709 ("%s: L2 table size (%u) larger than expected (%u)", 2710 __func__, sc->vres.l2t.size, L2T_SIZE)); 2711 2712 /* get capabilites */ 2713 bzero(&caps, sizeof(caps)); 2714 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2715 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2716 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2717 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2718 if (rc != 0) { 2719 device_printf(sc->dev, 2720 "failed to get card capabilities: %d.\n", rc); 2721 return (rc); 2722 } 2723 2724 #define READ_CAPS(x) do { \ 2725 sc->x = htobe16(caps.x); \ 2726 } while (0) 2727 READ_CAPS(linkcaps); 2728 READ_CAPS(niccaps); 2729 READ_CAPS(toecaps); 2730 READ_CAPS(rdmacaps); 2731 READ_CAPS(iscsicaps); 2732 READ_CAPS(fcoecaps); 2733 2734 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { 2735 param[0] = FW_PARAM_PFVF(ETHOFLD_START); 2736 param[1] = FW_PARAM_PFVF(ETHOFLD_END); 2737 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2738 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); 2739 if (rc != 0) { 2740 device_printf(sc->dev, 2741 "failed to query NIC parameters: %d.\n", rc); 2742 return (rc); 2743 } 2744 sc->tids.etid_base = val[0]; 2745 sc->params.etid_min = val[0]; 2746 sc->tids.netids = val[1] - val[0] + 1; 2747 sc->params.netids = sc->tids.netids; 2748 sc->params.eo_wr_cred = val[2]; 2749 sc->params.ethoffload = 1; 2750 } 2751 2752 if (sc->toecaps) { 2753 /* query offload-related parameters */ 2754 param[0] = FW_PARAM_DEV(NTID); 2755 param[1] = FW_PARAM_PFVF(SERVER_START); 2756 param[2] = FW_PARAM_PFVF(SERVER_END); 2757 param[3] = FW_PARAM_PFVF(TDDP_START); 2758 param[4] = FW_PARAM_PFVF(TDDP_END); 2759 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2760 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2761 if (rc != 0) { 2762 device_printf(sc->dev, 2763 "failed to query TOE parameters: %d.\n", rc); 2764 return (rc); 2765 } 2766 sc->tids.ntids = val[0]; 2767 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 2768 sc->tids.stid_base = val[1]; 2769 sc->tids.nstids = val[2] - val[1] + 1; 2770 sc->vres.ddp.start = val[3]; 2771 sc->vres.ddp.size = val[4] - val[3] + 1; 2772 sc->params.ofldq_wr_cred = val[5]; 2773 sc->params.offload = 1; 2774 } 2775 if (sc->rdmacaps) { 2776 param[0] = FW_PARAM_PFVF(STAG_START); 2777 param[1] = FW_PARAM_PFVF(STAG_END); 2778 param[2] = FW_PARAM_PFVF(RQ_START); 2779 param[3] = FW_PARAM_PFVF(RQ_END); 2780 param[4] = FW_PARAM_PFVF(PBL_START); 2781 param[5] = FW_PARAM_PFVF(PBL_END); 2782 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2783 if (rc != 0) { 2784 device_printf(sc->dev, 2785 "failed to query RDMA parameters(1): %d.\n", rc); 2786 return (rc); 2787 } 2788 sc->vres.stag.start = val[0]; 2789 sc->vres.stag.size = val[1] - val[0] + 1; 2790 sc->vres.rq.start = val[2]; 2791 sc->vres.rq.size = val[3] - val[2] + 1; 2792 sc->vres.pbl.start = val[4]; 2793 sc->vres.pbl.size = val[5] - val[4] + 1; 2794 2795 param[0] = FW_PARAM_PFVF(SQRQ_START); 2796 param[1] = FW_PARAM_PFVF(SQRQ_END); 2797 param[2] = FW_PARAM_PFVF(CQ_START); 2798 param[3] = FW_PARAM_PFVF(CQ_END); 2799 param[4] = FW_PARAM_PFVF(OCQ_START); 2800 param[5] = FW_PARAM_PFVF(OCQ_END); 2801 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2802 if (rc != 0) { 2803 device_printf(sc->dev, 2804 "failed to query RDMA parameters(2): %d.\n", rc); 2805 return (rc); 2806 } 2807 sc->vres.qp.start = val[0]; 2808 sc->vres.qp.size = val[1] - val[0] + 1; 2809 sc->vres.cq.start = val[2]; 2810 sc->vres.cq.size = val[3] - val[2] + 1; 2811 sc->vres.ocq.start = val[4]; 2812 sc->vres.ocq.size = val[5] - val[4] + 1; 2813 } 2814 if (sc->iscsicaps) { 2815 param[0] = FW_PARAM_PFVF(ISCSI_START); 2816 param[1] = FW_PARAM_PFVF(ISCSI_END); 2817 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2818 if (rc != 0) { 2819 device_printf(sc->dev, 2820 "failed to query iSCSI parameters: %d.\n", rc); 2821 return (rc); 2822 } 2823 sc->vres.iscsi.start = val[0]; 2824 sc->vres.iscsi.size = val[1] - val[0] + 1; 2825 } 2826 2827 /* 2828 * We've got the params we wanted to query via the firmware. Now grab 2829 * some others directly from the chip. 2830 */ 2831 rc = t4_read_chip_settings(sc); 2832 2833 return (rc); 2834 } 2835 2836 static int 2837 set_params__post_init(struct adapter *sc) 2838 { 2839 uint32_t param, val; 2840 2841 /* ask for encapsulated CPLs */ 2842 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 2843 val = 1; 2844 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2845 2846 return (0); 2847 } 2848 2849 #undef FW_PARAM_PFVF 2850 #undef FW_PARAM_DEV 2851 2852 static void 2853 t4_set_desc(struct adapter *sc) 2854 { 2855 char buf[128]; 2856 struct adapter_params *p = &sc->params; 2857 2858 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, " 2859 "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "", 2860 chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec); 2861 2862 device_set_desc_copy(sc->dev, buf); 2863 } 2864 2865 static void 2866 build_medialist(struct port_info *pi, struct ifmedia *media) 2867 { 2868 int m; 2869 2870 PORT_LOCK(pi); 2871 2872 ifmedia_removeall(media); 2873 2874 m = IFM_ETHER | IFM_FDX; 2875 2876 switch(pi->port_type) { 2877 case FW_PORT_TYPE_BT_XFI: 2878 case FW_PORT_TYPE_BT_XAUI: 2879 ifmedia_add(media, m | IFM_10G_T, 0, NULL); 2880 /* fall through */ 2881 2882 case FW_PORT_TYPE_BT_SGMII: 2883 ifmedia_add(media, m | IFM_1000_T, 0, NULL); 2884 ifmedia_add(media, m | IFM_100_TX, 0, NULL); 2885 ifmedia_add(media, IFM_ETHER | IFM_AUTO, 0, NULL); 2886 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2887 break; 2888 2889 case FW_PORT_TYPE_CX4: 2890 ifmedia_add(media, m | IFM_10G_CX4, 0, NULL); 2891 ifmedia_set(media, m | IFM_10G_CX4); 2892 break; 2893 2894 case FW_PORT_TYPE_QSFP_10G: 2895 case FW_PORT_TYPE_SFP: 2896 case FW_PORT_TYPE_FIBER_XFI: 2897 case FW_PORT_TYPE_FIBER_XAUI: 2898 switch (pi->mod_type) { 2899 2900 case FW_PORT_MOD_TYPE_LR: 2901 ifmedia_add(media, m | IFM_10G_LR, 0, NULL); 2902 ifmedia_set(media, m | IFM_10G_LR); 2903 break; 2904 2905 case FW_PORT_MOD_TYPE_SR: 2906 ifmedia_add(media, m | IFM_10G_SR, 0, NULL); 2907 ifmedia_set(media, m | IFM_10G_SR); 2908 break; 2909 2910 case FW_PORT_MOD_TYPE_LRM: 2911 ifmedia_add(media, m | IFM_10G_LRM, 0, NULL); 2912 ifmedia_set(media, m | IFM_10G_LRM); 2913 break; 2914 2915 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2916 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2917 ifmedia_add(media, m | IFM_10G_TWINAX, 0, NULL); 2918 ifmedia_set(media, m | IFM_10G_TWINAX); 2919 break; 2920 2921 case FW_PORT_MOD_TYPE_NONE: 2922 m &= ~IFM_FDX; 2923 ifmedia_add(media, m | IFM_NONE, 0, NULL); 2924 ifmedia_set(media, m | IFM_NONE); 2925 break; 2926 2927 case FW_PORT_MOD_TYPE_NA: 2928 case FW_PORT_MOD_TYPE_ER: 2929 default: 2930 device_printf(pi->dev, 2931 "unknown port_type (%d), mod_type (%d)\n", 2932 pi->port_type, pi->mod_type); 2933 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 2934 ifmedia_set(media, m | IFM_UNKNOWN); 2935 break; 2936 } 2937 break; 2938 2939 case FW_PORT_TYPE_QSFP: 2940 switch (pi->mod_type) { 2941 2942 case FW_PORT_MOD_TYPE_LR: 2943 ifmedia_add(media, m | IFM_40G_LR4, 0, NULL); 2944 ifmedia_set(media, m | IFM_40G_LR4); 2945 break; 2946 2947 case FW_PORT_MOD_TYPE_SR: 2948 ifmedia_add(media, m | IFM_40G_SR4, 0, NULL); 2949 ifmedia_set(media, m | IFM_40G_SR4); 2950 break; 2951 2952 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2953 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2954 ifmedia_add(media, m | IFM_40G_CR4, 0, NULL); 2955 ifmedia_set(media, m | IFM_40G_CR4); 2956 break; 2957 2958 case FW_PORT_MOD_TYPE_NONE: 2959 m &= ~IFM_FDX; 2960 ifmedia_add(media, m | IFM_NONE, 0, NULL); 2961 ifmedia_set(media, m | IFM_NONE); 2962 break; 2963 2964 default: 2965 device_printf(pi->dev, 2966 "unknown port_type (%d), mod_type (%d)\n", 2967 pi->port_type, pi->mod_type); 2968 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 2969 ifmedia_set(media, m | IFM_UNKNOWN); 2970 break; 2971 } 2972 break; 2973 2974 default: 2975 device_printf(pi->dev, 2976 "unknown port_type (%d), mod_type (%d)\n", pi->port_type, 2977 pi->mod_type); 2978 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 2979 ifmedia_set(media, m | IFM_UNKNOWN); 2980 break; 2981 } 2982 2983 PORT_UNLOCK(pi); 2984 } 2985 2986 #define FW_MAC_EXACT_CHUNK 7 2987 2988 /* 2989 * Program the port's XGMAC based on parameters in ifnet. The caller also 2990 * indicates which parameters should be programmed (the rest are left alone). 2991 */ 2992 int 2993 update_mac_settings(struct ifnet *ifp, int flags) 2994 { 2995 int rc = 0; 2996 struct port_info *pi = ifp->if_softc; 2997 struct adapter *sc = pi->adapter; 2998 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 2999 uint16_t viid = 0xffff; 3000 int16_t *xact_addr_filt = NULL; 3001 3002 ASSERT_SYNCHRONIZED_OP(sc); 3003 KASSERT(flags, ("%s: not told what to update.", __func__)); 3004 3005 if (ifp == pi->ifp) { 3006 viid = pi->viid; 3007 xact_addr_filt = &pi->xact_addr_filt; 3008 } 3009 #ifdef DEV_NETMAP 3010 else if (ifp == pi->nm_ifp) { 3011 viid = pi->nm_viid; 3012 xact_addr_filt = &pi->nm_xact_addr_filt; 3013 } 3014 #endif 3015 if (flags & XGMAC_MTU) 3016 mtu = ifp->if_mtu; 3017 3018 if (flags & XGMAC_PROMISC) 3019 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 3020 3021 if (flags & XGMAC_ALLMULTI) 3022 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 3023 3024 if (flags & XGMAC_VLANEX) 3025 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 3026 3027 if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) { 3028 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti, 3029 1, vlanex, false); 3030 if (rc) { 3031 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, 3032 rc); 3033 return (rc); 3034 } 3035 } 3036 3037 if (flags & XGMAC_UCADDR) { 3038 uint8_t ucaddr[ETHER_ADDR_LEN]; 3039 3040 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 3041 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr, 3042 true, true); 3043 if (rc < 0) { 3044 rc = -rc; 3045 if_printf(ifp, "change_mac failed: %d\n", rc); 3046 return (rc); 3047 } else { 3048 *xact_addr_filt = rc; 3049 rc = 0; 3050 } 3051 } 3052 3053 if (flags & XGMAC_MCADDRS) { 3054 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 3055 int del = 1; 3056 uint64_t hash = 0; 3057 struct ifmultiaddr *ifma; 3058 int i = 0, j; 3059 3060 if_maddr_rlock(ifp); 3061 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3062 if (ifma->ifma_addr->sa_family != AF_LINK) 3063 continue; 3064 mcaddr[i] = 3065 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 3066 MPASS(ETHER_IS_MULTICAST(mcaddr[i])); 3067 i++; 3068 3069 if (i == FW_MAC_EXACT_CHUNK) { 3070 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, 3071 i, mcaddr, NULL, &hash, 0); 3072 if (rc < 0) { 3073 rc = -rc; 3074 for (j = 0; j < i; j++) { 3075 if_printf(ifp, 3076 "failed to add mc address" 3077 " %02x:%02x:%02x:" 3078 "%02x:%02x:%02x rc=%d\n", 3079 mcaddr[j][0], mcaddr[j][1], 3080 mcaddr[j][2], mcaddr[j][3], 3081 mcaddr[j][4], mcaddr[j][5], 3082 rc); 3083 } 3084 goto mcfail; 3085 } 3086 del = 0; 3087 i = 0; 3088 } 3089 } 3090 if (i > 0) { 3091 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i, 3092 mcaddr, NULL, &hash, 0); 3093 if (rc < 0) { 3094 rc = -rc; 3095 for (j = 0; j < i; j++) { 3096 if_printf(ifp, 3097 "failed to add mc address" 3098 " %02x:%02x:%02x:" 3099 "%02x:%02x:%02x rc=%d\n", 3100 mcaddr[j][0], mcaddr[j][1], 3101 mcaddr[j][2], mcaddr[j][3], 3102 mcaddr[j][4], mcaddr[j][5], 3103 rc); 3104 } 3105 goto mcfail; 3106 } 3107 } 3108 3109 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0); 3110 if (rc != 0) 3111 if_printf(ifp, "failed to set mc address hash: %d", rc); 3112 mcfail: 3113 if_maddr_runlock(ifp); 3114 } 3115 3116 return (rc); 3117 } 3118 3119 /* 3120 * {begin|end}_synchronized_op must be called from the same thread. 3121 */ 3122 int 3123 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags, 3124 char *wmesg) 3125 { 3126 int rc, pri; 3127 3128 #ifdef WITNESS 3129 /* the caller thinks it's ok to sleep, but is it really? */ 3130 if (flags & SLEEP_OK) 3131 pause("t4slptst", 1); 3132 #endif 3133 3134 if (INTR_OK) 3135 pri = PCATCH; 3136 else 3137 pri = 0; 3138 3139 ADAPTER_LOCK(sc); 3140 for (;;) { 3141 3142 if (pi && IS_DOOMED(pi)) { 3143 rc = ENXIO; 3144 goto done; 3145 } 3146 3147 if (!IS_BUSY(sc)) { 3148 rc = 0; 3149 break; 3150 } 3151 3152 if (!(flags & SLEEP_OK)) { 3153 rc = EBUSY; 3154 goto done; 3155 } 3156 3157 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 3158 rc = EINTR; 3159 goto done; 3160 } 3161 } 3162 3163 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 3164 SET_BUSY(sc); 3165 #ifdef INVARIANTS 3166 sc->last_op = wmesg; 3167 sc->last_op_thr = curthread; 3168 #endif 3169 3170 done: 3171 if (!(flags & HOLD_LOCK) || rc) 3172 ADAPTER_UNLOCK(sc); 3173 3174 return (rc); 3175 } 3176 3177 /* 3178 * {begin|end}_synchronized_op must be called from the same thread. 3179 */ 3180 void 3181 end_synchronized_op(struct adapter *sc, int flags) 3182 { 3183 3184 if (flags & LOCK_HELD) 3185 ADAPTER_LOCK_ASSERT_OWNED(sc); 3186 else 3187 ADAPTER_LOCK(sc); 3188 3189 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 3190 CLR_BUSY(sc); 3191 wakeup(&sc->flags); 3192 ADAPTER_UNLOCK(sc); 3193 } 3194 3195 static int 3196 cxgbe_init_synchronized(struct port_info *pi) 3197 { 3198 struct adapter *sc = pi->adapter; 3199 struct ifnet *ifp = pi->ifp; 3200 int rc = 0, i; 3201 struct sge_txq *txq; 3202 3203 ASSERT_SYNCHRONIZED_OP(sc); 3204 3205 if (isset(&sc->open_device_map, pi->port_id)) { 3206 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 3207 ("mismatch between open_device_map and if_drv_flags")); 3208 return (0); /* already running */ 3209 } 3210 3211 if (!(sc->flags & FULL_INIT_DONE) && 3212 ((rc = adapter_full_init(sc)) != 0)) 3213 return (rc); /* error message displayed already */ 3214 3215 if (!(pi->flags & PORT_INIT_DONE) && 3216 ((rc = port_full_init(pi)) != 0)) 3217 return (rc); /* error message displayed already */ 3218 3219 rc = update_mac_settings(ifp, XGMAC_ALL); 3220 if (rc) 3221 goto done; /* error message displayed already */ 3222 3223 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 3224 if (rc != 0) { 3225 if_printf(ifp, "enable_vi failed: %d\n", rc); 3226 goto done; 3227 } 3228 3229 /* 3230 * Can't fail from this point onwards. Review cxgbe_uninit_synchronized 3231 * if this changes. 3232 */ 3233 3234 for_each_txq(pi, i, txq) { 3235 TXQ_LOCK(txq); 3236 txq->eq.flags |= EQ_ENABLED; 3237 TXQ_UNLOCK(txq); 3238 } 3239 3240 /* 3241 * The first iq of the first port to come up is used for tracing. 3242 */ 3243 if (sc->traceq < 0) { 3244 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id; 3245 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : 3246 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | 3247 V_QUEUENUMBER(sc->traceq)); 3248 pi->flags |= HAS_TRACEQ; 3249 } 3250 3251 /* all ok */ 3252 setbit(&sc->open_device_map, pi->port_id); 3253 PORT_LOCK(pi); 3254 ifp->if_drv_flags |= IFF_DRV_RUNNING; 3255 PORT_UNLOCK(pi); 3256 3257 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 3258 done: 3259 if (rc != 0) 3260 cxgbe_uninit_synchronized(pi); 3261 3262 return (rc); 3263 } 3264 3265 /* 3266 * Idempotent. 3267 */ 3268 static int 3269 cxgbe_uninit_synchronized(struct port_info *pi) 3270 { 3271 struct adapter *sc = pi->adapter; 3272 struct ifnet *ifp = pi->ifp; 3273 int rc, i; 3274 struct sge_txq *txq; 3275 3276 ASSERT_SYNCHRONIZED_OP(sc); 3277 3278 if (!(pi->flags & PORT_INIT_DONE)) { 3279 KASSERT(!(ifp->if_drv_flags & IFF_DRV_RUNNING), 3280 ("uninited port is running")); 3281 return (0); 3282 } 3283 3284 /* 3285 * Disable the VI so that all its data in either direction is discarded 3286 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 3287 * tick) intact as the TP can deliver negative advice or data that it's 3288 * holding in its RAM (for an offloaded connection) even after the VI is 3289 * disabled. 3290 */ 3291 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 3292 if (rc) { 3293 if_printf(ifp, "disable_vi failed: %d\n", rc); 3294 return (rc); 3295 } 3296 3297 for_each_txq(pi, i, txq) { 3298 TXQ_LOCK(txq); 3299 txq->eq.flags &= ~EQ_ENABLED; 3300 TXQ_UNLOCK(txq); 3301 } 3302 3303 clrbit(&sc->open_device_map, pi->port_id); 3304 PORT_LOCK(pi); 3305 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3306 PORT_UNLOCK(pi); 3307 3308 pi->link_cfg.link_ok = 0; 3309 pi->link_cfg.speed = 0; 3310 pi->linkdnrc = -1; 3311 t4_os_link_changed(sc, pi->port_id, 0, -1); 3312 3313 return (0); 3314 } 3315 3316 /* 3317 * It is ok for this function to fail midway and return right away. t4_detach 3318 * will walk the entire sc->irq list and clean up whatever is valid. 3319 */ 3320 static int 3321 setup_intr_handlers(struct adapter *sc) 3322 { 3323 int rc, rid, p, q; 3324 char s[8]; 3325 struct irq *irq; 3326 struct port_info *pi; 3327 struct sge_rxq *rxq; 3328 #ifdef TCP_OFFLOAD 3329 struct sge_ofld_rxq *ofld_rxq; 3330 #endif 3331 #ifdef DEV_NETMAP 3332 struct sge_nm_rxq *nm_rxq; 3333 #endif 3334 3335 /* 3336 * Setup interrupts. 3337 */ 3338 irq = &sc->irq[0]; 3339 rid = sc->intr_type == INTR_INTX ? 0 : 1; 3340 if (sc->intr_count == 1) 3341 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all")); 3342 3343 /* Multiple interrupts. */ 3344 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 3345 ("%s: too few intr.", __func__)); 3346 3347 /* The first one is always error intr */ 3348 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 3349 if (rc != 0) 3350 return (rc); 3351 irq++; 3352 rid++; 3353 3354 /* The second one is always the firmware event queue */ 3355 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt"); 3356 if (rc != 0) 3357 return (rc); 3358 irq++; 3359 rid++; 3360 3361 for_each_port(sc, p) { 3362 pi = sc->port[p]; 3363 3364 if (pi->flags & INTR_RXQ) { 3365 for_each_rxq(pi, q, rxq) { 3366 snprintf(s, sizeof(s), "%d.%d", p, q); 3367 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq, 3368 s); 3369 if (rc != 0) 3370 return (rc); 3371 irq++; 3372 rid++; 3373 } 3374 } 3375 #ifdef TCP_OFFLOAD 3376 if (pi->flags & INTR_OFLD_RXQ) { 3377 for_each_ofld_rxq(pi, q, ofld_rxq) { 3378 snprintf(s, sizeof(s), "%d,%d", p, q); 3379 rc = t4_alloc_irq(sc, irq, rid, t4_intr, 3380 ofld_rxq, s); 3381 if (rc != 0) 3382 return (rc); 3383 irq++; 3384 rid++; 3385 } 3386 } 3387 #endif 3388 #ifdef DEV_NETMAP 3389 if (pi->flags & INTR_NM_RXQ) { 3390 for_each_nm_rxq(pi, q, nm_rxq) { 3391 snprintf(s, sizeof(s), "%d-%d", p, q); 3392 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr, 3393 nm_rxq, s); 3394 if (rc != 0) 3395 return (rc); 3396 irq++; 3397 rid++; 3398 } 3399 } 3400 #endif 3401 } 3402 MPASS(irq == &sc->irq[sc->intr_count]); 3403 3404 return (0); 3405 } 3406 3407 int 3408 adapter_full_init(struct adapter *sc) 3409 { 3410 int rc, i; 3411 3412 ASSERT_SYNCHRONIZED_OP(sc); 3413 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3414 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 3415 ("%s: FULL_INIT_DONE already", __func__)); 3416 3417 /* 3418 * queues that belong to the adapter (not any particular port). 3419 */ 3420 rc = t4_setup_adapter_queues(sc); 3421 if (rc != 0) 3422 goto done; 3423 3424 for (i = 0; i < nitems(sc->tq); i++) { 3425 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 3426 taskqueue_thread_enqueue, &sc->tq[i]); 3427 if (sc->tq[i] == NULL) { 3428 device_printf(sc->dev, 3429 "failed to allocate task queue %d\n", i); 3430 rc = ENOMEM; 3431 goto done; 3432 } 3433 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3434 device_get_nameunit(sc->dev), i); 3435 } 3436 3437 t4_intr_enable(sc); 3438 sc->flags |= FULL_INIT_DONE; 3439 done: 3440 if (rc != 0) 3441 adapter_full_uninit(sc); 3442 3443 return (rc); 3444 } 3445 3446 int 3447 adapter_full_uninit(struct adapter *sc) 3448 { 3449 int i; 3450 3451 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3452 3453 t4_teardown_adapter_queues(sc); 3454 3455 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3456 taskqueue_free(sc->tq[i]); 3457 sc->tq[i] = NULL; 3458 } 3459 3460 sc->flags &= ~FULL_INIT_DONE; 3461 3462 return (0); 3463 } 3464 3465 int 3466 port_full_init(struct port_info *pi) 3467 { 3468 struct adapter *sc = pi->adapter; 3469 struct ifnet *ifp = pi->ifp; 3470 uint16_t *rss; 3471 struct sge_rxq *rxq; 3472 int rc, i, j; 3473 3474 ASSERT_SYNCHRONIZED_OP(sc); 3475 KASSERT((pi->flags & PORT_INIT_DONE) == 0, 3476 ("%s: PORT_INIT_DONE already", __func__)); 3477 3478 sysctl_ctx_init(&pi->ctx); 3479 pi->flags |= PORT_SYSCTL_CTX; 3480 3481 /* 3482 * Allocate tx/rx/fl queues for this port. 3483 */ 3484 rc = t4_setup_port_queues(pi); 3485 if (rc != 0) 3486 goto done; /* error message displayed already */ 3487 3488 /* 3489 * Setup RSS for this port. Save a copy of the RSS table for later use. 3490 */ 3491 rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); 3492 for (i = 0; i < pi->rss_size;) { 3493 for_each_rxq(pi, j, rxq) { 3494 rss[i++] = rxq->iq.abs_id; 3495 if (i == pi->rss_size) 3496 break; 3497 } 3498 } 3499 3500 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss, 3501 pi->rss_size); 3502 if (rc != 0) { 3503 if_printf(ifp, "rss_config failed: %d\n", rc); 3504 goto done; 3505 } 3506 3507 pi->rss = rss; 3508 pi->flags |= PORT_INIT_DONE; 3509 done: 3510 if (rc != 0) 3511 port_full_uninit(pi); 3512 3513 return (rc); 3514 } 3515 3516 /* 3517 * Idempotent. 3518 */ 3519 int 3520 port_full_uninit(struct port_info *pi) 3521 { 3522 struct adapter *sc = pi->adapter; 3523 int i; 3524 struct sge_rxq *rxq; 3525 struct sge_txq *txq; 3526 #ifdef TCP_OFFLOAD 3527 struct sge_ofld_rxq *ofld_rxq; 3528 struct sge_wrq *ofld_txq; 3529 #endif 3530 3531 if (pi->flags & PORT_INIT_DONE) { 3532 3533 /* Need to quiesce queues. */ 3534 3535 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]); 3536 3537 for_each_txq(pi, i, txq) { 3538 quiesce_txq(sc, txq); 3539 } 3540 3541 #ifdef TCP_OFFLOAD 3542 for_each_ofld_txq(pi, i, ofld_txq) { 3543 quiesce_wrq(sc, ofld_txq); 3544 } 3545 #endif 3546 3547 for_each_rxq(pi, i, rxq) { 3548 quiesce_iq(sc, &rxq->iq); 3549 quiesce_fl(sc, &rxq->fl); 3550 } 3551 3552 #ifdef TCP_OFFLOAD 3553 for_each_ofld_rxq(pi, i, ofld_rxq) { 3554 quiesce_iq(sc, &ofld_rxq->iq); 3555 quiesce_fl(sc, &ofld_rxq->fl); 3556 } 3557 #endif 3558 free(pi->rss, M_CXGBE); 3559 } 3560 3561 t4_teardown_port_queues(pi); 3562 pi->flags &= ~PORT_INIT_DONE; 3563 3564 return (0); 3565 } 3566 3567 static void 3568 quiesce_txq(struct adapter *sc, struct sge_txq *txq) 3569 { 3570 struct sge_eq *eq = &txq->eq; 3571 struct sge_qstat *spg = (void *)&eq->desc[eq->sidx]; 3572 3573 (void) sc; /* unused */ 3574 3575 #ifdef INVARIANTS 3576 TXQ_LOCK(txq); 3577 MPASS((eq->flags & EQ_ENABLED) == 0); 3578 TXQ_UNLOCK(txq); 3579 #endif 3580 3581 /* Wait for the mp_ring to empty. */ 3582 while (!mp_ring_is_idle(txq->r)) { 3583 mp_ring_check_drainage(txq->r, 0); 3584 pause("rquiesce", 1); 3585 } 3586 3587 /* Then wait for the hardware to finish. */ 3588 while (spg->cidx != htobe16(eq->pidx)) 3589 pause("equiesce", 1); 3590 3591 /* Finally, wait for the driver to reclaim all descriptors. */ 3592 while (eq->cidx != eq->pidx) 3593 pause("dquiesce", 1); 3594 } 3595 3596 static void 3597 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq) 3598 { 3599 3600 /* XXXTX */ 3601 } 3602 3603 static void 3604 quiesce_iq(struct adapter *sc, struct sge_iq *iq) 3605 { 3606 (void) sc; /* unused */ 3607 3608 /* Synchronize with the interrupt handler */ 3609 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 3610 pause("iqfree", 1); 3611 } 3612 3613 static void 3614 quiesce_fl(struct adapter *sc, struct sge_fl *fl) 3615 { 3616 mtx_lock(&sc->sfl_lock); 3617 FL_LOCK(fl); 3618 fl->flags |= FL_DOOMED; 3619 FL_UNLOCK(fl); 3620 mtx_unlock(&sc->sfl_lock); 3621 3622 callout_drain(&sc->sfl_callout); 3623 KASSERT((fl->flags & FL_STARVING) == 0, 3624 ("%s: still starving", __func__)); 3625 } 3626 3627 static int 3628 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 3629 driver_intr_t *handler, void *arg, char *name) 3630 { 3631 int rc; 3632 3633 irq->rid = rid; 3634 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 3635 RF_SHAREABLE | RF_ACTIVE); 3636 if (irq->res == NULL) { 3637 device_printf(sc->dev, 3638 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 3639 return (ENOMEM); 3640 } 3641 3642 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 3643 NULL, handler, arg, &irq->tag); 3644 if (rc != 0) { 3645 device_printf(sc->dev, 3646 "failed to setup interrupt for rid %d, name %s: %d\n", 3647 rid, name, rc); 3648 } else if (name) 3649 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 3650 3651 return (rc); 3652 } 3653 3654 static int 3655 t4_free_irq(struct adapter *sc, struct irq *irq) 3656 { 3657 if (irq->tag) 3658 bus_teardown_intr(sc->dev, irq->res, irq->tag); 3659 if (irq->res) 3660 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 3661 3662 bzero(irq, sizeof(*irq)); 3663 3664 return (0); 3665 } 3666 3667 static void 3668 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 3669 unsigned int end) 3670 { 3671 uint32_t *p = (uint32_t *)(buf + start); 3672 3673 for ( ; start <= end; start += sizeof(uint32_t)) 3674 *p++ = t4_read_reg(sc, start); 3675 } 3676 3677 static void 3678 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 3679 { 3680 int i, n; 3681 const unsigned int *reg_ranges; 3682 static const unsigned int t4_reg_ranges[] = { 3683 0x1008, 0x1108, 3684 0x1180, 0x11b4, 3685 0x11fc, 0x123c, 3686 0x1300, 0x173c, 3687 0x1800, 0x18fc, 3688 0x3000, 0x30d8, 3689 0x30e0, 0x5924, 3690 0x5960, 0x59d4, 3691 0x5a00, 0x5af8, 3692 0x6000, 0x6098, 3693 0x6100, 0x6150, 3694 0x6200, 0x6208, 3695 0x6240, 0x6248, 3696 0x6280, 0x6338, 3697 0x6370, 0x638c, 3698 0x6400, 0x643c, 3699 0x6500, 0x6524, 3700 0x6a00, 0x6a38, 3701 0x6a60, 0x6a78, 3702 0x6b00, 0x6b84, 3703 0x6bf0, 0x6c84, 3704 0x6cf0, 0x6d84, 3705 0x6df0, 0x6e84, 3706 0x6ef0, 0x6f84, 3707 0x6ff0, 0x7084, 3708 0x70f0, 0x7184, 3709 0x71f0, 0x7284, 3710 0x72f0, 0x7384, 3711 0x73f0, 0x7450, 3712 0x7500, 0x7530, 3713 0x7600, 0x761c, 3714 0x7680, 0x76cc, 3715 0x7700, 0x7798, 3716 0x77c0, 0x77fc, 3717 0x7900, 0x79fc, 3718 0x7b00, 0x7c38, 3719 0x7d00, 0x7efc, 3720 0x8dc0, 0x8e1c, 3721 0x8e30, 0x8e78, 3722 0x8ea0, 0x8f6c, 3723 0x8fc0, 0x9074, 3724 0x90fc, 0x90fc, 3725 0x9400, 0x9458, 3726 0x9600, 0x96bc, 3727 0x9800, 0x9808, 3728 0x9820, 0x983c, 3729 0x9850, 0x9864, 3730 0x9c00, 0x9c6c, 3731 0x9c80, 0x9cec, 3732 0x9d00, 0x9d6c, 3733 0x9d80, 0x9dec, 3734 0x9e00, 0x9e6c, 3735 0x9e80, 0x9eec, 3736 0x9f00, 0x9f6c, 3737 0x9f80, 0x9fec, 3738 0xd004, 0xd03c, 3739 0xdfc0, 0xdfe0, 3740 0xe000, 0xea7c, 3741 0xf000, 0x11110, 3742 0x11118, 0x11190, 3743 0x19040, 0x1906c, 3744 0x19078, 0x19080, 3745 0x1908c, 0x19124, 3746 0x19150, 0x191b0, 3747 0x191d0, 0x191e8, 3748 0x19238, 0x1924c, 3749 0x193f8, 0x19474, 3750 0x19490, 0x194f8, 3751 0x19800, 0x19f30, 3752 0x1a000, 0x1a06c, 3753 0x1a0b0, 0x1a120, 3754 0x1a128, 0x1a138, 3755 0x1a190, 0x1a1c4, 3756 0x1a1fc, 0x1a1fc, 3757 0x1e040, 0x1e04c, 3758 0x1e284, 0x1e28c, 3759 0x1e2c0, 0x1e2c0, 3760 0x1e2e0, 0x1e2e0, 3761 0x1e300, 0x1e384, 3762 0x1e3c0, 0x1e3c8, 3763 0x1e440, 0x1e44c, 3764 0x1e684, 0x1e68c, 3765 0x1e6c0, 0x1e6c0, 3766 0x1e6e0, 0x1e6e0, 3767 0x1e700, 0x1e784, 3768 0x1e7c0, 0x1e7c8, 3769 0x1e840, 0x1e84c, 3770 0x1ea84, 0x1ea8c, 3771 0x1eac0, 0x1eac0, 3772 0x1eae0, 0x1eae0, 3773 0x1eb00, 0x1eb84, 3774 0x1ebc0, 0x1ebc8, 3775 0x1ec40, 0x1ec4c, 3776 0x1ee84, 0x1ee8c, 3777 0x1eec0, 0x1eec0, 3778 0x1eee0, 0x1eee0, 3779 0x1ef00, 0x1ef84, 3780 0x1efc0, 0x1efc8, 3781 0x1f040, 0x1f04c, 3782 0x1f284, 0x1f28c, 3783 0x1f2c0, 0x1f2c0, 3784 0x1f2e0, 0x1f2e0, 3785 0x1f300, 0x1f384, 3786 0x1f3c0, 0x1f3c8, 3787 0x1f440, 0x1f44c, 3788 0x1f684, 0x1f68c, 3789 0x1f6c0, 0x1f6c0, 3790 0x1f6e0, 0x1f6e0, 3791 0x1f700, 0x1f784, 3792 0x1f7c0, 0x1f7c8, 3793 0x1f840, 0x1f84c, 3794 0x1fa84, 0x1fa8c, 3795 0x1fac0, 0x1fac0, 3796 0x1fae0, 0x1fae0, 3797 0x1fb00, 0x1fb84, 3798 0x1fbc0, 0x1fbc8, 3799 0x1fc40, 0x1fc4c, 3800 0x1fe84, 0x1fe8c, 3801 0x1fec0, 0x1fec0, 3802 0x1fee0, 0x1fee0, 3803 0x1ff00, 0x1ff84, 3804 0x1ffc0, 0x1ffc8, 3805 0x20000, 0x2002c, 3806 0x20100, 0x2013c, 3807 0x20190, 0x201c8, 3808 0x20200, 0x20318, 3809 0x20400, 0x20528, 3810 0x20540, 0x20614, 3811 0x21000, 0x21040, 3812 0x2104c, 0x21060, 3813 0x210c0, 0x210ec, 3814 0x21200, 0x21268, 3815 0x21270, 0x21284, 3816 0x212fc, 0x21388, 3817 0x21400, 0x21404, 3818 0x21500, 0x21518, 3819 0x2152c, 0x2153c, 3820 0x21550, 0x21554, 3821 0x21600, 0x21600, 3822 0x21608, 0x21628, 3823 0x21630, 0x2163c, 3824 0x21700, 0x2171c, 3825 0x21780, 0x2178c, 3826 0x21800, 0x21c38, 3827 0x21c80, 0x21d7c, 3828 0x21e00, 0x21e04, 3829 0x22000, 0x2202c, 3830 0x22100, 0x2213c, 3831 0x22190, 0x221c8, 3832 0x22200, 0x22318, 3833 0x22400, 0x22528, 3834 0x22540, 0x22614, 3835 0x23000, 0x23040, 3836 0x2304c, 0x23060, 3837 0x230c0, 0x230ec, 3838 0x23200, 0x23268, 3839 0x23270, 0x23284, 3840 0x232fc, 0x23388, 3841 0x23400, 0x23404, 3842 0x23500, 0x23518, 3843 0x2352c, 0x2353c, 3844 0x23550, 0x23554, 3845 0x23600, 0x23600, 3846 0x23608, 0x23628, 3847 0x23630, 0x2363c, 3848 0x23700, 0x2371c, 3849 0x23780, 0x2378c, 3850 0x23800, 0x23c38, 3851 0x23c80, 0x23d7c, 3852 0x23e00, 0x23e04, 3853 0x24000, 0x2402c, 3854 0x24100, 0x2413c, 3855 0x24190, 0x241c8, 3856 0x24200, 0x24318, 3857 0x24400, 0x24528, 3858 0x24540, 0x24614, 3859 0x25000, 0x25040, 3860 0x2504c, 0x25060, 3861 0x250c0, 0x250ec, 3862 0x25200, 0x25268, 3863 0x25270, 0x25284, 3864 0x252fc, 0x25388, 3865 0x25400, 0x25404, 3866 0x25500, 0x25518, 3867 0x2552c, 0x2553c, 3868 0x25550, 0x25554, 3869 0x25600, 0x25600, 3870 0x25608, 0x25628, 3871 0x25630, 0x2563c, 3872 0x25700, 0x2571c, 3873 0x25780, 0x2578c, 3874 0x25800, 0x25c38, 3875 0x25c80, 0x25d7c, 3876 0x25e00, 0x25e04, 3877 0x26000, 0x2602c, 3878 0x26100, 0x2613c, 3879 0x26190, 0x261c8, 3880 0x26200, 0x26318, 3881 0x26400, 0x26528, 3882 0x26540, 0x26614, 3883 0x27000, 0x27040, 3884 0x2704c, 0x27060, 3885 0x270c0, 0x270ec, 3886 0x27200, 0x27268, 3887 0x27270, 0x27284, 3888 0x272fc, 0x27388, 3889 0x27400, 0x27404, 3890 0x27500, 0x27518, 3891 0x2752c, 0x2753c, 3892 0x27550, 0x27554, 3893 0x27600, 0x27600, 3894 0x27608, 0x27628, 3895 0x27630, 0x2763c, 3896 0x27700, 0x2771c, 3897 0x27780, 0x2778c, 3898 0x27800, 0x27c38, 3899 0x27c80, 0x27d7c, 3900 0x27e00, 0x27e04 3901 }; 3902 static const unsigned int t5_reg_ranges[] = { 3903 0x1008, 0x1148, 3904 0x1180, 0x11b4, 3905 0x11fc, 0x123c, 3906 0x1280, 0x173c, 3907 0x1800, 0x18fc, 3908 0x3000, 0x3028, 3909 0x3060, 0x30d8, 3910 0x30e0, 0x30fc, 3911 0x3140, 0x357c, 3912 0x35a8, 0x35cc, 3913 0x35ec, 0x35ec, 3914 0x3600, 0x5624, 3915 0x56cc, 0x575c, 3916 0x580c, 0x5814, 3917 0x5890, 0x58bc, 3918 0x5940, 0x59dc, 3919 0x59fc, 0x5a18, 3920 0x5a60, 0x5a9c, 3921 0x5b94, 0x5bfc, 3922 0x6000, 0x6040, 3923 0x6058, 0x614c, 3924 0x7700, 0x7798, 3925 0x77c0, 0x78fc, 3926 0x7b00, 0x7c54, 3927 0x7d00, 0x7efc, 3928 0x8dc0, 0x8de0, 3929 0x8df8, 0x8e84, 3930 0x8ea0, 0x8f84, 3931 0x8fc0, 0x90f8, 3932 0x9400, 0x9470, 3933 0x9600, 0x96f4, 3934 0x9800, 0x9808, 3935 0x9820, 0x983c, 3936 0x9850, 0x9864, 3937 0x9c00, 0x9c6c, 3938 0x9c80, 0x9cec, 3939 0x9d00, 0x9d6c, 3940 0x9d80, 0x9dec, 3941 0x9e00, 0x9e6c, 3942 0x9e80, 0x9eec, 3943 0x9f00, 0x9f6c, 3944 0x9f80, 0xa020, 3945 0xd004, 0xd03c, 3946 0xdfc0, 0xdfe0, 3947 0xe000, 0x11088, 3948 0x1109c, 0x11110, 3949 0x11118, 0x1117c, 3950 0x11190, 0x11204, 3951 0x19040, 0x1906c, 3952 0x19078, 0x19080, 3953 0x1908c, 0x19124, 3954 0x19150, 0x191b0, 3955 0x191d0, 0x191e8, 3956 0x19238, 0x19290, 3957 0x193f8, 0x19474, 3958 0x19490, 0x194cc, 3959 0x194f0, 0x194f8, 3960 0x19c00, 0x19c60, 3961 0x19c94, 0x19e10, 3962 0x19e50, 0x19f34, 3963 0x19f40, 0x19f50, 3964 0x19f90, 0x19fe4, 3965 0x1a000, 0x1a06c, 3966 0x1a0b0, 0x1a120, 3967 0x1a128, 0x1a138, 3968 0x1a190, 0x1a1c4, 3969 0x1a1fc, 0x1a1fc, 3970 0x1e008, 0x1e00c, 3971 0x1e040, 0x1e04c, 3972 0x1e284, 0x1e290, 3973 0x1e2c0, 0x1e2c0, 3974 0x1e2e0, 0x1e2e0, 3975 0x1e300, 0x1e384, 3976 0x1e3c0, 0x1e3c8, 3977 0x1e408, 0x1e40c, 3978 0x1e440, 0x1e44c, 3979 0x1e684, 0x1e690, 3980 0x1e6c0, 0x1e6c0, 3981 0x1e6e0, 0x1e6e0, 3982 0x1e700, 0x1e784, 3983 0x1e7c0, 0x1e7c8, 3984 0x1e808, 0x1e80c, 3985 0x1e840, 0x1e84c, 3986 0x1ea84, 0x1ea90, 3987 0x1eac0, 0x1eac0, 3988 0x1eae0, 0x1eae0, 3989 0x1eb00, 0x1eb84, 3990 0x1ebc0, 0x1ebc8, 3991 0x1ec08, 0x1ec0c, 3992 0x1ec40, 0x1ec4c, 3993 0x1ee84, 0x1ee90, 3994 0x1eec0, 0x1eec0, 3995 0x1eee0, 0x1eee0, 3996 0x1ef00, 0x1ef84, 3997 0x1efc0, 0x1efc8, 3998 0x1f008, 0x1f00c, 3999 0x1f040, 0x1f04c, 4000 0x1f284, 0x1f290, 4001 0x1f2c0, 0x1f2c0, 4002 0x1f2e0, 0x1f2e0, 4003 0x1f300, 0x1f384, 4004 0x1f3c0, 0x1f3c8, 4005 0x1f408, 0x1f40c, 4006 0x1f440, 0x1f44c, 4007 0x1f684, 0x1f690, 4008 0x1f6c0, 0x1f6c0, 4009 0x1f6e0, 0x1f6e0, 4010 0x1f700, 0x1f784, 4011 0x1f7c0, 0x1f7c8, 4012 0x1f808, 0x1f80c, 4013 0x1f840, 0x1f84c, 4014 0x1fa84, 0x1fa90, 4015 0x1fac0, 0x1fac0, 4016 0x1fae0, 0x1fae0, 4017 0x1fb00, 0x1fb84, 4018 0x1fbc0, 0x1fbc8, 4019 0x1fc08, 0x1fc0c, 4020 0x1fc40, 0x1fc4c, 4021 0x1fe84, 0x1fe90, 4022 0x1fec0, 0x1fec0, 4023 0x1fee0, 0x1fee0, 4024 0x1ff00, 0x1ff84, 4025 0x1ffc0, 0x1ffc8, 4026 0x30000, 0x30030, 4027 0x30100, 0x30144, 4028 0x30190, 0x301d0, 4029 0x30200, 0x30318, 4030 0x30400, 0x3052c, 4031 0x30540, 0x3061c, 4032 0x30800, 0x30834, 4033 0x308c0, 0x30908, 4034 0x30910, 0x309ac, 4035 0x30a00, 0x30a2c, 4036 0x30a44, 0x30a50, 4037 0x30a74, 0x30c24, 4038 0x30d00, 0x30d00, 4039 0x30d08, 0x30d14, 4040 0x30d1c, 0x30d20, 4041 0x30d3c, 0x30d50, 4042 0x31200, 0x3120c, 4043 0x31220, 0x31220, 4044 0x31240, 0x31240, 4045 0x31600, 0x3160c, 4046 0x31a00, 0x31a1c, 4047 0x31e00, 0x31e20, 4048 0x31e38, 0x31e3c, 4049 0x31e80, 0x31e80, 4050 0x31e88, 0x31ea8, 4051 0x31eb0, 0x31eb4, 4052 0x31ec8, 0x31ed4, 4053 0x31fb8, 0x32004, 4054 0x32200, 0x32200, 4055 0x32208, 0x32240, 4056 0x32248, 0x32280, 4057 0x32288, 0x322c0, 4058 0x322c8, 0x322fc, 4059 0x32600, 0x32630, 4060 0x32a00, 0x32abc, 4061 0x32b00, 0x32b70, 4062 0x33000, 0x33048, 4063 0x33060, 0x3309c, 4064 0x330f0, 0x33148, 4065 0x33160, 0x3319c, 4066 0x331f0, 0x332e4, 4067 0x332f8, 0x333e4, 4068 0x333f8, 0x33448, 4069 0x33460, 0x3349c, 4070 0x334f0, 0x33548, 4071 0x33560, 0x3359c, 4072 0x335f0, 0x336e4, 4073 0x336f8, 0x337e4, 4074 0x337f8, 0x337fc, 4075 0x33814, 0x33814, 4076 0x3382c, 0x3382c, 4077 0x33880, 0x3388c, 4078 0x338e8, 0x338ec, 4079 0x33900, 0x33948, 4080 0x33960, 0x3399c, 4081 0x339f0, 0x33ae4, 4082 0x33af8, 0x33b10, 4083 0x33b28, 0x33b28, 4084 0x33b3c, 0x33b50, 4085 0x33bf0, 0x33c10, 4086 0x33c28, 0x33c28, 4087 0x33c3c, 0x33c50, 4088 0x33cf0, 0x33cfc, 4089 0x34000, 0x34030, 4090 0x34100, 0x34144, 4091 0x34190, 0x341d0, 4092 0x34200, 0x34318, 4093 0x34400, 0x3452c, 4094 0x34540, 0x3461c, 4095 0x34800, 0x34834, 4096 0x348c0, 0x34908, 4097 0x34910, 0x349ac, 4098 0x34a00, 0x34a2c, 4099 0x34a44, 0x34a50, 4100 0x34a74, 0x34c24, 4101 0x34d00, 0x34d00, 4102 0x34d08, 0x34d14, 4103 0x34d1c, 0x34d20, 4104 0x34d3c, 0x34d50, 4105 0x35200, 0x3520c, 4106 0x35220, 0x35220, 4107 0x35240, 0x35240, 4108 0x35600, 0x3560c, 4109 0x35a00, 0x35a1c, 4110 0x35e00, 0x35e20, 4111 0x35e38, 0x35e3c, 4112 0x35e80, 0x35e80, 4113 0x35e88, 0x35ea8, 4114 0x35eb0, 0x35eb4, 4115 0x35ec8, 0x35ed4, 4116 0x35fb8, 0x36004, 4117 0x36200, 0x36200, 4118 0x36208, 0x36240, 4119 0x36248, 0x36280, 4120 0x36288, 0x362c0, 4121 0x362c8, 0x362fc, 4122 0x36600, 0x36630, 4123 0x36a00, 0x36abc, 4124 0x36b00, 0x36b70, 4125 0x37000, 0x37048, 4126 0x37060, 0x3709c, 4127 0x370f0, 0x37148, 4128 0x37160, 0x3719c, 4129 0x371f0, 0x372e4, 4130 0x372f8, 0x373e4, 4131 0x373f8, 0x37448, 4132 0x37460, 0x3749c, 4133 0x374f0, 0x37548, 4134 0x37560, 0x3759c, 4135 0x375f0, 0x376e4, 4136 0x376f8, 0x377e4, 4137 0x377f8, 0x377fc, 4138 0x37814, 0x37814, 4139 0x3782c, 0x3782c, 4140 0x37880, 0x3788c, 4141 0x378e8, 0x378ec, 4142 0x37900, 0x37948, 4143 0x37960, 0x3799c, 4144 0x379f0, 0x37ae4, 4145 0x37af8, 0x37b10, 4146 0x37b28, 0x37b28, 4147 0x37b3c, 0x37b50, 4148 0x37bf0, 0x37c10, 4149 0x37c28, 0x37c28, 4150 0x37c3c, 0x37c50, 4151 0x37cf0, 0x37cfc, 4152 0x38000, 0x38030, 4153 0x38100, 0x38144, 4154 0x38190, 0x381d0, 4155 0x38200, 0x38318, 4156 0x38400, 0x3852c, 4157 0x38540, 0x3861c, 4158 0x38800, 0x38834, 4159 0x388c0, 0x38908, 4160 0x38910, 0x389ac, 4161 0x38a00, 0x38a2c, 4162 0x38a44, 0x38a50, 4163 0x38a74, 0x38c24, 4164 0x38d00, 0x38d00, 4165 0x38d08, 0x38d14, 4166 0x38d1c, 0x38d20, 4167 0x38d3c, 0x38d50, 4168 0x39200, 0x3920c, 4169 0x39220, 0x39220, 4170 0x39240, 0x39240, 4171 0x39600, 0x3960c, 4172 0x39a00, 0x39a1c, 4173 0x39e00, 0x39e20, 4174 0x39e38, 0x39e3c, 4175 0x39e80, 0x39e80, 4176 0x39e88, 0x39ea8, 4177 0x39eb0, 0x39eb4, 4178 0x39ec8, 0x39ed4, 4179 0x39fb8, 0x3a004, 4180 0x3a200, 0x3a200, 4181 0x3a208, 0x3a240, 4182 0x3a248, 0x3a280, 4183 0x3a288, 0x3a2c0, 4184 0x3a2c8, 0x3a2fc, 4185 0x3a600, 0x3a630, 4186 0x3aa00, 0x3aabc, 4187 0x3ab00, 0x3ab70, 4188 0x3b000, 0x3b048, 4189 0x3b060, 0x3b09c, 4190 0x3b0f0, 0x3b148, 4191 0x3b160, 0x3b19c, 4192 0x3b1f0, 0x3b2e4, 4193 0x3b2f8, 0x3b3e4, 4194 0x3b3f8, 0x3b448, 4195 0x3b460, 0x3b49c, 4196 0x3b4f0, 0x3b548, 4197 0x3b560, 0x3b59c, 4198 0x3b5f0, 0x3b6e4, 4199 0x3b6f8, 0x3b7e4, 4200 0x3b7f8, 0x3b7fc, 4201 0x3b814, 0x3b814, 4202 0x3b82c, 0x3b82c, 4203 0x3b880, 0x3b88c, 4204 0x3b8e8, 0x3b8ec, 4205 0x3b900, 0x3b948, 4206 0x3b960, 0x3b99c, 4207 0x3b9f0, 0x3bae4, 4208 0x3baf8, 0x3bb10, 4209 0x3bb28, 0x3bb28, 4210 0x3bb3c, 0x3bb50, 4211 0x3bbf0, 0x3bc10, 4212 0x3bc28, 0x3bc28, 4213 0x3bc3c, 0x3bc50, 4214 0x3bcf0, 0x3bcfc, 4215 0x3c000, 0x3c030, 4216 0x3c100, 0x3c144, 4217 0x3c190, 0x3c1d0, 4218 0x3c200, 0x3c318, 4219 0x3c400, 0x3c52c, 4220 0x3c540, 0x3c61c, 4221 0x3c800, 0x3c834, 4222 0x3c8c0, 0x3c908, 4223 0x3c910, 0x3c9ac, 4224 0x3ca00, 0x3ca2c, 4225 0x3ca44, 0x3ca50, 4226 0x3ca74, 0x3cc24, 4227 0x3cd00, 0x3cd00, 4228 0x3cd08, 0x3cd14, 4229 0x3cd1c, 0x3cd20, 4230 0x3cd3c, 0x3cd50, 4231 0x3d200, 0x3d20c, 4232 0x3d220, 0x3d220, 4233 0x3d240, 0x3d240, 4234 0x3d600, 0x3d60c, 4235 0x3da00, 0x3da1c, 4236 0x3de00, 0x3de20, 4237 0x3de38, 0x3de3c, 4238 0x3de80, 0x3de80, 4239 0x3de88, 0x3dea8, 4240 0x3deb0, 0x3deb4, 4241 0x3dec8, 0x3ded4, 4242 0x3dfb8, 0x3e004, 4243 0x3e200, 0x3e200, 4244 0x3e208, 0x3e240, 4245 0x3e248, 0x3e280, 4246 0x3e288, 0x3e2c0, 4247 0x3e2c8, 0x3e2fc, 4248 0x3e600, 0x3e630, 4249 0x3ea00, 0x3eabc, 4250 0x3eb00, 0x3eb70, 4251 0x3f000, 0x3f048, 4252 0x3f060, 0x3f09c, 4253 0x3f0f0, 0x3f148, 4254 0x3f160, 0x3f19c, 4255 0x3f1f0, 0x3f2e4, 4256 0x3f2f8, 0x3f3e4, 4257 0x3f3f8, 0x3f448, 4258 0x3f460, 0x3f49c, 4259 0x3f4f0, 0x3f548, 4260 0x3f560, 0x3f59c, 4261 0x3f5f0, 0x3f6e4, 4262 0x3f6f8, 0x3f7e4, 4263 0x3f7f8, 0x3f7fc, 4264 0x3f814, 0x3f814, 4265 0x3f82c, 0x3f82c, 4266 0x3f880, 0x3f88c, 4267 0x3f8e8, 0x3f8ec, 4268 0x3f900, 0x3f948, 4269 0x3f960, 0x3f99c, 4270 0x3f9f0, 0x3fae4, 4271 0x3faf8, 0x3fb10, 4272 0x3fb28, 0x3fb28, 4273 0x3fb3c, 0x3fb50, 4274 0x3fbf0, 0x3fc10, 4275 0x3fc28, 0x3fc28, 4276 0x3fc3c, 0x3fc50, 4277 0x3fcf0, 0x3fcfc, 4278 0x40000, 0x4000c, 4279 0x40040, 0x40068, 4280 0x4007c, 0x40144, 4281 0x40180, 0x4018c, 4282 0x40200, 0x40298, 4283 0x402ac, 0x4033c, 4284 0x403f8, 0x403fc, 4285 0x41304, 0x413c4, 4286 0x41400, 0x4141c, 4287 0x41480, 0x414d0, 4288 0x44000, 0x44078, 4289 0x440c0, 0x44278, 4290 0x442c0, 0x44478, 4291 0x444c0, 0x44678, 4292 0x446c0, 0x44878, 4293 0x448c0, 0x449fc, 4294 0x45000, 0x45068, 4295 0x45080, 0x45084, 4296 0x450a0, 0x450b0, 4297 0x45200, 0x45268, 4298 0x45280, 0x45284, 4299 0x452a0, 0x452b0, 4300 0x460c0, 0x460e4, 4301 0x47000, 0x4708c, 4302 0x47200, 0x47250, 4303 0x47400, 0x47420, 4304 0x47600, 0x47618, 4305 0x47800, 0x47814, 4306 0x48000, 0x4800c, 4307 0x48040, 0x48068, 4308 0x4807c, 0x48144, 4309 0x48180, 0x4818c, 4310 0x48200, 0x48298, 4311 0x482ac, 0x4833c, 4312 0x483f8, 0x483fc, 4313 0x49304, 0x493c4, 4314 0x49400, 0x4941c, 4315 0x49480, 0x494d0, 4316 0x4c000, 0x4c078, 4317 0x4c0c0, 0x4c278, 4318 0x4c2c0, 0x4c478, 4319 0x4c4c0, 0x4c678, 4320 0x4c6c0, 0x4c878, 4321 0x4c8c0, 0x4c9fc, 4322 0x4d000, 0x4d068, 4323 0x4d080, 0x4d084, 4324 0x4d0a0, 0x4d0b0, 4325 0x4d200, 0x4d268, 4326 0x4d280, 0x4d284, 4327 0x4d2a0, 0x4d2b0, 4328 0x4e0c0, 0x4e0e4, 4329 0x4f000, 0x4f08c, 4330 0x4f200, 0x4f250, 4331 0x4f400, 0x4f420, 4332 0x4f600, 0x4f618, 4333 0x4f800, 0x4f814, 4334 0x50000, 0x500cc, 4335 0x50400, 0x50400, 4336 0x50800, 0x508cc, 4337 0x50c00, 0x50c00, 4338 0x51000, 0x5101c, 4339 0x51300, 0x51308, 4340 }; 4341 4342 if (is_t4(sc)) { 4343 reg_ranges = &t4_reg_ranges[0]; 4344 n = nitems(t4_reg_ranges); 4345 } else { 4346 reg_ranges = &t5_reg_ranges[0]; 4347 n = nitems(t5_reg_ranges); 4348 } 4349 4350 regs->version = chip_id(sc) | chip_rev(sc) << 10; 4351 for (i = 0; i < n; i += 2) 4352 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 4353 } 4354 4355 static void 4356 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi) 4357 { 4358 int i; 4359 u_int v, tnl_cong_drops; 4360 struct timeval tv; 4361 const struct timeval interval = {0, 250000}; /* 250ms */ 4362 4363 getmicrotime(&tv); 4364 timevalsub(&tv, &interval); 4365 if (timevalcmp(&tv, &pi->last_refreshed, <)) 4366 return; 4367 4368 tnl_cong_drops = 0; 4369 t4_get_port_stats(sc, pi->tx_chan, &pi->stats); 4370 for (i = 0; i < NCHAN; i++) { 4371 if (pi->rx_chan_map & (1 << i)) { 4372 mtx_lock(&sc->regwin_lock); 4373 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 4374 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 4375 mtx_unlock(&sc->regwin_lock); 4376 tnl_cong_drops += v; 4377 } 4378 } 4379 pi->tnl_cong_drops = tnl_cong_drops; 4380 getmicrotime(&pi->last_refreshed); 4381 } 4382 4383 static void 4384 cxgbe_tick(void *arg) 4385 { 4386 struct port_info *pi = arg; 4387 struct adapter *sc = pi->adapter; 4388 struct ifnet *ifp = pi->ifp; 4389 4390 PORT_LOCK(pi); 4391 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 4392 PORT_UNLOCK(pi); 4393 return; /* without scheduling another callout */ 4394 } 4395 4396 cxgbe_refresh_stats(sc, pi); 4397 4398 callout_schedule(&pi->tick, hz); 4399 PORT_UNLOCK(pi); 4400 } 4401 4402 static void 4403 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 4404 { 4405 struct ifnet *vlan; 4406 4407 if (arg != ifp || ifp->if_type != IFT_ETHER) 4408 return; 4409 4410 vlan = VLAN_DEVAT(ifp, vid); 4411 VLAN_SETCOOKIE(vlan, ifp); 4412 } 4413 4414 static int 4415 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 4416 { 4417 4418 #ifdef INVARIANTS 4419 panic("%s: opcode 0x%02x on iq %p with payload %p", 4420 __func__, rss->opcode, iq, m); 4421 #else 4422 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 4423 __func__, rss->opcode, iq, m); 4424 m_freem(m); 4425 #endif 4426 return (EDOOFUS); 4427 } 4428 4429 int 4430 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 4431 { 4432 uintptr_t *loc, new; 4433 4434 if (opcode >= nitems(sc->cpl_handler)) 4435 return (EINVAL); 4436 4437 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 4438 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 4439 atomic_store_rel_ptr(loc, new); 4440 4441 return (0); 4442 } 4443 4444 static int 4445 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 4446 { 4447 4448 #ifdef INVARIANTS 4449 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 4450 #else 4451 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 4452 __func__, iq, ctrl); 4453 #endif 4454 return (EDOOFUS); 4455 } 4456 4457 int 4458 t4_register_an_handler(struct adapter *sc, an_handler_t h) 4459 { 4460 uintptr_t *loc, new; 4461 4462 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 4463 loc = (uintptr_t *) &sc->an_handler; 4464 atomic_store_rel_ptr(loc, new); 4465 4466 return (0); 4467 } 4468 4469 static int 4470 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 4471 { 4472 const struct cpl_fw6_msg *cpl = 4473 __containerof(rpl, struct cpl_fw6_msg, data[0]); 4474 4475 #ifdef INVARIANTS 4476 panic("%s: fw_msg type %d", __func__, cpl->type); 4477 #else 4478 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 4479 #endif 4480 return (EDOOFUS); 4481 } 4482 4483 int 4484 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 4485 { 4486 uintptr_t *loc, new; 4487 4488 if (type >= nitems(sc->fw_msg_handler)) 4489 return (EINVAL); 4490 4491 /* 4492 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 4493 * handler dispatch table. Reject any attempt to install a handler for 4494 * this subtype. 4495 */ 4496 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 4497 return (EINVAL); 4498 4499 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 4500 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 4501 atomic_store_rel_ptr(loc, new); 4502 4503 return (0); 4504 } 4505 4506 static int 4507 t4_sysctls(struct adapter *sc) 4508 { 4509 struct sysctl_ctx_list *ctx; 4510 struct sysctl_oid *oid; 4511 struct sysctl_oid_list *children, *c0; 4512 static char *caps[] = { 4513 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 4514 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL" /* caps[1] niccaps */ 4515 "\6HASHFILTER\7ETHOFLD", 4516 "\20\1TOE", /* caps[2] toecaps */ 4517 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 4518 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 4519 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 4520 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 4521 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 4522 "\4PO_INITIAOR\5PO_TARGET" 4523 }; 4524 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 4525 4526 ctx = device_get_sysctl_ctx(sc->dev); 4527 4528 /* 4529 * dev.t4nex.X. 4530 */ 4531 oid = device_get_sysctl_tree(sc->dev); 4532 c0 = children = SYSCTL_CHILDREN(oid); 4533 4534 sc->sc_do_rxcopy = 1; 4535 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, 4536 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); 4537 4538 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 4539 sc->params.nports, "# of ports"); 4540 4541 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 4542 NULL, chip_rev(sc), "chip hardware revision"); 4543 4544 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 4545 CTLFLAG_RD, sc->fw_version, 0, "firmware version"); 4546 4547 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 4548 CTLFLAG_RD, sc->cfg_file, 0, "configuration file"); 4549 4550 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 4551 sc->cfcsum, "config file checksum"); 4552 4553 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 4554 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 4555 sysctl_bitfield, "A", "available doorbells"); 4556 4557 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 4558 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 4559 sysctl_bitfield, "A", "available link capabilities"); 4560 4561 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 4562 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 4563 sysctl_bitfield, "A", "available NIC capabilities"); 4564 4565 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 4566 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 4567 sysctl_bitfield, "A", "available TCP offload capabilities"); 4568 4569 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 4570 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 4571 sysctl_bitfield, "A", "available RDMA capabilities"); 4572 4573 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 4574 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 4575 sysctl_bitfield, "A", "available iSCSI capabilities"); 4576 4577 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 4578 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 4579 sysctl_bitfield, "A", "available FCoE capabilities"); 4580 4581 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 4582 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 4583 4584 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 4585 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 4586 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 4587 "interrupt holdoff timer values (us)"); 4588 4589 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 4590 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 4591 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 4592 "interrupt holdoff packet counter values"); 4593 4594 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, 4595 NULL, sc->tids.nftids, "number of filters"); 4596 4597 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | 4598 CTLFLAG_RD, sc, 0, sysctl_temperature, "I", 4599 "chip temperature (in Celsius)"); 4600 4601 t4_sge_sysctls(sc, ctx, children); 4602 4603 sc->lro_timeout = 100; 4604 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, 4605 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); 4606 4607 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "debug_flags", CTLFLAG_RW, 4608 &sc->debug_flags, 0, "flags to enable runtime debugging"); 4609 4610 #ifdef SBUF_DRAIN 4611 /* 4612 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 4613 */ 4614 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 4615 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 4616 "logs and miscellaneous information"); 4617 children = SYSCTL_CHILDREN(oid); 4618 4619 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 4620 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4621 sysctl_cctrl, "A", "congestion control"); 4622 4623 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 4624 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4625 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 4626 4627 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 4628 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 4629 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 4630 4631 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 4632 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 4633 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 4634 4635 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 4636 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 4637 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 4638 4639 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 4640 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 4641 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 4642 4643 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 4644 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 4645 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 4646 4647 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 4648 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4649 sysctl_cim_la, "A", "CIM logic analyzer"); 4650 4651 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", 4652 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4653 sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); 4654 4655 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 4656 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 4657 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 4658 4659 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 4660 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 4661 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 4662 4663 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 4664 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 4665 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 4666 4667 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 4668 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 4669 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 4670 4671 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 4672 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 4673 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 4674 4675 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 4676 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 4677 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 4678 4679 if (is_t5(sc)) { 4680 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 4681 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 4682 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 4683 4684 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 4685 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 4686 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 4687 } 4688 4689 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", 4690 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4691 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); 4692 4693 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 4694 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4695 sysctl_cim_qcfg, "A", "CIM queue configuration"); 4696 4697 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 4698 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4699 sysctl_cpl_stats, "A", "CPL statistics"); 4700 4701 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 4702 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4703 sysctl_ddp_stats, "A", "non-TCP DDP statistics"); 4704 4705 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 4706 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4707 sysctl_devlog, "A", "firmware's device log"); 4708 4709 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 4710 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4711 sysctl_fcoe_stats, "A", "FCoE statistics"); 4712 4713 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 4714 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4715 sysctl_hw_sched, "A", "hardware scheduler "); 4716 4717 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 4718 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4719 sysctl_l2t, "A", "hardware L2 table"); 4720 4721 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 4722 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4723 sysctl_lb_stats, "A", "loopback statistics"); 4724 4725 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 4726 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4727 sysctl_meminfo, "A", "memory regions"); 4728 4729 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", 4730 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4731 sysctl_mps_tcam, "A", "MPS TCAM entries"); 4732 4733 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 4734 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4735 sysctl_path_mtus, "A", "path MTUs"); 4736 4737 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 4738 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4739 sysctl_pm_stats, "A", "PM statistics"); 4740 4741 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 4742 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4743 sysctl_rdma_stats, "A", "RDMA statistics"); 4744 4745 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 4746 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4747 sysctl_tcp_stats, "A", "TCP statistics"); 4748 4749 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 4750 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4751 sysctl_tids, "A", "TID information"); 4752 4753 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 4754 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4755 sysctl_tp_err_stats, "A", "TP error statistics"); 4756 4757 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", 4758 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4759 sysctl_tp_la, "A", "TP logic analyzer"); 4760 4761 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 4762 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4763 sysctl_tx_rate, "A", "Tx rate"); 4764 4765 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", 4766 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4767 sysctl_ulprx_la, "A", "ULPRX logic analyzer"); 4768 4769 if (is_t5(sc)) { 4770 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 4771 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4772 sysctl_wcwr_stats, "A", "write combined work requests"); 4773 } 4774 #endif 4775 4776 #ifdef TCP_OFFLOAD 4777 if (is_offload(sc)) { 4778 /* 4779 * dev.t4nex.X.toe. 4780 */ 4781 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 4782 NULL, "TOE parameters"); 4783 children = SYSCTL_CHILDREN(oid); 4784 4785 sc->tt.sndbuf = 256 * 1024; 4786 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 4787 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 4788 4789 sc->tt.ddp = 0; 4790 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 4791 &sc->tt.ddp, 0, "DDP allowed"); 4792 4793 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 4794 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 4795 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 4796 4797 sc->tt.ddp_thres = 4798 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 4799 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 4800 &sc->tt.ddp_thres, 0, "DDP threshold"); 4801 4802 sc->tt.rx_coalesce = 1; 4803 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", 4804 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); 4805 4806 sc->tt.tx_align = 1; 4807 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align", 4808 CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload"); 4809 } 4810 #endif 4811 4812 4813 return (0); 4814 } 4815 4816 static int 4817 cxgbe_sysctls(struct port_info *pi) 4818 { 4819 struct sysctl_ctx_list *ctx; 4820 struct sysctl_oid *oid; 4821 struct sysctl_oid_list *children; 4822 struct adapter *sc = pi->adapter; 4823 4824 ctx = device_get_sysctl_ctx(pi->dev); 4825 4826 /* 4827 * dev.cxgbe.X. 4828 */ 4829 oid = device_get_sysctl_tree(pi->dev); 4830 children = SYSCTL_CHILDREN(oid); 4831 4832 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | 4833 CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); 4834 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { 4835 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 4836 CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I", 4837 "PHY temperature (in Celsius)"); 4838 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", 4839 CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I", 4840 "PHY firmware version"); 4841 } 4842 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 4843 &pi->nrxq, 0, "# of rx queues"); 4844 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 4845 &pi->ntxq, 0, "# of tx queues"); 4846 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 4847 &pi->first_rxq, 0, "index of first rx queue"); 4848 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 4849 &pi->first_txq, 0, "index of first tx queue"); 4850 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT | 4851 CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU", 4852 "Reserve queue 0 for non-flowid packets"); 4853 4854 #ifdef TCP_OFFLOAD 4855 if (is_offload(sc)) { 4856 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 4857 &pi->nofldrxq, 0, 4858 "# of rx queues for offloaded TCP connections"); 4859 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 4860 &pi->nofldtxq, 0, 4861 "# of tx queues for offloaded TCP connections"); 4862 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 4863 CTLFLAG_RD, &pi->first_ofld_rxq, 0, 4864 "index of first TOE rx queue"); 4865 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 4866 CTLFLAG_RD, &pi->first_ofld_txq, 0, 4867 "index of first TOE tx queue"); 4868 } 4869 #endif 4870 #ifdef DEV_NETMAP 4871 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD, 4872 &pi->nnmrxq, 0, "# of rx queues for netmap"); 4873 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD, 4874 &pi->nnmtxq, 0, "# of tx queues for netmap"); 4875 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq", 4876 CTLFLAG_RD, &pi->first_nm_rxq, 0, 4877 "index of first netmap rx queue"); 4878 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq", 4879 CTLFLAG_RD, &pi->first_nm_txq, 0, 4880 "index of first netmap tx queue"); 4881 #endif 4882 4883 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 4884 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 4885 "holdoff timer index"); 4886 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 4887 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 4888 "holdoff packet counter index"); 4889 4890 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 4891 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 4892 "rx queue size"); 4893 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 4894 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 4895 "tx queue size"); 4896 4897 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings", 4898 CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings, 4899 "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)"); 4900 4901 /* 4902 * dev.cxgbe.X.stats. 4903 */ 4904 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 4905 NULL, "port statistics"); 4906 children = SYSCTL_CHILDREN(oid); 4907 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD, 4908 &pi->tx_parse_error, 0, 4909 "# of tx packets with invalid length or # of segments"); 4910 4911 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 4912 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 4913 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ 4914 sysctl_handle_t4_reg64, "QU", desc) 4915 4916 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 4917 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 4918 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 4919 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 4920 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 4921 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 4922 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 4923 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 4924 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 4925 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 4926 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 4927 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 4928 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 4929 "# of tx frames in this range", 4930 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 4931 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 4932 "# of tx frames in this range", 4933 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 4934 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 4935 "# of tx frames in this range", 4936 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 4937 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 4938 "# of tx frames in this range", 4939 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 4940 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 4941 "# of tx frames in this range", 4942 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 4943 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 4944 "# of tx frames in this range", 4945 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 4946 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 4947 "# of tx frames in this range", 4948 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 4949 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 4950 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 4951 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 4952 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 4953 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 4954 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 4955 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 4956 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 4957 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 4958 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 4959 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 4960 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 4961 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 4962 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 4963 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 4964 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 4965 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 4966 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 4967 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 4968 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 4969 4970 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 4971 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 4972 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 4973 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 4974 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 4975 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 4976 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 4977 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 4978 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 4979 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 4980 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 4981 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 4982 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 4983 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 4984 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 4985 "# of frames received with bad FCS", 4986 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 4987 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 4988 "# of frames received with length error", 4989 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 4990 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 4991 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 4992 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 4993 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 4994 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 4995 "# of rx frames in this range", 4996 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 4997 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 4998 "# of rx frames in this range", 4999 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 5000 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 5001 "# of rx frames in this range", 5002 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 5003 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 5004 "# of rx frames in this range", 5005 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 5006 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 5007 "# of rx frames in this range", 5008 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 5009 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 5010 "# of rx frames in this range", 5011 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 5012 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 5013 "# of rx frames in this range", 5014 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 5015 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 5016 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 5017 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 5018 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 5019 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 5020 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 5021 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 5022 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 5023 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 5024 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 5025 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 5026 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 5027 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 5028 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 5029 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 5030 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 5031 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 5032 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 5033 5034 #undef SYSCTL_ADD_T4_REG64 5035 5036 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 5037 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 5038 &pi->stats.name, desc) 5039 5040 /* We get these from port_stats and they may be stale by upto 1s */ 5041 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 5042 "# drops due to buffer-group 0 overflows"); 5043 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 5044 "# drops due to buffer-group 1 overflows"); 5045 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 5046 "# drops due to buffer-group 2 overflows"); 5047 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 5048 "# drops due to buffer-group 3 overflows"); 5049 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 5050 "# of buffer-group 0 truncated packets"); 5051 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 5052 "# of buffer-group 1 truncated packets"); 5053 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 5054 "# of buffer-group 2 truncated packets"); 5055 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 5056 "# of buffer-group 3 truncated packets"); 5057 5058 #undef SYSCTL_ADD_T4_PORTSTAT 5059 5060 return (0); 5061 } 5062 5063 static int 5064 sysctl_int_array(SYSCTL_HANDLER_ARGS) 5065 { 5066 int rc, *i, space = 0; 5067 struct sbuf sb; 5068 5069 sbuf_new_for_sysctl(&sb, NULL, 64, req); 5070 for (i = arg1; arg2; arg2 -= sizeof(int), i++) { 5071 if (space) 5072 sbuf_printf(&sb, " "); 5073 sbuf_printf(&sb, "%d", *i); 5074 space = 1; 5075 } 5076 rc = sbuf_finish(&sb); 5077 sbuf_delete(&sb); 5078 return (rc); 5079 } 5080 5081 static int 5082 sysctl_bitfield(SYSCTL_HANDLER_ARGS) 5083 { 5084 int rc; 5085 struct sbuf *sb; 5086 5087 rc = sysctl_wire_old_buffer(req, 0); 5088 if (rc != 0) 5089 return(rc); 5090 5091 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5092 if (sb == NULL) 5093 return (ENOMEM); 5094 5095 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 5096 rc = sbuf_finish(sb); 5097 sbuf_delete(sb); 5098 5099 return (rc); 5100 } 5101 5102 static int 5103 sysctl_btphy(SYSCTL_HANDLER_ARGS) 5104 { 5105 struct port_info *pi = arg1; 5106 int op = arg2; 5107 struct adapter *sc = pi->adapter; 5108 u_int v; 5109 int rc; 5110 5111 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt"); 5112 if (rc) 5113 return (rc); 5114 /* XXX: magic numbers */ 5115 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, 5116 &v); 5117 end_synchronized_op(sc, 0); 5118 if (rc) 5119 return (rc); 5120 if (op == 0) 5121 v /= 256; 5122 5123 rc = sysctl_handle_int(oidp, &v, 0, req); 5124 return (rc); 5125 } 5126 5127 static int 5128 sysctl_noflowq(SYSCTL_HANDLER_ARGS) 5129 { 5130 struct port_info *pi = arg1; 5131 int rc, val; 5132 5133 val = pi->rsrv_noflowq; 5134 rc = sysctl_handle_int(oidp, &val, 0, req); 5135 if (rc != 0 || req->newptr == NULL) 5136 return (rc); 5137 5138 if ((val >= 1) && (pi->ntxq > 1)) 5139 pi->rsrv_noflowq = 1; 5140 else 5141 pi->rsrv_noflowq = 0; 5142 5143 return (rc); 5144 } 5145 5146 static int 5147 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 5148 { 5149 struct port_info *pi = arg1; 5150 struct adapter *sc = pi->adapter; 5151 int idx, rc, i; 5152 struct sge_rxq *rxq; 5153 #ifdef TCP_OFFLOAD 5154 struct sge_ofld_rxq *ofld_rxq; 5155 #endif 5156 uint8_t v; 5157 5158 idx = pi->tmr_idx; 5159 5160 rc = sysctl_handle_int(oidp, &idx, 0, req); 5161 if (rc != 0 || req->newptr == NULL) 5162 return (rc); 5163 5164 if (idx < 0 || idx >= SGE_NTIMERS) 5165 return (EINVAL); 5166 5167 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5168 "t4tmr"); 5169 if (rc) 5170 return (rc); 5171 5172 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 5173 for_each_rxq(pi, i, rxq) { 5174 #ifdef atomic_store_rel_8 5175 atomic_store_rel_8(&rxq->iq.intr_params, v); 5176 #else 5177 rxq->iq.intr_params = v; 5178 #endif 5179 } 5180 #ifdef TCP_OFFLOAD 5181 for_each_ofld_rxq(pi, i, ofld_rxq) { 5182 #ifdef atomic_store_rel_8 5183 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); 5184 #else 5185 ofld_rxq->iq.intr_params = v; 5186 #endif 5187 } 5188 #endif 5189 pi->tmr_idx = idx; 5190 5191 end_synchronized_op(sc, LOCK_HELD); 5192 return (0); 5193 } 5194 5195 static int 5196 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 5197 { 5198 struct port_info *pi = arg1; 5199 struct adapter *sc = pi->adapter; 5200 int idx, rc; 5201 5202 idx = pi->pktc_idx; 5203 5204 rc = sysctl_handle_int(oidp, &idx, 0, req); 5205 if (rc != 0 || req->newptr == NULL) 5206 return (rc); 5207 5208 if (idx < -1 || idx >= SGE_NCOUNTERS) 5209 return (EINVAL); 5210 5211 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5212 "t4pktc"); 5213 if (rc) 5214 return (rc); 5215 5216 if (pi->flags & PORT_INIT_DONE) 5217 rc = EBUSY; /* cannot be changed once the queues are created */ 5218 else 5219 pi->pktc_idx = idx; 5220 5221 end_synchronized_op(sc, LOCK_HELD); 5222 return (rc); 5223 } 5224 5225 static int 5226 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 5227 { 5228 struct port_info *pi = arg1; 5229 struct adapter *sc = pi->adapter; 5230 int qsize, rc; 5231 5232 qsize = pi->qsize_rxq; 5233 5234 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5235 if (rc != 0 || req->newptr == NULL) 5236 return (rc); 5237 5238 if (qsize < 128 || (qsize & 7)) 5239 return (EINVAL); 5240 5241 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5242 "t4rxqs"); 5243 if (rc) 5244 return (rc); 5245 5246 if (pi->flags & PORT_INIT_DONE) 5247 rc = EBUSY; /* cannot be changed once the queues are created */ 5248 else 5249 pi->qsize_rxq = qsize; 5250 5251 end_synchronized_op(sc, LOCK_HELD); 5252 return (rc); 5253 } 5254 5255 static int 5256 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 5257 { 5258 struct port_info *pi = arg1; 5259 struct adapter *sc = pi->adapter; 5260 int qsize, rc; 5261 5262 qsize = pi->qsize_txq; 5263 5264 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5265 if (rc != 0 || req->newptr == NULL) 5266 return (rc); 5267 5268 if (qsize < 128 || qsize > 65536) 5269 return (EINVAL); 5270 5271 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5272 "t4txqs"); 5273 if (rc) 5274 return (rc); 5275 5276 if (pi->flags & PORT_INIT_DONE) 5277 rc = EBUSY; /* cannot be changed once the queues are created */ 5278 else 5279 pi->qsize_txq = qsize; 5280 5281 end_synchronized_op(sc, LOCK_HELD); 5282 return (rc); 5283 } 5284 5285 static int 5286 sysctl_pause_settings(SYSCTL_HANDLER_ARGS) 5287 { 5288 struct port_info *pi = arg1; 5289 struct adapter *sc = pi->adapter; 5290 struct link_config *lc = &pi->link_cfg; 5291 int rc; 5292 5293 if (req->newptr == NULL) { 5294 struct sbuf *sb; 5295 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX"; 5296 5297 rc = sysctl_wire_old_buffer(req, 0); 5298 if (rc != 0) 5299 return(rc); 5300 5301 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5302 if (sb == NULL) 5303 return (ENOMEM); 5304 5305 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits); 5306 rc = sbuf_finish(sb); 5307 sbuf_delete(sb); 5308 } else { 5309 char s[2]; 5310 int n; 5311 5312 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX)); 5313 s[1] = 0; 5314 5315 rc = sysctl_handle_string(oidp, s, sizeof(s), req); 5316 if (rc != 0) 5317 return(rc); 5318 5319 if (s[1] != 0) 5320 return (EINVAL); 5321 if (s[0] < '0' || s[0] > '9') 5322 return (EINVAL); /* not a number */ 5323 n = s[0] - '0'; 5324 if (n & ~(PAUSE_TX | PAUSE_RX)) 5325 return (EINVAL); /* some other bit is set too */ 5326 5327 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE"); 5328 if (rc) 5329 return (rc); 5330 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) { 5331 int link_ok = lc->link_ok; 5332 5333 lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX); 5334 lc->requested_fc |= n; 5335 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc); 5336 lc->link_ok = link_ok; /* restore */ 5337 } 5338 end_synchronized_op(sc, 0); 5339 } 5340 5341 return (rc); 5342 } 5343 5344 static int 5345 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 5346 { 5347 struct adapter *sc = arg1; 5348 int reg = arg2; 5349 uint64_t val; 5350 5351 val = t4_read_reg64(sc, reg); 5352 5353 return (sysctl_handle_64(oidp, &val, 0, req)); 5354 } 5355 5356 static int 5357 sysctl_temperature(SYSCTL_HANDLER_ARGS) 5358 { 5359 struct adapter *sc = arg1; 5360 int rc, t; 5361 uint32_t param, val; 5362 5363 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); 5364 if (rc) 5365 return (rc); 5366 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 5367 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 5368 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); 5369 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 5370 end_synchronized_op(sc, 0); 5371 if (rc) 5372 return (rc); 5373 5374 /* unknown is returned as 0 but we display -1 in that case */ 5375 t = val == 0 ? -1 : val; 5376 5377 rc = sysctl_handle_int(oidp, &t, 0, req); 5378 return (rc); 5379 } 5380 5381 #ifdef SBUF_DRAIN 5382 static int 5383 sysctl_cctrl(SYSCTL_HANDLER_ARGS) 5384 { 5385 struct adapter *sc = arg1; 5386 struct sbuf *sb; 5387 int rc, i; 5388 uint16_t incr[NMTUS][NCCTRL_WIN]; 5389 static const char *dec_fac[] = { 5390 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 5391 "0.9375" 5392 }; 5393 5394 rc = sysctl_wire_old_buffer(req, 0); 5395 if (rc != 0) 5396 return (rc); 5397 5398 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5399 if (sb == NULL) 5400 return (ENOMEM); 5401 5402 t4_read_cong_tbl(sc, incr); 5403 5404 for (i = 0; i < NCCTRL_WIN; ++i) { 5405 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 5406 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 5407 incr[5][i], incr[6][i], incr[7][i]); 5408 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 5409 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 5410 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 5411 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 5412 } 5413 5414 rc = sbuf_finish(sb); 5415 sbuf_delete(sb); 5416 5417 return (rc); 5418 } 5419 5420 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 5421 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 5422 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 5423 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 5424 }; 5425 5426 static int 5427 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 5428 { 5429 struct adapter *sc = arg1; 5430 struct sbuf *sb; 5431 int rc, i, n, qid = arg2; 5432 uint32_t *buf, *p; 5433 char *qtype; 5434 u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5; 5435 5436 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 5437 ("%s: bad qid %d\n", __func__, qid)); 5438 5439 if (qid < CIM_NUM_IBQ) { 5440 /* inbound queue */ 5441 qtype = "IBQ"; 5442 n = 4 * CIM_IBQ_SIZE; 5443 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5444 rc = t4_read_cim_ibq(sc, qid, buf, n); 5445 } else { 5446 /* outbound queue */ 5447 qtype = "OBQ"; 5448 qid -= CIM_NUM_IBQ; 5449 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 5450 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5451 rc = t4_read_cim_obq(sc, qid, buf, n); 5452 } 5453 5454 if (rc < 0) { 5455 rc = -rc; 5456 goto done; 5457 } 5458 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 5459 5460 rc = sysctl_wire_old_buffer(req, 0); 5461 if (rc != 0) 5462 goto done; 5463 5464 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5465 if (sb == NULL) { 5466 rc = ENOMEM; 5467 goto done; 5468 } 5469 5470 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 5471 for (i = 0, p = buf; i < n; i += 16, p += 4) 5472 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 5473 p[2], p[3]); 5474 5475 rc = sbuf_finish(sb); 5476 sbuf_delete(sb); 5477 done: 5478 free(buf, M_CXGBE); 5479 return (rc); 5480 } 5481 5482 static int 5483 sysctl_cim_la(SYSCTL_HANDLER_ARGS) 5484 { 5485 struct adapter *sc = arg1; 5486 u_int cfg; 5487 struct sbuf *sb; 5488 uint32_t *buf, *p; 5489 int rc; 5490 5491 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 5492 if (rc != 0) 5493 return (rc); 5494 5495 rc = sysctl_wire_old_buffer(req, 0); 5496 if (rc != 0) 5497 return (rc); 5498 5499 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5500 if (sb == NULL) 5501 return (ENOMEM); 5502 5503 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 5504 M_ZERO | M_WAITOK); 5505 5506 rc = -t4_cim_read_la(sc, buf, NULL); 5507 if (rc != 0) 5508 goto done; 5509 5510 sbuf_printf(sb, "Status Data PC%s", 5511 cfg & F_UPDBGLACAPTPCONLY ? "" : 5512 " LS0Stat LS0Addr LS0Data"); 5513 5514 KASSERT((sc->params.cim_la_size & 7) == 0, 5515 ("%s: p will walk off the end of buf", __func__)); 5516 5517 for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) { 5518 if (cfg & F_UPDBGLACAPTPCONLY) { 5519 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 5520 p[6], p[7]); 5521 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 5522 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 5523 p[4] & 0xff, p[5] >> 8); 5524 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 5525 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5526 p[1] & 0xf, p[2] >> 4); 5527 } else { 5528 sbuf_printf(sb, 5529 "\n %02x %x%07x %x%07x %08x %08x " 5530 "%08x%08x%08x%08x", 5531 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5532 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 5533 p[6], p[7]); 5534 } 5535 } 5536 5537 rc = sbuf_finish(sb); 5538 sbuf_delete(sb); 5539 done: 5540 free(buf, M_CXGBE); 5541 return (rc); 5542 } 5543 5544 static int 5545 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) 5546 { 5547 struct adapter *sc = arg1; 5548 u_int i; 5549 struct sbuf *sb; 5550 uint32_t *buf, *p; 5551 int rc; 5552 5553 rc = sysctl_wire_old_buffer(req, 0); 5554 if (rc != 0) 5555 return (rc); 5556 5557 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5558 if (sb == NULL) 5559 return (ENOMEM); 5560 5561 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, 5562 M_ZERO | M_WAITOK); 5563 5564 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); 5565 p = buf; 5566 5567 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5568 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], 5569 p[1], p[0]); 5570 } 5571 5572 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); 5573 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5574 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", 5575 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, 5576 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, 5577 (p[1] >> 2) | ((p[2] & 3) << 30), 5578 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, 5579 p[0] & 1); 5580 } 5581 5582 rc = sbuf_finish(sb); 5583 sbuf_delete(sb); 5584 free(buf, M_CXGBE); 5585 return (rc); 5586 } 5587 5588 static int 5589 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) 5590 { 5591 struct adapter *sc = arg1; 5592 u_int i; 5593 struct sbuf *sb; 5594 uint32_t *buf, *p; 5595 int rc; 5596 5597 rc = sysctl_wire_old_buffer(req, 0); 5598 if (rc != 0) 5599 return (rc); 5600 5601 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5602 if (sb == NULL) 5603 return (ENOMEM); 5604 5605 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, 5606 M_ZERO | M_WAITOK); 5607 5608 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); 5609 p = buf; 5610 5611 sbuf_printf(sb, "Cntl ID DataBE Addr Data"); 5612 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5613 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", 5614 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, 5615 p[4], p[3], p[2], p[1], p[0]); 5616 } 5617 5618 sbuf_printf(sb, "\n\nCntl ID Data"); 5619 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5620 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", 5621 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); 5622 } 5623 5624 rc = sbuf_finish(sb); 5625 sbuf_delete(sb); 5626 free(buf, M_CXGBE); 5627 return (rc); 5628 } 5629 5630 static int 5631 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 5632 { 5633 struct adapter *sc = arg1; 5634 struct sbuf *sb; 5635 int rc, i; 5636 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5637 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5638 uint16_t thres[CIM_NUM_IBQ]; 5639 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 5640 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 5641 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 5642 5643 if (is_t4(sc)) { 5644 cim_num_obq = CIM_NUM_OBQ; 5645 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 5646 obq_rdaddr = A_UP_OBQ_0_REALADDR; 5647 } else { 5648 cim_num_obq = CIM_NUM_OBQ_T5; 5649 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 5650 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 5651 } 5652 nq = CIM_NUM_IBQ + cim_num_obq; 5653 5654 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 5655 if (rc == 0) 5656 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 5657 if (rc != 0) 5658 return (rc); 5659 5660 t4_read_cimq_cfg(sc, base, size, thres); 5661 5662 rc = sysctl_wire_old_buffer(req, 0); 5663 if (rc != 0) 5664 return (rc); 5665 5666 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5667 if (sb == NULL) 5668 return (ENOMEM); 5669 5670 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 5671 5672 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 5673 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 5674 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 5675 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5676 G_QUEREMFLITS(p[2]) * 16); 5677 for ( ; i < nq; i++, p += 4, wr += 2) 5678 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 5679 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 5680 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5681 G_QUEREMFLITS(p[2]) * 16); 5682 5683 rc = sbuf_finish(sb); 5684 sbuf_delete(sb); 5685 5686 return (rc); 5687 } 5688 5689 static int 5690 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 5691 { 5692 struct adapter *sc = arg1; 5693 struct sbuf *sb; 5694 int rc; 5695 struct tp_cpl_stats stats; 5696 5697 rc = sysctl_wire_old_buffer(req, 0); 5698 if (rc != 0) 5699 return (rc); 5700 5701 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5702 if (sb == NULL) 5703 return (ENOMEM); 5704 5705 t4_tp_get_cpl_stats(sc, &stats); 5706 5707 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 5708 "channel 3\n"); 5709 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 5710 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 5711 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 5712 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 5713 5714 rc = sbuf_finish(sb); 5715 sbuf_delete(sb); 5716 5717 return (rc); 5718 } 5719 5720 static int 5721 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 5722 { 5723 struct adapter *sc = arg1; 5724 struct sbuf *sb; 5725 int rc; 5726 struct tp_usm_stats stats; 5727 5728 rc = sysctl_wire_old_buffer(req, 0); 5729 if (rc != 0) 5730 return(rc); 5731 5732 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5733 if (sb == NULL) 5734 return (ENOMEM); 5735 5736 t4_get_usm_stats(sc, &stats); 5737 5738 sbuf_printf(sb, "Frames: %u\n", stats.frames); 5739 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 5740 sbuf_printf(sb, "Drops: %u", stats.drops); 5741 5742 rc = sbuf_finish(sb); 5743 sbuf_delete(sb); 5744 5745 return (rc); 5746 } 5747 5748 const char *devlog_level_strings[] = { 5749 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 5750 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 5751 [FW_DEVLOG_LEVEL_ERR] = "ERR", 5752 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 5753 [FW_DEVLOG_LEVEL_INFO] = "INFO", 5754 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 5755 }; 5756 5757 const char *devlog_facility_strings[] = { 5758 [FW_DEVLOG_FACILITY_CORE] = "CORE", 5759 [FW_DEVLOG_FACILITY_CF] = "CF", 5760 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 5761 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 5762 [FW_DEVLOG_FACILITY_RES] = "RES", 5763 [FW_DEVLOG_FACILITY_HW] = "HW", 5764 [FW_DEVLOG_FACILITY_FLR] = "FLR", 5765 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 5766 [FW_DEVLOG_FACILITY_PHY] = "PHY", 5767 [FW_DEVLOG_FACILITY_MAC] = "MAC", 5768 [FW_DEVLOG_FACILITY_PORT] = "PORT", 5769 [FW_DEVLOG_FACILITY_VI] = "VI", 5770 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 5771 [FW_DEVLOG_FACILITY_ACL] = "ACL", 5772 [FW_DEVLOG_FACILITY_TM] = "TM", 5773 [FW_DEVLOG_FACILITY_QFC] = "QFC", 5774 [FW_DEVLOG_FACILITY_DCB] = "DCB", 5775 [FW_DEVLOG_FACILITY_ETH] = "ETH", 5776 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 5777 [FW_DEVLOG_FACILITY_RI] = "RI", 5778 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 5779 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 5780 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 5781 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 5782 }; 5783 5784 static int 5785 sysctl_devlog(SYSCTL_HANDLER_ARGS) 5786 { 5787 struct adapter *sc = arg1; 5788 struct devlog_params *dparams = &sc->params.devlog; 5789 struct fw_devlog_e *buf, *e; 5790 int i, j, rc, nentries, first = 0, m; 5791 struct sbuf *sb; 5792 uint64_t ftstamp = UINT64_MAX; 5793 5794 if (dparams->start == 0) { 5795 dparams->memtype = FW_MEMTYPE_EDC0; 5796 dparams->start = 0x84000; 5797 dparams->size = 32768; 5798 } 5799 5800 nentries = dparams->size / sizeof(struct fw_devlog_e); 5801 5802 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 5803 if (buf == NULL) 5804 return (ENOMEM); 5805 5806 m = fwmtype_to_hwmtype(dparams->memtype); 5807 rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf); 5808 if (rc != 0) 5809 goto done; 5810 5811 for (i = 0; i < nentries; i++) { 5812 e = &buf[i]; 5813 5814 if (e->timestamp == 0) 5815 break; /* end */ 5816 5817 e->timestamp = be64toh(e->timestamp); 5818 e->seqno = be32toh(e->seqno); 5819 for (j = 0; j < 8; j++) 5820 e->params[j] = be32toh(e->params[j]); 5821 5822 if (e->timestamp < ftstamp) { 5823 ftstamp = e->timestamp; 5824 first = i; 5825 } 5826 } 5827 5828 if (buf[first].timestamp == 0) 5829 goto done; /* nothing in the log */ 5830 5831 rc = sysctl_wire_old_buffer(req, 0); 5832 if (rc != 0) 5833 goto done; 5834 5835 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5836 if (sb == NULL) { 5837 rc = ENOMEM; 5838 goto done; 5839 } 5840 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 5841 "Seq#", "Tstamp", "Level", "Facility", "Message"); 5842 5843 i = first; 5844 do { 5845 e = &buf[i]; 5846 if (e->timestamp == 0) 5847 break; /* end */ 5848 5849 sbuf_printf(sb, "%10d %15ju %8s %8s ", 5850 e->seqno, e->timestamp, 5851 (e->level < nitems(devlog_level_strings) ? 5852 devlog_level_strings[e->level] : "UNKNOWN"), 5853 (e->facility < nitems(devlog_facility_strings) ? 5854 devlog_facility_strings[e->facility] : "UNKNOWN")); 5855 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 5856 e->params[2], e->params[3], e->params[4], 5857 e->params[5], e->params[6], e->params[7]); 5858 5859 if (++i == nentries) 5860 i = 0; 5861 } while (i != first); 5862 5863 rc = sbuf_finish(sb); 5864 sbuf_delete(sb); 5865 done: 5866 free(buf, M_CXGBE); 5867 return (rc); 5868 } 5869 5870 static int 5871 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 5872 { 5873 struct adapter *sc = arg1; 5874 struct sbuf *sb; 5875 int rc; 5876 struct tp_fcoe_stats stats[4]; 5877 5878 rc = sysctl_wire_old_buffer(req, 0); 5879 if (rc != 0) 5880 return (rc); 5881 5882 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5883 if (sb == NULL) 5884 return (ENOMEM); 5885 5886 t4_get_fcoe_stats(sc, 0, &stats[0]); 5887 t4_get_fcoe_stats(sc, 1, &stats[1]); 5888 t4_get_fcoe_stats(sc, 2, &stats[2]); 5889 t4_get_fcoe_stats(sc, 3, &stats[3]); 5890 5891 sbuf_printf(sb, " channel 0 channel 1 " 5892 "channel 2 channel 3\n"); 5893 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 5894 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 5895 stats[3].octetsDDP); 5896 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 5897 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 5898 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 5899 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 5900 stats[3].framesDrop); 5901 5902 rc = sbuf_finish(sb); 5903 sbuf_delete(sb); 5904 5905 return (rc); 5906 } 5907 5908 static int 5909 sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 5910 { 5911 struct adapter *sc = arg1; 5912 struct sbuf *sb; 5913 int rc, i; 5914 unsigned int map, kbps, ipg, mode; 5915 unsigned int pace_tab[NTX_SCHED]; 5916 5917 rc = sysctl_wire_old_buffer(req, 0); 5918 if (rc != 0) 5919 return (rc); 5920 5921 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5922 if (sb == NULL) 5923 return (ENOMEM); 5924 5925 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 5926 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 5927 t4_read_pace_tbl(sc, pace_tab); 5928 5929 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 5930 "Class IPG (0.1 ns) Flow IPG (us)"); 5931 5932 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 5933 t4_get_tx_sched(sc, i, &kbps, &ipg); 5934 sbuf_printf(sb, "\n %u %-5s %u ", i, 5935 (mode & (1 << i)) ? "flow" : "class", map & 3); 5936 if (kbps) 5937 sbuf_printf(sb, "%9u ", kbps); 5938 else 5939 sbuf_printf(sb, " disabled "); 5940 5941 if (ipg) 5942 sbuf_printf(sb, "%13u ", ipg); 5943 else 5944 sbuf_printf(sb, " disabled "); 5945 5946 if (pace_tab[i]) 5947 sbuf_printf(sb, "%10u", pace_tab[i]); 5948 else 5949 sbuf_printf(sb, " disabled"); 5950 } 5951 5952 rc = sbuf_finish(sb); 5953 sbuf_delete(sb); 5954 5955 return (rc); 5956 } 5957 5958 static int 5959 sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 5960 { 5961 struct adapter *sc = arg1; 5962 struct sbuf *sb; 5963 int rc, i, j; 5964 uint64_t *p0, *p1; 5965 struct lb_port_stats s[2]; 5966 static const char *stat_name[] = { 5967 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 5968 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 5969 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 5970 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 5971 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 5972 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 5973 "BG2FramesTrunc:", "BG3FramesTrunc:" 5974 }; 5975 5976 rc = sysctl_wire_old_buffer(req, 0); 5977 if (rc != 0) 5978 return (rc); 5979 5980 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5981 if (sb == NULL) 5982 return (ENOMEM); 5983 5984 memset(s, 0, sizeof(s)); 5985 5986 for (i = 0; i < 4; i += 2) { 5987 t4_get_lb_stats(sc, i, &s[0]); 5988 t4_get_lb_stats(sc, i + 1, &s[1]); 5989 5990 p0 = &s[0].octets; 5991 p1 = &s[1].octets; 5992 sbuf_printf(sb, "%s Loopback %u" 5993 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 5994 5995 for (j = 0; j < nitems(stat_name); j++) 5996 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 5997 *p0++, *p1++); 5998 } 5999 6000 rc = sbuf_finish(sb); 6001 sbuf_delete(sb); 6002 6003 return (rc); 6004 } 6005 6006 static int 6007 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) 6008 { 6009 int rc = 0; 6010 struct port_info *pi = arg1; 6011 struct sbuf *sb; 6012 static const char *linkdnreasons[] = { 6013 "non-specific", "remote fault", "autoneg failed", "reserved3", 6014 "PHY overheated", "unknown", "rx los", "reserved7" 6015 }; 6016 6017 rc = sysctl_wire_old_buffer(req, 0); 6018 if (rc != 0) 6019 return(rc); 6020 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); 6021 if (sb == NULL) 6022 return (ENOMEM); 6023 6024 if (pi->linkdnrc < 0) 6025 sbuf_printf(sb, "n/a"); 6026 else if (pi->linkdnrc < nitems(linkdnreasons)) 6027 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]); 6028 else 6029 sbuf_printf(sb, "%d", pi->linkdnrc); 6030 6031 rc = sbuf_finish(sb); 6032 sbuf_delete(sb); 6033 6034 return (rc); 6035 } 6036 6037 struct mem_desc { 6038 unsigned int base; 6039 unsigned int limit; 6040 unsigned int idx; 6041 }; 6042 6043 static int 6044 mem_desc_cmp(const void *a, const void *b) 6045 { 6046 return ((const struct mem_desc *)a)->base - 6047 ((const struct mem_desc *)b)->base; 6048 } 6049 6050 static void 6051 mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 6052 unsigned int to) 6053 { 6054 unsigned int size; 6055 6056 size = to - from + 1; 6057 if (size == 0) 6058 return; 6059 6060 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 6061 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 6062 } 6063 6064 static int 6065 sysctl_meminfo(SYSCTL_HANDLER_ARGS) 6066 { 6067 struct adapter *sc = arg1; 6068 struct sbuf *sb; 6069 int rc, i, n; 6070 uint32_t lo, hi, used, alloc; 6071 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 6072 static const char *region[] = { 6073 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 6074 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 6075 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 6076 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 6077 "RQUDP region:", "PBL region:", "TXPBL region:", 6078 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 6079 "On-chip queues:" 6080 }; 6081 struct mem_desc avail[4]; 6082 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 6083 struct mem_desc *md = mem; 6084 6085 rc = sysctl_wire_old_buffer(req, 0); 6086 if (rc != 0) 6087 return (rc); 6088 6089 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6090 if (sb == NULL) 6091 return (ENOMEM); 6092 6093 for (i = 0; i < nitems(mem); i++) { 6094 mem[i].limit = 0; 6095 mem[i].idx = i; 6096 } 6097 6098 /* Find and sort the populated memory ranges */ 6099 i = 0; 6100 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 6101 if (lo & F_EDRAM0_ENABLE) { 6102 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 6103 avail[i].base = G_EDRAM0_BASE(hi) << 20; 6104 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 6105 avail[i].idx = 0; 6106 i++; 6107 } 6108 if (lo & F_EDRAM1_ENABLE) { 6109 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 6110 avail[i].base = G_EDRAM1_BASE(hi) << 20; 6111 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 6112 avail[i].idx = 1; 6113 i++; 6114 } 6115 if (lo & F_EXT_MEM_ENABLE) { 6116 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 6117 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 6118 avail[i].limit = avail[i].base + 6119 (G_EXT_MEM_SIZE(hi) << 20); 6120 avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */ 6121 i++; 6122 } 6123 if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) { 6124 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 6125 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 6126 avail[i].limit = avail[i].base + 6127 (G_EXT_MEM1_SIZE(hi) << 20); 6128 avail[i].idx = 4; 6129 i++; 6130 } 6131 if (!i) /* no memory available */ 6132 return 0; 6133 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 6134 6135 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 6136 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 6137 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 6138 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 6139 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 6140 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 6141 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 6142 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 6143 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 6144 6145 /* the next few have explicit upper bounds */ 6146 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 6147 md->limit = md->base - 1 + 6148 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 6149 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 6150 md++; 6151 6152 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 6153 md->limit = md->base - 1 + 6154 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 6155 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 6156 md++; 6157 6158 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6159 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 6160 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 6161 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 6162 } else { 6163 md->base = 0; 6164 md->idx = nitems(region); /* hide it */ 6165 } 6166 md++; 6167 6168 #define ulp_region(reg) \ 6169 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 6170 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 6171 6172 ulp_region(RX_ISCSI); 6173 ulp_region(RX_TDDP); 6174 ulp_region(TX_TPT); 6175 ulp_region(RX_STAG); 6176 ulp_region(RX_RQ); 6177 ulp_region(RX_RQUDP); 6178 ulp_region(RX_PBL); 6179 ulp_region(TX_PBL); 6180 #undef ulp_region 6181 6182 md->base = 0; 6183 md->idx = nitems(region); 6184 if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) { 6185 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); 6186 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc, 6187 A_SGE_DBVFIFO_SIZE))) << 2) - 1; 6188 } 6189 md++; 6190 6191 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 6192 md->limit = md->base + sc->tids.ntids - 1; 6193 md++; 6194 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 6195 md->limit = md->base + sc->tids.ntids - 1; 6196 md++; 6197 6198 md->base = sc->vres.ocq.start; 6199 if (sc->vres.ocq.size) 6200 md->limit = md->base + sc->vres.ocq.size - 1; 6201 else 6202 md->idx = nitems(region); /* hide it */ 6203 md++; 6204 6205 /* add any address-space holes, there can be up to 3 */ 6206 for (n = 0; n < i - 1; n++) 6207 if (avail[n].limit < avail[n + 1].base) 6208 (md++)->base = avail[n].limit; 6209 if (avail[n].limit) 6210 (md++)->base = avail[n].limit; 6211 6212 n = md - mem; 6213 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 6214 6215 for (lo = 0; lo < i; lo++) 6216 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 6217 avail[lo].limit - 1); 6218 6219 sbuf_printf(sb, "\n"); 6220 for (i = 0; i < n; i++) { 6221 if (mem[i].idx >= nitems(region)) 6222 continue; /* skip holes */ 6223 if (!mem[i].limit) 6224 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 6225 mem_region_show(sb, region[mem[i].idx], mem[i].base, 6226 mem[i].limit); 6227 } 6228 6229 sbuf_printf(sb, "\n"); 6230 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 6231 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 6232 mem_region_show(sb, "uP RAM:", lo, hi); 6233 6234 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 6235 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 6236 mem_region_show(sb, "uP Extmem2:", lo, hi); 6237 6238 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 6239 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 6240 G_PMRXMAXPAGE(lo), 6241 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 6242 (lo & F_PMRXNUMCHN) ? 2 : 1); 6243 6244 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 6245 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 6246 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 6247 G_PMTXMAXPAGE(lo), 6248 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 6249 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 6250 sbuf_printf(sb, "%u p-structs\n", 6251 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 6252 6253 for (i = 0; i < 4; i++) { 6254 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 6255 if (is_t4(sc)) { 6256 used = G_USED(lo); 6257 alloc = G_ALLOC(lo); 6258 } else { 6259 used = G_T5_USED(lo); 6260 alloc = G_T5_ALLOC(lo); 6261 } 6262 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 6263 i, used, alloc); 6264 } 6265 for (i = 0; i < 4; i++) { 6266 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 6267 if (is_t4(sc)) { 6268 used = G_USED(lo); 6269 alloc = G_ALLOC(lo); 6270 } else { 6271 used = G_T5_USED(lo); 6272 alloc = G_T5_ALLOC(lo); 6273 } 6274 sbuf_printf(sb, 6275 "\nLoopback %d using %u pages out of %u allocated", 6276 i, used, alloc); 6277 } 6278 6279 rc = sbuf_finish(sb); 6280 sbuf_delete(sb); 6281 6282 return (rc); 6283 } 6284 6285 static inline void 6286 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) 6287 { 6288 *mask = x | y; 6289 y = htobe64(y); 6290 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); 6291 } 6292 6293 static int 6294 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) 6295 { 6296 struct adapter *sc = arg1; 6297 struct sbuf *sb; 6298 int rc, i, n; 6299 6300 rc = sysctl_wire_old_buffer(req, 0); 6301 if (rc != 0) 6302 return (rc); 6303 6304 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6305 if (sb == NULL) 6306 return (ENOMEM); 6307 6308 sbuf_printf(sb, 6309 "Idx Ethernet address Mask Vld Ports PF" 6310 " VF Replication P0 P1 P2 P3 ML"); 6311 n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES : 6312 NUM_MPS_T5_CLS_SRAM_L_INSTANCES; 6313 for (i = 0; i < n; i++) { 6314 uint64_t tcamx, tcamy, mask; 6315 uint32_t cls_lo, cls_hi; 6316 uint8_t addr[ETHER_ADDR_LEN]; 6317 6318 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); 6319 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); 6320 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 6321 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 6322 6323 if (tcamx & tcamy) 6324 continue; 6325 6326 tcamxy2valmask(tcamx, tcamy, addr, &mask); 6327 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" 6328 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], 6329 addr[3], addr[4], addr[5], (uintmax_t)mask, 6330 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', 6331 G_PORTMAP(cls_hi), G_PF(cls_lo), 6332 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); 6333 6334 if (cls_lo & F_REPLICATE) { 6335 struct fw_ldst_cmd ldst_cmd; 6336 6337 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 6338 ldst_cmd.op_to_addrspace = 6339 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 6340 F_FW_CMD_REQUEST | F_FW_CMD_READ | 6341 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 6342 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 6343 ldst_cmd.u.mps.rplc.fid_idx = 6344 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 6345 V_FW_LDST_CMD_IDX(i)); 6346 6347 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 6348 "t4mps"); 6349 if (rc) 6350 break; 6351 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 6352 sizeof(ldst_cmd), &ldst_cmd); 6353 end_synchronized_op(sc, 0); 6354 6355 if (rc != 0) { 6356 sbuf_printf(sb, 6357 " ------------ error %3u ------------", rc); 6358 rc = 0; 6359 } else { 6360 sbuf_printf(sb, " %08x %08x %08x %08x", 6361 be32toh(ldst_cmd.u.mps.rplc.rplc127_96), 6362 be32toh(ldst_cmd.u.mps.rplc.rplc95_64), 6363 be32toh(ldst_cmd.u.mps.rplc.rplc63_32), 6364 be32toh(ldst_cmd.u.mps.rplc.rplc31_0)); 6365 } 6366 } else 6367 sbuf_printf(sb, "%36s", ""); 6368 6369 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), 6370 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), 6371 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); 6372 } 6373 6374 if (rc) 6375 (void) sbuf_finish(sb); 6376 else 6377 rc = sbuf_finish(sb); 6378 sbuf_delete(sb); 6379 6380 return (rc); 6381 } 6382 6383 static int 6384 sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 6385 { 6386 struct adapter *sc = arg1; 6387 struct sbuf *sb; 6388 int rc; 6389 uint16_t mtus[NMTUS]; 6390 6391 rc = sysctl_wire_old_buffer(req, 0); 6392 if (rc != 0) 6393 return (rc); 6394 6395 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6396 if (sb == NULL) 6397 return (ENOMEM); 6398 6399 t4_read_mtu_tbl(sc, mtus, NULL); 6400 6401 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 6402 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 6403 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 6404 mtus[14], mtus[15]); 6405 6406 rc = sbuf_finish(sb); 6407 sbuf_delete(sb); 6408 6409 return (rc); 6410 } 6411 6412 static int 6413 sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 6414 { 6415 struct adapter *sc = arg1; 6416 struct sbuf *sb; 6417 int rc, i; 6418 uint32_t cnt[PM_NSTATS]; 6419 uint64_t cyc[PM_NSTATS]; 6420 static const char *rx_stats[] = { 6421 "Read:", "Write bypass:", "Write mem:", "Flush:" 6422 }; 6423 static const char *tx_stats[] = { 6424 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:" 6425 }; 6426 6427 rc = sysctl_wire_old_buffer(req, 0); 6428 if (rc != 0) 6429 return (rc); 6430 6431 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6432 if (sb == NULL) 6433 return (ENOMEM); 6434 6435 t4_pmtx_get_stats(sc, cnt, cyc); 6436 sbuf_printf(sb, " Tx pcmds Tx bytes"); 6437 for (i = 0; i < ARRAY_SIZE(tx_stats); i++) 6438 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i], 6439 cyc[i]); 6440 6441 t4_pmrx_get_stats(sc, cnt, cyc); 6442 sbuf_printf(sb, "\n Rx pcmds Rx bytes"); 6443 for (i = 0; i < ARRAY_SIZE(rx_stats); i++) 6444 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i], 6445 cyc[i]); 6446 6447 rc = sbuf_finish(sb); 6448 sbuf_delete(sb); 6449 6450 return (rc); 6451 } 6452 6453 static int 6454 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 6455 { 6456 struct adapter *sc = arg1; 6457 struct sbuf *sb; 6458 int rc; 6459 struct tp_rdma_stats stats; 6460 6461 rc = sysctl_wire_old_buffer(req, 0); 6462 if (rc != 0) 6463 return (rc); 6464 6465 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6466 if (sb == NULL) 6467 return (ENOMEM); 6468 6469 t4_tp_get_rdma_stats(sc, &stats); 6470 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 6471 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 6472 6473 rc = sbuf_finish(sb); 6474 sbuf_delete(sb); 6475 6476 return (rc); 6477 } 6478 6479 static int 6480 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 6481 { 6482 struct adapter *sc = arg1; 6483 struct sbuf *sb; 6484 int rc; 6485 struct tp_tcp_stats v4, v6; 6486 6487 rc = sysctl_wire_old_buffer(req, 0); 6488 if (rc != 0) 6489 return (rc); 6490 6491 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6492 if (sb == NULL) 6493 return (ENOMEM); 6494 6495 t4_tp_get_tcp_stats(sc, &v4, &v6); 6496 sbuf_printf(sb, 6497 " IP IPv6\n"); 6498 sbuf_printf(sb, "OutRsts: %20u %20u\n", 6499 v4.tcpOutRsts, v6.tcpOutRsts); 6500 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 6501 v4.tcpInSegs, v6.tcpInSegs); 6502 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 6503 v4.tcpOutSegs, v6.tcpOutSegs); 6504 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 6505 v4.tcpRetransSegs, v6.tcpRetransSegs); 6506 6507 rc = sbuf_finish(sb); 6508 sbuf_delete(sb); 6509 6510 return (rc); 6511 } 6512 6513 static int 6514 sysctl_tids(SYSCTL_HANDLER_ARGS) 6515 { 6516 struct adapter *sc = arg1; 6517 struct sbuf *sb; 6518 int rc; 6519 struct tid_info *t = &sc->tids; 6520 6521 rc = sysctl_wire_old_buffer(req, 0); 6522 if (rc != 0) 6523 return (rc); 6524 6525 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6526 if (sb == NULL) 6527 return (ENOMEM); 6528 6529 if (t->natids) { 6530 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 6531 t->atids_in_use); 6532 } 6533 6534 if (t->ntids) { 6535 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6536 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 6537 6538 if (b) { 6539 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 6540 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6541 t->ntids - 1); 6542 } else { 6543 sbuf_printf(sb, "TID range: %u-%u", 6544 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6545 t->ntids - 1); 6546 } 6547 } else 6548 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 6549 sbuf_printf(sb, ", in use: %u\n", 6550 atomic_load_acq_int(&t->tids_in_use)); 6551 } 6552 6553 if (t->nstids) { 6554 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 6555 t->stid_base + t->nstids - 1, t->stids_in_use); 6556 } 6557 6558 if (t->nftids) { 6559 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 6560 t->ftid_base + t->nftids - 1); 6561 } 6562 6563 if (t->netids) { 6564 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base, 6565 t->etid_base + t->netids - 1); 6566 } 6567 6568 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 6569 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 6570 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 6571 6572 rc = sbuf_finish(sb); 6573 sbuf_delete(sb); 6574 6575 return (rc); 6576 } 6577 6578 static int 6579 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 6580 { 6581 struct adapter *sc = arg1; 6582 struct sbuf *sb; 6583 int rc; 6584 struct tp_err_stats stats; 6585 6586 rc = sysctl_wire_old_buffer(req, 0); 6587 if (rc != 0) 6588 return (rc); 6589 6590 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6591 if (sb == NULL) 6592 return (ENOMEM); 6593 6594 t4_tp_get_err_stats(sc, &stats); 6595 6596 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6597 "channel 3\n"); 6598 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 6599 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 6600 stats.macInErrs[3]); 6601 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 6602 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 6603 stats.hdrInErrs[3]); 6604 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 6605 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 6606 stats.tcpInErrs[3]); 6607 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 6608 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 6609 stats.tcp6InErrs[3]); 6610 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 6611 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 6612 stats.tnlCongDrops[3]); 6613 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 6614 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 6615 stats.tnlTxDrops[3]); 6616 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 6617 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 6618 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 6619 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 6620 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 6621 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 6622 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 6623 stats.ofldNoNeigh, stats.ofldCongDefer); 6624 6625 rc = sbuf_finish(sb); 6626 sbuf_delete(sb); 6627 6628 return (rc); 6629 } 6630 6631 struct field_desc { 6632 const char *name; 6633 u_int start; 6634 u_int width; 6635 }; 6636 6637 static void 6638 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) 6639 { 6640 char buf[32]; 6641 int line_size = 0; 6642 6643 while (f->name) { 6644 uint64_t mask = (1ULL << f->width) - 1; 6645 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, 6646 ((uintmax_t)v >> f->start) & mask); 6647 6648 if (line_size + len >= 79) { 6649 line_size = 8; 6650 sbuf_printf(sb, "\n "); 6651 } 6652 sbuf_printf(sb, "%s ", buf); 6653 line_size += len + 1; 6654 f++; 6655 } 6656 sbuf_printf(sb, "\n"); 6657 } 6658 6659 static struct field_desc tp_la0[] = { 6660 { "RcfOpCodeOut", 60, 4 }, 6661 { "State", 56, 4 }, 6662 { "WcfState", 52, 4 }, 6663 { "RcfOpcSrcOut", 50, 2 }, 6664 { "CRxError", 49, 1 }, 6665 { "ERxError", 48, 1 }, 6666 { "SanityFailed", 47, 1 }, 6667 { "SpuriousMsg", 46, 1 }, 6668 { "FlushInputMsg", 45, 1 }, 6669 { "FlushInputCpl", 44, 1 }, 6670 { "RssUpBit", 43, 1 }, 6671 { "RssFilterHit", 42, 1 }, 6672 { "Tid", 32, 10 }, 6673 { "InitTcb", 31, 1 }, 6674 { "LineNumber", 24, 7 }, 6675 { "Emsg", 23, 1 }, 6676 { "EdataOut", 22, 1 }, 6677 { "Cmsg", 21, 1 }, 6678 { "CdataOut", 20, 1 }, 6679 { "EreadPdu", 19, 1 }, 6680 { "CreadPdu", 18, 1 }, 6681 { "TunnelPkt", 17, 1 }, 6682 { "RcfPeerFin", 16, 1 }, 6683 { "RcfReasonOut", 12, 4 }, 6684 { "TxCchannel", 10, 2 }, 6685 { "RcfTxChannel", 8, 2 }, 6686 { "RxEchannel", 6, 2 }, 6687 { "RcfRxChannel", 5, 1 }, 6688 { "RcfDataOutSrdy", 4, 1 }, 6689 { "RxDvld", 3, 1 }, 6690 { "RxOoDvld", 2, 1 }, 6691 { "RxCongestion", 1, 1 }, 6692 { "TxCongestion", 0, 1 }, 6693 { NULL } 6694 }; 6695 6696 static struct field_desc tp_la1[] = { 6697 { "CplCmdIn", 56, 8 }, 6698 { "CplCmdOut", 48, 8 }, 6699 { "ESynOut", 47, 1 }, 6700 { "EAckOut", 46, 1 }, 6701 { "EFinOut", 45, 1 }, 6702 { "ERstOut", 44, 1 }, 6703 { "SynIn", 43, 1 }, 6704 { "AckIn", 42, 1 }, 6705 { "FinIn", 41, 1 }, 6706 { "RstIn", 40, 1 }, 6707 { "DataIn", 39, 1 }, 6708 { "DataInVld", 38, 1 }, 6709 { "PadIn", 37, 1 }, 6710 { "RxBufEmpty", 36, 1 }, 6711 { "RxDdp", 35, 1 }, 6712 { "RxFbCongestion", 34, 1 }, 6713 { "TxFbCongestion", 33, 1 }, 6714 { "TxPktSumSrdy", 32, 1 }, 6715 { "RcfUlpType", 28, 4 }, 6716 { "Eread", 27, 1 }, 6717 { "Ebypass", 26, 1 }, 6718 { "Esave", 25, 1 }, 6719 { "Static0", 24, 1 }, 6720 { "Cread", 23, 1 }, 6721 { "Cbypass", 22, 1 }, 6722 { "Csave", 21, 1 }, 6723 { "CPktOut", 20, 1 }, 6724 { "RxPagePoolFull", 18, 2 }, 6725 { "RxLpbkPkt", 17, 1 }, 6726 { "TxLpbkPkt", 16, 1 }, 6727 { "RxVfValid", 15, 1 }, 6728 { "SynLearned", 14, 1 }, 6729 { "SetDelEntry", 13, 1 }, 6730 { "SetInvEntry", 12, 1 }, 6731 { "CpcmdDvld", 11, 1 }, 6732 { "CpcmdSave", 10, 1 }, 6733 { "RxPstructsFull", 8, 2 }, 6734 { "EpcmdDvld", 7, 1 }, 6735 { "EpcmdFlush", 6, 1 }, 6736 { "EpcmdTrimPrefix", 5, 1 }, 6737 { "EpcmdTrimPostfix", 4, 1 }, 6738 { "ERssIp4Pkt", 3, 1 }, 6739 { "ERssIp6Pkt", 2, 1 }, 6740 { "ERssTcpUdpPkt", 1, 1 }, 6741 { "ERssFceFipPkt", 0, 1 }, 6742 { NULL } 6743 }; 6744 6745 static struct field_desc tp_la2[] = { 6746 { "CplCmdIn", 56, 8 }, 6747 { "MpsVfVld", 55, 1 }, 6748 { "MpsPf", 52, 3 }, 6749 { "MpsVf", 44, 8 }, 6750 { "SynIn", 43, 1 }, 6751 { "AckIn", 42, 1 }, 6752 { "FinIn", 41, 1 }, 6753 { "RstIn", 40, 1 }, 6754 { "DataIn", 39, 1 }, 6755 { "DataInVld", 38, 1 }, 6756 { "PadIn", 37, 1 }, 6757 { "RxBufEmpty", 36, 1 }, 6758 { "RxDdp", 35, 1 }, 6759 { "RxFbCongestion", 34, 1 }, 6760 { "TxFbCongestion", 33, 1 }, 6761 { "TxPktSumSrdy", 32, 1 }, 6762 { "RcfUlpType", 28, 4 }, 6763 { "Eread", 27, 1 }, 6764 { "Ebypass", 26, 1 }, 6765 { "Esave", 25, 1 }, 6766 { "Static0", 24, 1 }, 6767 { "Cread", 23, 1 }, 6768 { "Cbypass", 22, 1 }, 6769 { "Csave", 21, 1 }, 6770 { "CPktOut", 20, 1 }, 6771 { "RxPagePoolFull", 18, 2 }, 6772 { "RxLpbkPkt", 17, 1 }, 6773 { "TxLpbkPkt", 16, 1 }, 6774 { "RxVfValid", 15, 1 }, 6775 { "SynLearned", 14, 1 }, 6776 { "SetDelEntry", 13, 1 }, 6777 { "SetInvEntry", 12, 1 }, 6778 { "CpcmdDvld", 11, 1 }, 6779 { "CpcmdSave", 10, 1 }, 6780 { "RxPstructsFull", 8, 2 }, 6781 { "EpcmdDvld", 7, 1 }, 6782 { "EpcmdFlush", 6, 1 }, 6783 { "EpcmdTrimPrefix", 5, 1 }, 6784 { "EpcmdTrimPostfix", 4, 1 }, 6785 { "ERssIp4Pkt", 3, 1 }, 6786 { "ERssIp6Pkt", 2, 1 }, 6787 { "ERssTcpUdpPkt", 1, 1 }, 6788 { "ERssFceFipPkt", 0, 1 }, 6789 { NULL } 6790 }; 6791 6792 static void 6793 tp_la_show(struct sbuf *sb, uint64_t *p, int idx) 6794 { 6795 6796 field_desc_show(sb, *p, tp_la0); 6797 } 6798 6799 static void 6800 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) 6801 { 6802 6803 if (idx) 6804 sbuf_printf(sb, "\n"); 6805 field_desc_show(sb, p[0], tp_la0); 6806 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6807 field_desc_show(sb, p[1], tp_la0); 6808 } 6809 6810 static void 6811 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) 6812 { 6813 6814 if (idx) 6815 sbuf_printf(sb, "\n"); 6816 field_desc_show(sb, p[0], tp_la0); 6817 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6818 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); 6819 } 6820 6821 static int 6822 sysctl_tp_la(SYSCTL_HANDLER_ARGS) 6823 { 6824 struct adapter *sc = arg1; 6825 struct sbuf *sb; 6826 uint64_t *buf, *p; 6827 int rc; 6828 u_int i, inc; 6829 void (*show_func)(struct sbuf *, uint64_t *, int); 6830 6831 rc = sysctl_wire_old_buffer(req, 0); 6832 if (rc != 0) 6833 return (rc); 6834 6835 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6836 if (sb == NULL) 6837 return (ENOMEM); 6838 6839 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); 6840 6841 t4_tp_read_la(sc, buf, NULL); 6842 p = buf; 6843 6844 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { 6845 case 2: 6846 inc = 2; 6847 show_func = tp_la_show2; 6848 break; 6849 case 3: 6850 inc = 2; 6851 show_func = tp_la_show3; 6852 break; 6853 default: 6854 inc = 1; 6855 show_func = tp_la_show; 6856 } 6857 6858 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) 6859 (*show_func)(sb, p, i); 6860 6861 rc = sbuf_finish(sb); 6862 sbuf_delete(sb); 6863 free(buf, M_CXGBE); 6864 return (rc); 6865 } 6866 6867 static int 6868 sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 6869 { 6870 struct adapter *sc = arg1; 6871 struct sbuf *sb; 6872 int rc; 6873 u64 nrate[NCHAN], orate[NCHAN]; 6874 6875 rc = sysctl_wire_old_buffer(req, 0); 6876 if (rc != 0) 6877 return (rc); 6878 6879 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6880 if (sb == NULL) 6881 return (ENOMEM); 6882 6883 t4_get_chan_txrate(sc, nrate, orate); 6884 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6885 "channel 3\n"); 6886 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 6887 nrate[0], nrate[1], nrate[2], nrate[3]); 6888 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 6889 orate[0], orate[1], orate[2], orate[3]); 6890 6891 rc = sbuf_finish(sb); 6892 sbuf_delete(sb); 6893 6894 return (rc); 6895 } 6896 6897 static int 6898 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) 6899 { 6900 struct adapter *sc = arg1; 6901 struct sbuf *sb; 6902 uint32_t *buf, *p; 6903 int rc, i; 6904 6905 rc = sysctl_wire_old_buffer(req, 0); 6906 if (rc != 0) 6907 return (rc); 6908 6909 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6910 if (sb == NULL) 6911 return (ENOMEM); 6912 6913 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, 6914 M_ZERO | M_WAITOK); 6915 6916 t4_ulprx_read_la(sc, buf); 6917 p = buf; 6918 6919 sbuf_printf(sb, " Pcmd Type Message" 6920 " Data"); 6921 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { 6922 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", 6923 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); 6924 } 6925 6926 rc = sbuf_finish(sb); 6927 sbuf_delete(sb); 6928 free(buf, M_CXGBE); 6929 return (rc); 6930 } 6931 6932 static int 6933 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 6934 { 6935 struct adapter *sc = arg1; 6936 struct sbuf *sb; 6937 int rc, v; 6938 6939 rc = sysctl_wire_old_buffer(req, 0); 6940 if (rc != 0) 6941 return (rc); 6942 6943 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6944 if (sb == NULL) 6945 return (ENOMEM); 6946 6947 v = t4_read_reg(sc, A_SGE_STAT_CFG); 6948 if (G_STATSOURCE_T5(v) == 7) { 6949 if (G_STATMODE(v) == 0) { 6950 sbuf_printf(sb, "total %d, incomplete %d", 6951 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6952 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6953 } else if (G_STATMODE(v) == 1) { 6954 sbuf_printf(sb, "total %d, data overflow %d", 6955 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6956 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6957 } 6958 } 6959 rc = sbuf_finish(sb); 6960 sbuf_delete(sb); 6961 6962 return (rc); 6963 } 6964 #endif 6965 6966 static uint32_t 6967 fconf_to_mode(uint32_t fconf) 6968 { 6969 uint32_t mode; 6970 6971 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 6972 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 6973 6974 if (fconf & F_FRAGMENTATION) 6975 mode |= T4_FILTER_IP_FRAGMENT; 6976 6977 if (fconf & F_MPSHITTYPE) 6978 mode |= T4_FILTER_MPS_HIT_TYPE; 6979 6980 if (fconf & F_MACMATCH) 6981 mode |= T4_FILTER_MAC_IDX; 6982 6983 if (fconf & F_ETHERTYPE) 6984 mode |= T4_FILTER_ETH_TYPE; 6985 6986 if (fconf & F_PROTOCOL) 6987 mode |= T4_FILTER_IP_PROTO; 6988 6989 if (fconf & F_TOS) 6990 mode |= T4_FILTER_IP_TOS; 6991 6992 if (fconf & F_VLAN) 6993 mode |= T4_FILTER_VLAN; 6994 6995 if (fconf & F_VNIC_ID) 6996 mode |= T4_FILTER_VNIC; 6997 6998 if (fconf & F_PORT) 6999 mode |= T4_FILTER_PORT; 7000 7001 if (fconf & F_FCOE) 7002 mode |= T4_FILTER_FCoE; 7003 7004 return (mode); 7005 } 7006 7007 static uint32_t 7008 mode_to_fconf(uint32_t mode) 7009 { 7010 uint32_t fconf = 0; 7011 7012 if (mode & T4_FILTER_IP_FRAGMENT) 7013 fconf |= F_FRAGMENTATION; 7014 7015 if (mode & T4_FILTER_MPS_HIT_TYPE) 7016 fconf |= F_MPSHITTYPE; 7017 7018 if (mode & T4_FILTER_MAC_IDX) 7019 fconf |= F_MACMATCH; 7020 7021 if (mode & T4_FILTER_ETH_TYPE) 7022 fconf |= F_ETHERTYPE; 7023 7024 if (mode & T4_FILTER_IP_PROTO) 7025 fconf |= F_PROTOCOL; 7026 7027 if (mode & T4_FILTER_IP_TOS) 7028 fconf |= F_TOS; 7029 7030 if (mode & T4_FILTER_VLAN) 7031 fconf |= F_VLAN; 7032 7033 if (mode & T4_FILTER_VNIC) 7034 fconf |= F_VNIC_ID; 7035 7036 if (mode & T4_FILTER_PORT) 7037 fconf |= F_PORT; 7038 7039 if (mode & T4_FILTER_FCoE) 7040 fconf |= F_FCOE; 7041 7042 return (fconf); 7043 } 7044 7045 static uint32_t 7046 fspec_to_fconf(struct t4_filter_specification *fs) 7047 { 7048 uint32_t fconf = 0; 7049 7050 if (fs->val.frag || fs->mask.frag) 7051 fconf |= F_FRAGMENTATION; 7052 7053 if (fs->val.matchtype || fs->mask.matchtype) 7054 fconf |= F_MPSHITTYPE; 7055 7056 if (fs->val.macidx || fs->mask.macidx) 7057 fconf |= F_MACMATCH; 7058 7059 if (fs->val.ethtype || fs->mask.ethtype) 7060 fconf |= F_ETHERTYPE; 7061 7062 if (fs->val.proto || fs->mask.proto) 7063 fconf |= F_PROTOCOL; 7064 7065 if (fs->val.tos || fs->mask.tos) 7066 fconf |= F_TOS; 7067 7068 if (fs->val.vlan_vld || fs->mask.vlan_vld) 7069 fconf |= F_VLAN; 7070 7071 if (fs->val.vnic_vld || fs->mask.vnic_vld) 7072 fconf |= F_VNIC_ID; 7073 7074 if (fs->val.iport || fs->mask.iport) 7075 fconf |= F_PORT; 7076 7077 if (fs->val.fcoe || fs->mask.fcoe) 7078 fconf |= F_FCOE; 7079 7080 return (fconf); 7081 } 7082 7083 static int 7084 get_filter_mode(struct adapter *sc, uint32_t *mode) 7085 { 7086 int rc; 7087 uint32_t fconf; 7088 7089 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7090 "t4getfm"); 7091 if (rc) 7092 return (rc); 7093 7094 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 7095 A_TP_VLAN_PRI_MAP); 7096 7097 if (sc->params.tp.vlan_pri_map != fconf) { 7098 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 7099 device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map, 7100 fconf); 7101 } 7102 7103 *mode = fconf_to_mode(fconf); 7104 7105 end_synchronized_op(sc, LOCK_HELD); 7106 return (0); 7107 } 7108 7109 static int 7110 set_filter_mode(struct adapter *sc, uint32_t mode) 7111 { 7112 uint32_t fconf; 7113 int rc; 7114 7115 fconf = mode_to_fconf(mode); 7116 7117 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7118 "t4setfm"); 7119 if (rc) 7120 return (rc); 7121 7122 if (sc->tids.ftids_in_use > 0) { 7123 rc = EBUSY; 7124 goto done; 7125 } 7126 7127 #ifdef TCP_OFFLOAD 7128 if (uld_active(sc, ULD_TOM)) { 7129 rc = EBUSY; 7130 goto done; 7131 } 7132 #endif 7133 7134 rc = -t4_set_filter_mode(sc, fconf); 7135 done: 7136 end_synchronized_op(sc, LOCK_HELD); 7137 return (rc); 7138 } 7139 7140 static inline uint64_t 7141 get_filter_hits(struct adapter *sc, uint32_t fid) 7142 { 7143 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 7144 uint64_t hits; 7145 7146 memwin_info(sc, 0, &mw_base, NULL); 7147 off = position_memwin(sc, 0, 7148 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 7149 if (is_t4(sc)) { 7150 hits = t4_read_reg64(sc, mw_base + off + 16); 7151 hits = be64toh(hits); 7152 } else { 7153 hits = t4_read_reg(sc, mw_base + off + 24); 7154 hits = be32toh(hits); 7155 } 7156 7157 return (hits); 7158 } 7159 7160 static int 7161 get_filter(struct adapter *sc, struct t4_filter *t) 7162 { 7163 int i, rc, nfilters = sc->tids.nftids; 7164 struct filter_entry *f; 7165 7166 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7167 "t4getf"); 7168 if (rc) 7169 return (rc); 7170 7171 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 7172 t->idx >= nfilters) { 7173 t->idx = 0xffffffff; 7174 goto done; 7175 } 7176 7177 f = &sc->tids.ftid_tab[t->idx]; 7178 for (i = t->idx; i < nfilters; i++, f++) { 7179 if (f->valid) { 7180 t->idx = i; 7181 t->l2tidx = f->l2t ? f->l2t->idx : 0; 7182 t->smtidx = f->smtidx; 7183 if (f->fs.hitcnts) 7184 t->hits = get_filter_hits(sc, t->idx); 7185 else 7186 t->hits = UINT64_MAX; 7187 t->fs = f->fs; 7188 7189 goto done; 7190 } 7191 } 7192 7193 t->idx = 0xffffffff; 7194 done: 7195 end_synchronized_op(sc, LOCK_HELD); 7196 return (0); 7197 } 7198 7199 static int 7200 set_filter(struct adapter *sc, struct t4_filter *t) 7201 { 7202 unsigned int nfilters, nports; 7203 struct filter_entry *f; 7204 int i, rc; 7205 7206 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 7207 if (rc) 7208 return (rc); 7209 7210 nfilters = sc->tids.nftids; 7211 nports = sc->params.nports; 7212 7213 if (nfilters == 0) { 7214 rc = ENOTSUP; 7215 goto done; 7216 } 7217 7218 if (!(sc->flags & FULL_INIT_DONE)) { 7219 rc = EAGAIN; 7220 goto done; 7221 } 7222 7223 if (t->idx >= nfilters) { 7224 rc = EINVAL; 7225 goto done; 7226 } 7227 7228 /* Validate against the global filter mode */ 7229 if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) != 7230 sc->params.tp.vlan_pri_map) { 7231 rc = E2BIG; 7232 goto done; 7233 } 7234 7235 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 7236 rc = EINVAL; 7237 goto done; 7238 } 7239 7240 if (t->fs.val.iport >= nports) { 7241 rc = EINVAL; 7242 goto done; 7243 } 7244 7245 /* Can't specify an iq if not steering to it */ 7246 if (!t->fs.dirsteer && t->fs.iq) { 7247 rc = EINVAL; 7248 goto done; 7249 } 7250 7251 /* IPv6 filter idx must be 4 aligned */ 7252 if (t->fs.type == 1 && 7253 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 7254 rc = EINVAL; 7255 goto done; 7256 } 7257 7258 if (sc->tids.ftid_tab == NULL) { 7259 KASSERT(sc->tids.ftids_in_use == 0, 7260 ("%s: no memory allocated but filters_in_use > 0", 7261 __func__)); 7262 7263 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 7264 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 7265 if (sc->tids.ftid_tab == NULL) { 7266 rc = ENOMEM; 7267 goto done; 7268 } 7269 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 7270 } 7271 7272 for (i = 0; i < 4; i++) { 7273 f = &sc->tids.ftid_tab[t->idx + i]; 7274 7275 if (f->pending || f->valid) { 7276 rc = EBUSY; 7277 goto done; 7278 } 7279 if (f->locked) { 7280 rc = EPERM; 7281 goto done; 7282 } 7283 7284 if (t->fs.type == 0) 7285 break; 7286 } 7287 7288 f = &sc->tids.ftid_tab[t->idx]; 7289 f->fs = t->fs; 7290 7291 rc = set_filter_wr(sc, t->idx); 7292 done: 7293 end_synchronized_op(sc, 0); 7294 7295 if (rc == 0) { 7296 mtx_lock(&sc->tids.ftid_lock); 7297 for (;;) { 7298 if (f->pending == 0) { 7299 rc = f->valid ? 0 : EIO; 7300 break; 7301 } 7302 7303 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7304 PCATCH, "t4setfw", 0)) { 7305 rc = EINPROGRESS; 7306 break; 7307 } 7308 } 7309 mtx_unlock(&sc->tids.ftid_lock); 7310 } 7311 return (rc); 7312 } 7313 7314 static int 7315 del_filter(struct adapter *sc, struct t4_filter *t) 7316 { 7317 unsigned int nfilters; 7318 struct filter_entry *f; 7319 int rc; 7320 7321 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 7322 if (rc) 7323 return (rc); 7324 7325 nfilters = sc->tids.nftids; 7326 7327 if (nfilters == 0) { 7328 rc = ENOTSUP; 7329 goto done; 7330 } 7331 7332 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 7333 t->idx >= nfilters) { 7334 rc = EINVAL; 7335 goto done; 7336 } 7337 7338 if (!(sc->flags & FULL_INIT_DONE)) { 7339 rc = EAGAIN; 7340 goto done; 7341 } 7342 7343 f = &sc->tids.ftid_tab[t->idx]; 7344 7345 if (f->pending) { 7346 rc = EBUSY; 7347 goto done; 7348 } 7349 if (f->locked) { 7350 rc = EPERM; 7351 goto done; 7352 } 7353 7354 if (f->valid) { 7355 t->fs = f->fs; /* extra info for the caller */ 7356 rc = del_filter_wr(sc, t->idx); 7357 } 7358 7359 done: 7360 end_synchronized_op(sc, 0); 7361 7362 if (rc == 0) { 7363 mtx_lock(&sc->tids.ftid_lock); 7364 for (;;) { 7365 if (f->pending == 0) { 7366 rc = f->valid ? EIO : 0; 7367 break; 7368 } 7369 7370 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7371 PCATCH, "t4delfw", 0)) { 7372 rc = EINPROGRESS; 7373 break; 7374 } 7375 } 7376 mtx_unlock(&sc->tids.ftid_lock); 7377 } 7378 7379 return (rc); 7380 } 7381 7382 static void 7383 clear_filter(struct filter_entry *f) 7384 { 7385 if (f->l2t) 7386 t4_l2t_release(f->l2t); 7387 7388 bzero(f, sizeof (*f)); 7389 } 7390 7391 static int 7392 set_filter_wr(struct adapter *sc, int fidx) 7393 { 7394 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7395 struct fw_filter_wr *fwr; 7396 unsigned int ftid; 7397 struct wrq_cookie cookie; 7398 7399 ASSERT_SYNCHRONIZED_OP(sc); 7400 7401 if (f->fs.newdmac || f->fs.newvlan) { 7402 /* This filter needs an L2T entry; allocate one. */ 7403 f->l2t = t4_l2t_alloc_switching(sc->l2t); 7404 if (f->l2t == NULL) 7405 return (EAGAIN); 7406 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 7407 f->fs.dmac)) { 7408 t4_l2t_release(f->l2t); 7409 f->l2t = NULL; 7410 return (ENOMEM); 7411 } 7412 } 7413 7414 ftid = sc->tids.ftid_base + fidx; 7415 7416 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 7417 if (fwr == NULL) 7418 return (ENOMEM); 7419 bzero(fwr, sizeof(*fwr)); 7420 7421 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 7422 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 7423 fwr->tid_to_iq = 7424 htobe32(V_FW_FILTER_WR_TID(ftid) | 7425 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 7426 V_FW_FILTER_WR_NOREPLY(0) | 7427 V_FW_FILTER_WR_IQ(f->fs.iq)); 7428 fwr->del_filter_to_l2tix = 7429 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 7430 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 7431 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 7432 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 7433 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 7434 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 7435 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 7436 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 7437 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 7438 f->fs.newvlan == VLAN_REWRITE) | 7439 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 7440 f->fs.newvlan == VLAN_REWRITE) | 7441 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 7442 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 7443 V_FW_FILTER_WR_PRIO(f->fs.prio) | 7444 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 7445 fwr->ethtype = htobe16(f->fs.val.ethtype); 7446 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 7447 fwr->frag_to_ovlan_vldm = 7448 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 7449 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 7450 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 7451 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 7452 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 7453 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 7454 fwr->smac_sel = 0; 7455 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 7456 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 7457 fwr->maci_to_matchtypem = 7458 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 7459 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 7460 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 7461 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 7462 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 7463 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 7464 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 7465 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 7466 fwr->ptcl = f->fs.val.proto; 7467 fwr->ptclm = f->fs.mask.proto; 7468 fwr->ttyp = f->fs.val.tos; 7469 fwr->ttypm = f->fs.mask.tos; 7470 fwr->ivlan = htobe16(f->fs.val.vlan); 7471 fwr->ivlanm = htobe16(f->fs.mask.vlan); 7472 fwr->ovlan = htobe16(f->fs.val.vnic); 7473 fwr->ovlanm = htobe16(f->fs.mask.vnic); 7474 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 7475 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 7476 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 7477 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 7478 fwr->lp = htobe16(f->fs.val.dport); 7479 fwr->lpm = htobe16(f->fs.mask.dport); 7480 fwr->fp = htobe16(f->fs.val.sport); 7481 fwr->fpm = htobe16(f->fs.mask.sport); 7482 if (f->fs.newsmac) 7483 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 7484 7485 f->pending = 1; 7486 sc->tids.ftids_in_use++; 7487 7488 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 7489 return (0); 7490 } 7491 7492 static int 7493 del_filter_wr(struct adapter *sc, int fidx) 7494 { 7495 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7496 struct fw_filter_wr *fwr; 7497 unsigned int ftid; 7498 struct wrq_cookie cookie; 7499 7500 ftid = sc->tids.ftid_base + fidx; 7501 7502 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 7503 if (fwr == NULL) 7504 return (ENOMEM); 7505 bzero(fwr, sizeof (*fwr)); 7506 7507 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 7508 7509 f->pending = 1; 7510 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 7511 return (0); 7512 } 7513 7514 int 7515 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 7516 { 7517 struct adapter *sc = iq->adapter; 7518 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 7519 unsigned int idx = GET_TID(rpl); 7520 unsigned int rc; 7521 struct filter_entry *f; 7522 7523 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 7524 rss->opcode)); 7525 7526 if (is_ftid(sc, idx)) { 7527 7528 idx -= sc->tids.ftid_base; 7529 f = &sc->tids.ftid_tab[idx]; 7530 rc = G_COOKIE(rpl->cookie); 7531 7532 mtx_lock(&sc->tids.ftid_lock); 7533 if (rc == FW_FILTER_WR_FLT_ADDED) { 7534 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 7535 __func__, idx)); 7536 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 7537 f->pending = 0; /* asynchronous setup completed */ 7538 f->valid = 1; 7539 } else { 7540 if (rc != FW_FILTER_WR_FLT_DELETED) { 7541 /* Add or delete failed, display an error */ 7542 log(LOG_ERR, 7543 "filter %u setup failed with error %u\n", 7544 idx, rc); 7545 } 7546 7547 clear_filter(f); 7548 sc->tids.ftids_in_use--; 7549 } 7550 wakeup(&sc->tids.ftid_tab); 7551 mtx_unlock(&sc->tids.ftid_lock); 7552 } 7553 7554 return (0); 7555 } 7556 7557 static int 7558 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 7559 { 7560 int rc; 7561 7562 if (cntxt->cid > M_CTXTQID) 7563 return (EINVAL); 7564 7565 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 7566 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 7567 return (EINVAL); 7568 7569 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 7570 if (rc) 7571 return (rc); 7572 7573 if (sc->flags & FW_OK) { 7574 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 7575 &cntxt->data[0]); 7576 if (rc == 0) 7577 goto done; 7578 } 7579 7580 /* 7581 * Read via firmware failed or wasn't even attempted. Read directly via 7582 * the backdoor. 7583 */ 7584 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 7585 done: 7586 end_synchronized_op(sc, 0); 7587 return (rc); 7588 } 7589 7590 static int 7591 load_fw(struct adapter *sc, struct t4_data *fw) 7592 { 7593 int rc; 7594 uint8_t *fw_data; 7595 7596 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 7597 if (rc) 7598 return (rc); 7599 7600 if (sc->flags & FULL_INIT_DONE) { 7601 rc = EBUSY; 7602 goto done; 7603 } 7604 7605 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 7606 if (fw_data == NULL) { 7607 rc = ENOMEM; 7608 goto done; 7609 } 7610 7611 rc = copyin(fw->data, fw_data, fw->len); 7612 if (rc == 0) 7613 rc = -t4_load_fw(sc, fw_data, fw->len); 7614 7615 free(fw_data, M_CXGBE); 7616 done: 7617 end_synchronized_op(sc, 0); 7618 return (rc); 7619 } 7620 7621 static int 7622 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 7623 { 7624 uint32_t addr, off, remaining, i, n; 7625 uint32_t *buf, *b; 7626 uint32_t mw_base, mw_aperture; 7627 int rc; 7628 uint8_t *dst; 7629 7630 rc = validate_mem_range(sc, mr->addr, mr->len); 7631 if (rc != 0) 7632 return (rc); 7633 7634 memwin_info(sc, win, &mw_base, &mw_aperture); 7635 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 7636 addr = mr->addr; 7637 remaining = mr->len; 7638 dst = (void *)mr->data; 7639 7640 while (remaining) { 7641 off = position_memwin(sc, win, addr); 7642 7643 /* number of bytes that we'll copy in the inner loop */ 7644 n = min(remaining, mw_aperture - off); 7645 for (i = 0; i < n; i += 4) 7646 *b++ = t4_read_reg(sc, mw_base + off + i); 7647 7648 rc = copyout(buf, dst, n); 7649 if (rc != 0) 7650 break; 7651 7652 b = buf; 7653 dst += n; 7654 remaining -= n; 7655 addr += n; 7656 } 7657 7658 free(buf, M_CXGBE); 7659 return (rc); 7660 } 7661 7662 static int 7663 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 7664 { 7665 int rc; 7666 7667 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 7668 return (EINVAL); 7669 7670 if (i2cd->len > sizeof(i2cd->data)) 7671 return (EFBIG); 7672 7673 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 7674 if (rc) 7675 return (rc); 7676 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 7677 i2cd->offset, i2cd->len, &i2cd->data[0]); 7678 end_synchronized_op(sc, 0); 7679 7680 return (rc); 7681 } 7682 7683 static int 7684 in_range(int val, int lo, int hi) 7685 { 7686 7687 return (val < 0 || (val <= hi && val >= lo)); 7688 } 7689 7690 static int 7691 set_sched_class(struct adapter *sc, struct t4_sched_params *p) 7692 { 7693 int fw_subcmd, fw_type, rc; 7694 7695 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc"); 7696 if (rc) 7697 return (rc); 7698 7699 if (!(sc->flags & FULL_INIT_DONE)) { 7700 rc = EAGAIN; 7701 goto done; 7702 } 7703 7704 /* 7705 * Translate the cxgbetool parameters into T4 firmware parameters. (The 7706 * sub-command and type are in common locations.) 7707 */ 7708 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG) 7709 fw_subcmd = FW_SCHED_SC_CONFIG; 7710 else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS) 7711 fw_subcmd = FW_SCHED_SC_PARAMS; 7712 else { 7713 rc = EINVAL; 7714 goto done; 7715 } 7716 if (p->type == SCHED_CLASS_TYPE_PACKET) 7717 fw_type = FW_SCHED_TYPE_PKTSCHED; 7718 else { 7719 rc = EINVAL; 7720 goto done; 7721 } 7722 7723 if (fw_subcmd == FW_SCHED_SC_CONFIG) { 7724 /* Vet our parameters ..*/ 7725 if (p->u.config.minmax < 0) { 7726 rc = EINVAL; 7727 goto done; 7728 } 7729 7730 /* And pass the request to the firmware ...*/ 7731 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1); 7732 goto done; 7733 } 7734 7735 if (fw_subcmd == FW_SCHED_SC_PARAMS) { 7736 int fw_level; 7737 int fw_mode; 7738 int fw_rateunit; 7739 int fw_ratemode; 7740 7741 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL) 7742 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL; 7743 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) 7744 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR; 7745 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) 7746 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL; 7747 else { 7748 rc = EINVAL; 7749 goto done; 7750 } 7751 7752 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS) 7753 fw_mode = FW_SCHED_PARAMS_MODE_CLASS; 7754 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW) 7755 fw_mode = FW_SCHED_PARAMS_MODE_FLOW; 7756 else { 7757 rc = EINVAL; 7758 goto done; 7759 } 7760 7761 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS) 7762 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 7763 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS) 7764 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE; 7765 else { 7766 rc = EINVAL; 7767 goto done; 7768 } 7769 7770 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL) 7771 fw_ratemode = FW_SCHED_PARAMS_RATE_REL; 7772 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS) 7773 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS; 7774 else { 7775 rc = EINVAL; 7776 goto done; 7777 } 7778 7779 /* Vet our parameters ... */ 7780 if (!in_range(p->u.params.channel, 0, 3) || 7781 !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) || 7782 !in_range(p->u.params.minrate, 0, 10000000) || 7783 !in_range(p->u.params.maxrate, 0, 10000000) || 7784 !in_range(p->u.params.weight, 0, 100)) { 7785 rc = ERANGE; 7786 goto done; 7787 } 7788 7789 /* 7790 * Translate any unset parameters into the firmware's 7791 * nomenclature and/or fail the call if the parameters 7792 * are required ... 7793 */ 7794 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 || 7795 p->u.params.channel < 0 || p->u.params.cl < 0) { 7796 rc = EINVAL; 7797 goto done; 7798 } 7799 if (p->u.params.minrate < 0) 7800 p->u.params.minrate = 0; 7801 if (p->u.params.maxrate < 0) { 7802 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7803 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7804 rc = EINVAL; 7805 goto done; 7806 } else 7807 p->u.params.maxrate = 0; 7808 } 7809 if (p->u.params.weight < 0) { 7810 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) { 7811 rc = EINVAL; 7812 goto done; 7813 } else 7814 p->u.params.weight = 0; 7815 } 7816 if (p->u.params.pktsize < 0) { 7817 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7818 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7819 rc = EINVAL; 7820 goto done; 7821 } else 7822 p->u.params.pktsize = 0; 7823 } 7824 7825 /* See what the firmware thinks of the request ... */ 7826 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode, 7827 fw_rateunit, fw_ratemode, p->u.params.channel, 7828 p->u.params.cl, p->u.params.minrate, p->u.params.maxrate, 7829 p->u.params.weight, p->u.params.pktsize, 1); 7830 goto done; 7831 } 7832 7833 rc = EINVAL; 7834 done: 7835 end_synchronized_op(sc, 0); 7836 return (rc); 7837 } 7838 7839 static int 7840 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p) 7841 { 7842 struct port_info *pi = NULL; 7843 struct sge_txq *txq; 7844 uint32_t fw_mnem, fw_queue, fw_class; 7845 int i, rc; 7846 7847 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq"); 7848 if (rc) 7849 return (rc); 7850 7851 if (!(sc->flags & FULL_INIT_DONE)) { 7852 rc = EAGAIN; 7853 goto done; 7854 } 7855 7856 if (p->port >= sc->params.nports) { 7857 rc = EINVAL; 7858 goto done; 7859 } 7860 7861 pi = sc->port[p->port]; 7862 if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) { 7863 rc = EINVAL; 7864 goto done; 7865 } 7866 7867 /* 7868 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX 7869 * Scheduling Class in this case). 7870 */ 7871 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 7872 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH)); 7873 fw_class = p->cl < 0 ? 0xffffffff : p->cl; 7874 7875 /* 7876 * If op.queue is non-negative, then we're only changing the scheduling 7877 * on a single specified TX queue. 7878 */ 7879 if (p->queue >= 0) { 7880 txq = &sc->sge.txq[pi->first_txq + p->queue]; 7881 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7882 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7883 &fw_class); 7884 goto done; 7885 } 7886 7887 /* 7888 * Change the scheduling on all the TX queues for the 7889 * interface. 7890 */ 7891 for_each_txq(pi, i, txq) { 7892 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7893 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7894 &fw_class); 7895 if (rc) 7896 goto done; 7897 } 7898 7899 rc = 0; 7900 done: 7901 end_synchronized_op(sc, 0); 7902 return (rc); 7903 } 7904 7905 int 7906 t4_os_find_pci_capability(struct adapter *sc, int cap) 7907 { 7908 int i; 7909 7910 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 7911 } 7912 7913 int 7914 t4_os_pci_save_state(struct adapter *sc) 7915 { 7916 device_t dev; 7917 struct pci_devinfo *dinfo; 7918 7919 dev = sc->dev; 7920 dinfo = device_get_ivars(dev); 7921 7922 pci_cfg_save(dev, dinfo, 0); 7923 return (0); 7924 } 7925 7926 int 7927 t4_os_pci_restore_state(struct adapter *sc) 7928 { 7929 device_t dev; 7930 struct pci_devinfo *dinfo; 7931 7932 dev = sc->dev; 7933 dinfo = device_get_ivars(dev); 7934 7935 pci_cfg_restore(dev, dinfo); 7936 return (0); 7937 } 7938 7939 void 7940 t4_os_portmod_changed(const struct adapter *sc, int idx) 7941 { 7942 struct port_info *pi = sc->port[idx]; 7943 static const char *mod_str[] = { 7944 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 7945 }; 7946 7947 build_medialist(pi, &pi->media); 7948 #ifdef DEV_NETMAP 7949 build_medialist(pi, &pi->nm_media); 7950 #endif 7951 7952 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 7953 if_printf(pi->ifp, "transceiver unplugged.\n"); 7954 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 7955 if_printf(pi->ifp, "unknown transceiver inserted.\n"); 7956 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 7957 if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 7958 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 7959 if_printf(pi->ifp, "%s transceiver inserted.\n", 7960 mod_str[pi->mod_type]); 7961 } else { 7962 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 7963 pi->mod_type); 7964 } 7965 } 7966 7967 void 7968 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason) 7969 { 7970 struct port_info *pi = sc->port[idx]; 7971 struct ifnet *ifp = pi->ifp; 7972 7973 if (link_stat) { 7974 pi->linkdnrc = -1; 7975 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 7976 if_link_state_change(ifp, LINK_STATE_UP); 7977 } else { 7978 if (reason >= 0) 7979 pi->linkdnrc = reason; 7980 if_link_state_change(ifp, LINK_STATE_DOWN); 7981 } 7982 } 7983 7984 void 7985 t4_iterate(void (*func)(struct adapter *, void *), void *arg) 7986 { 7987 struct adapter *sc; 7988 7989 sx_slock(&t4_list_lock); 7990 SLIST_FOREACH(sc, &t4_list, link) { 7991 /* 7992 * func should not make any assumptions about what state sc is 7993 * in - the only guarantee is that sc->sc_lock is a valid lock. 7994 */ 7995 func(sc, arg); 7996 } 7997 sx_sunlock(&t4_list_lock); 7998 } 7999 8000 static int 8001 t4_open(struct cdev *dev, int flags, int type, struct thread *td) 8002 { 8003 return (0); 8004 } 8005 8006 static int 8007 t4_close(struct cdev *dev, int flags, int type, struct thread *td) 8008 { 8009 return (0); 8010 } 8011 8012 static int 8013 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 8014 struct thread *td) 8015 { 8016 int rc; 8017 struct adapter *sc = dev->si_drv1; 8018 8019 rc = priv_check(td, PRIV_DRIVER); 8020 if (rc != 0) 8021 return (rc); 8022 8023 switch (cmd) { 8024 case CHELSIO_T4_GETREG: { 8025 struct t4_reg *edata = (struct t4_reg *)data; 8026 8027 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8028 return (EFAULT); 8029 8030 if (edata->size == 4) 8031 edata->val = t4_read_reg(sc, edata->addr); 8032 else if (edata->size == 8) 8033 edata->val = t4_read_reg64(sc, edata->addr); 8034 else 8035 return (EINVAL); 8036 8037 break; 8038 } 8039 case CHELSIO_T4_SETREG: { 8040 struct t4_reg *edata = (struct t4_reg *)data; 8041 8042 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8043 return (EFAULT); 8044 8045 if (edata->size == 4) { 8046 if (edata->val & 0xffffffff00000000) 8047 return (EINVAL); 8048 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 8049 } else if (edata->size == 8) 8050 t4_write_reg64(sc, edata->addr, edata->val); 8051 else 8052 return (EINVAL); 8053 break; 8054 } 8055 case CHELSIO_T4_REGDUMP: { 8056 struct t4_regdump *regs = (struct t4_regdump *)data; 8057 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 8058 uint8_t *buf; 8059 8060 if (regs->len < reglen) { 8061 regs->len = reglen; /* hint to the caller */ 8062 return (ENOBUFS); 8063 } 8064 8065 regs->len = reglen; 8066 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 8067 t4_get_regs(sc, regs, buf); 8068 rc = copyout(buf, regs->data, reglen); 8069 free(buf, M_CXGBE); 8070 break; 8071 } 8072 case CHELSIO_T4_GET_FILTER_MODE: 8073 rc = get_filter_mode(sc, (uint32_t *)data); 8074 break; 8075 case CHELSIO_T4_SET_FILTER_MODE: 8076 rc = set_filter_mode(sc, *(uint32_t *)data); 8077 break; 8078 case CHELSIO_T4_GET_FILTER: 8079 rc = get_filter(sc, (struct t4_filter *)data); 8080 break; 8081 case CHELSIO_T4_SET_FILTER: 8082 rc = set_filter(sc, (struct t4_filter *)data); 8083 break; 8084 case CHELSIO_T4_DEL_FILTER: 8085 rc = del_filter(sc, (struct t4_filter *)data); 8086 break; 8087 case CHELSIO_T4_GET_SGE_CONTEXT: 8088 rc = get_sge_context(sc, (struct t4_sge_context *)data); 8089 break; 8090 case CHELSIO_T4_LOAD_FW: 8091 rc = load_fw(sc, (struct t4_data *)data); 8092 break; 8093 case CHELSIO_T4_GET_MEM: 8094 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 8095 break; 8096 case CHELSIO_T4_GET_I2C: 8097 rc = read_i2c(sc, (struct t4_i2c_data *)data); 8098 break; 8099 case CHELSIO_T4_CLEAR_STATS: { 8100 int i; 8101 u_int port_id = *(uint32_t *)data; 8102 struct port_info *pi; 8103 8104 if (port_id >= sc->params.nports) 8105 return (EINVAL); 8106 pi = sc->port[port_id]; 8107 8108 /* MAC stats */ 8109 t4_clr_port_stats(sc, pi->tx_chan); 8110 pi->tx_parse_error = 0; 8111 8112 if (pi->flags & PORT_INIT_DONE) { 8113 struct sge_rxq *rxq; 8114 struct sge_txq *txq; 8115 struct sge_wrq *wrq; 8116 8117 for_each_rxq(pi, i, rxq) { 8118 #if defined(INET) || defined(INET6) 8119 rxq->lro.lro_queued = 0; 8120 rxq->lro.lro_flushed = 0; 8121 #endif 8122 rxq->rxcsum = 0; 8123 rxq->vlan_extraction = 0; 8124 } 8125 8126 for_each_txq(pi, i, txq) { 8127 txq->txcsum = 0; 8128 txq->tso_wrs = 0; 8129 txq->vlan_insertion = 0; 8130 txq->imm_wrs = 0; 8131 txq->sgl_wrs = 0; 8132 txq->txpkt_wrs = 0; 8133 txq->txpkts0_wrs = 0; 8134 txq->txpkts1_wrs = 0; 8135 txq->txpkts0_pkts = 0; 8136 txq->txpkts1_pkts = 0; 8137 mp_ring_reset_stats(txq->r); 8138 } 8139 8140 #ifdef TCP_OFFLOAD 8141 /* nothing to clear for each ofld_rxq */ 8142 8143 for_each_ofld_txq(pi, i, wrq) { 8144 wrq->tx_wrs_direct = 0; 8145 wrq->tx_wrs_copied = 0; 8146 } 8147 #endif 8148 wrq = &sc->sge.ctrlq[pi->port_id]; 8149 wrq->tx_wrs_direct = 0; 8150 wrq->tx_wrs_copied = 0; 8151 } 8152 break; 8153 } 8154 case CHELSIO_T4_SCHED_CLASS: 8155 rc = set_sched_class(sc, (struct t4_sched_params *)data); 8156 break; 8157 case CHELSIO_T4_SCHED_QUEUE: 8158 rc = set_sched_queue(sc, (struct t4_sched_queue *)data); 8159 break; 8160 case CHELSIO_T4_GET_TRACER: 8161 rc = t4_get_tracer(sc, (struct t4_tracer *)data); 8162 break; 8163 case CHELSIO_T4_SET_TRACER: 8164 rc = t4_set_tracer(sc, (struct t4_tracer *)data); 8165 break; 8166 default: 8167 rc = EINVAL; 8168 } 8169 8170 return (rc); 8171 } 8172 8173 #ifdef TCP_OFFLOAD 8174 void 8175 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask, 8176 const unsigned int *pgsz_order) 8177 { 8178 struct port_info *pi = ifp->if_softc; 8179 struct adapter *sc = pi->adapter; 8180 8181 t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask); 8182 t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) | 8183 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) | 8184 V_HPZ3(pgsz_order[3])); 8185 } 8186 8187 static int 8188 toe_capability(struct port_info *pi, int enable) 8189 { 8190 int rc; 8191 struct adapter *sc = pi->adapter; 8192 8193 ASSERT_SYNCHRONIZED_OP(sc); 8194 8195 if (!is_offload(sc)) 8196 return (ENODEV); 8197 8198 if (enable) { 8199 /* 8200 * We need the port's queues around so that we're able to send 8201 * and receive CPLs to/from the TOE even if the ifnet for this 8202 * port has never been UP'd administratively. 8203 */ 8204 if (!(pi->flags & PORT_INIT_DONE)) { 8205 rc = cxgbe_init_synchronized(pi); 8206 if (rc) 8207 return (rc); 8208 } 8209 8210 if (isset(&sc->offload_map, pi->port_id)) 8211 return (0); 8212 8213 if (!uld_active(sc, ULD_TOM)) { 8214 rc = t4_activate_uld(sc, ULD_TOM); 8215 if (rc == EAGAIN) { 8216 log(LOG_WARNING, 8217 "You must kldload t4_tom.ko before trying " 8218 "to enable TOE on a cxgbe interface.\n"); 8219 } 8220 if (rc != 0) 8221 return (rc); 8222 KASSERT(sc->tom_softc != NULL, 8223 ("%s: TOM activated but softc NULL", __func__)); 8224 KASSERT(uld_active(sc, ULD_TOM), 8225 ("%s: TOM activated but flag not set", __func__)); 8226 } 8227 8228 /* Activate iWARP and iSCSI too, if the modules are loaded. */ 8229 if (!uld_active(sc, ULD_IWARP)) 8230 (void) t4_activate_uld(sc, ULD_IWARP); 8231 if (!uld_active(sc, ULD_ISCSI)) 8232 (void) t4_activate_uld(sc, ULD_ISCSI); 8233 8234 setbit(&sc->offload_map, pi->port_id); 8235 } else { 8236 if (!isset(&sc->offload_map, pi->port_id)) 8237 return (0); 8238 8239 KASSERT(uld_active(sc, ULD_TOM), 8240 ("%s: TOM never initialized?", __func__)); 8241 clrbit(&sc->offload_map, pi->port_id); 8242 } 8243 8244 return (0); 8245 } 8246 8247 /* 8248 * Add an upper layer driver to the global list. 8249 */ 8250 int 8251 t4_register_uld(struct uld_info *ui) 8252 { 8253 int rc = 0; 8254 struct uld_info *u; 8255 8256 sx_xlock(&t4_uld_list_lock); 8257 SLIST_FOREACH(u, &t4_uld_list, link) { 8258 if (u->uld_id == ui->uld_id) { 8259 rc = EEXIST; 8260 goto done; 8261 } 8262 } 8263 8264 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 8265 ui->refcount = 0; 8266 done: 8267 sx_xunlock(&t4_uld_list_lock); 8268 return (rc); 8269 } 8270 8271 int 8272 t4_unregister_uld(struct uld_info *ui) 8273 { 8274 int rc = EINVAL; 8275 struct uld_info *u; 8276 8277 sx_xlock(&t4_uld_list_lock); 8278 8279 SLIST_FOREACH(u, &t4_uld_list, link) { 8280 if (u == ui) { 8281 if (ui->refcount > 0) { 8282 rc = EBUSY; 8283 goto done; 8284 } 8285 8286 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 8287 rc = 0; 8288 goto done; 8289 } 8290 } 8291 done: 8292 sx_xunlock(&t4_uld_list_lock); 8293 return (rc); 8294 } 8295 8296 int 8297 t4_activate_uld(struct adapter *sc, int id) 8298 { 8299 int rc; 8300 struct uld_info *ui; 8301 8302 ASSERT_SYNCHRONIZED_OP(sc); 8303 8304 if (id < 0 || id > ULD_MAX) 8305 return (EINVAL); 8306 rc = EAGAIN; /* kldoad the module with this ULD and try again. */ 8307 8308 sx_slock(&t4_uld_list_lock); 8309 8310 SLIST_FOREACH(ui, &t4_uld_list, link) { 8311 if (ui->uld_id == id) { 8312 if (!(sc->flags & FULL_INIT_DONE)) { 8313 rc = adapter_full_init(sc); 8314 if (rc != 0) 8315 break; 8316 } 8317 8318 rc = ui->activate(sc); 8319 if (rc == 0) { 8320 setbit(&sc->active_ulds, id); 8321 ui->refcount++; 8322 } 8323 break; 8324 } 8325 } 8326 8327 sx_sunlock(&t4_uld_list_lock); 8328 8329 return (rc); 8330 } 8331 8332 int 8333 t4_deactivate_uld(struct adapter *sc, int id) 8334 { 8335 int rc; 8336 struct uld_info *ui; 8337 8338 ASSERT_SYNCHRONIZED_OP(sc); 8339 8340 if (id < 0 || id > ULD_MAX) 8341 return (EINVAL); 8342 rc = ENXIO; 8343 8344 sx_slock(&t4_uld_list_lock); 8345 8346 SLIST_FOREACH(ui, &t4_uld_list, link) { 8347 if (ui->uld_id == id) { 8348 rc = ui->deactivate(sc); 8349 if (rc == 0) { 8350 clrbit(&sc->active_ulds, id); 8351 ui->refcount--; 8352 } 8353 break; 8354 } 8355 } 8356 8357 sx_sunlock(&t4_uld_list_lock); 8358 8359 return (rc); 8360 } 8361 8362 int 8363 uld_active(struct adapter *sc, int uld_id) 8364 { 8365 8366 MPASS(uld_id >= 0 && uld_id <= ULD_MAX); 8367 8368 return (isset(&sc->active_ulds, uld_id)); 8369 } 8370 #endif 8371 8372 /* 8373 * Come up with reasonable defaults for some of the tunables, provided they're 8374 * not set by the user (in which case we'll use the values as is). 8375 */ 8376 static void 8377 tweak_tunables(void) 8378 { 8379 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 8380 8381 if (t4_ntxq10g < 1) 8382 t4_ntxq10g = min(nc, NTXQ_10G); 8383 8384 if (t4_ntxq1g < 1) 8385 t4_ntxq1g = min(nc, NTXQ_1G); 8386 8387 if (t4_nrxq10g < 1) 8388 t4_nrxq10g = min(nc, NRXQ_10G); 8389 8390 if (t4_nrxq1g < 1) 8391 t4_nrxq1g = min(nc, NRXQ_1G); 8392 8393 #ifdef TCP_OFFLOAD 8394 if (t4_nofldtxq10g < 1) 8395 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 8396 8397 if (t4_nofldtxq1g < 1) 8398 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 8399 8400 if (t4_nofldrxq10g < 1) 8401 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 8402 8403 if (t4_nofldrxq1g < 1) 8404 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 8405 8406 if (t4_toecaps_allowed == -1) 8407 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 8408 #else 8409 if (t4_toecaps_allowed == -1) 8410 t4_toecaps_allowed = 0; 8411 #endif 8412 8413 #ifdef DEV_NETMAP 8414 if (t4_nnmtxq10g < 1) 8415 t4_nnmtxq10g = min(nc, NNMTXQ_10G); 8416 8417 if (t4_nnmtxq1g < 1) 8418 t4_nnmtxq1g = min(nc, NNMTXQ_1G); 8419 8420 if (t4_nnmrxq10g < 1) 8421 t4_nnmrxq10g = min(nc, NNMRXQ_10G); 8422 8423 if (t4_nnmrxq1g < 1) 8424 t4_nnmrxq1g = min(nc, NNMRXQ_1G); 8425 #endif 8426 8427 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 8428 t4_tmr_idx_10g = TMR_IDX_10G; 8429 8430 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 8431 t4_pktc_idx_10g = PKTC_IDX_10G; 8432 8433 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 8434 t4_tmr_idx_1g = TMR_IDX_1G; 8435 8436 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 8437 t4_pktc_idx_1g = PKTC_IDX_1G; 8438 8439 if (t4_qsize_txq < 128) 8440 t4_qsize_txq = 128; 8441 8442 if (t4_qsize_rxq < 128) 8443 t4_qsize_rxq = 128; 8444 while (t4_qsize_rxq & 7) 8445 t4_qsize_rxq++; 8446 8447 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 8448 } 8449 8450 static struct sx mlu; /* mod load unload */ 8451 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload"); 8452 8453 static int 8454 mod_event(module_t mod, int cmd, void *arg) 8455 { 8456 int rc = 0; 8457 static int loaded = 0; 8458 8459 switch (cmd) { 8460 case MOD_LOAD: 8461 sx_xlock(&mlu); 8462 if (loaded++ == 0) { 8463 t4_sge_modload(); 8464 sx_init(&t4_list_lock, "T4/T5 adapters"); 8465 SLIST_INIT(&t4_list); 8466 #ifdef TCP_OFFLOAD 8467 sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); 8468 SLIST_INIT(&t4_uld_list); 8469 #endif 8470 t4_tracer_modload(); 8471 tweak_tunables(); 8472 } 8473 sx_xunlock(&mlu); 8474 break; 8475 8476 case MOD_UNLOAD: 8477 sx_xlock(&mlu); 8478 if (--loaded == 0) { 8479 int tries; 8480 8481 sx_slock(&t4_list_lock); 8482 if (!SLIST_EMPTY(&t4_list)) { 8483 rc = EBUSY; 8484 sx_sunlock(&t4_list_lock); 8485 goto done_unload; 8486 } 8487 #ifdef TCP_OFFLOAD 8488 sx_slock(&t4_uld_list_lock); 8489 if (!SLIST_EMPTY(&t4_uld_list)) { 8490 rc = EBUSY; 8491 sx_sunlock(&t4_uld_list_lock); 8492 sx_sunlock(&t4_list_lock); 8493 goto done_unload; 8494 } 8495 #endif 8496 tries = 0; 8497 while (tries++ < 5 && t4_sge_extfree_refs() != 0) { 8498 uprintf("%ju clusters with custom free routine " 8499 "still is use.\n", t4_sge_extfree_refs()); 8500 pause("t4unload", 2 * hz); 8501 } 8502 #ifdef TCP_OFFLOAD 8503 sx_sunlock(&t4_uld_list_lock); 8504 #endif 8505 sx_sunlock(&t4_list_lock); 8506 8507 if (t4_sge_extfree_refs() == 0) { 8508 t4_tracer_modunload(); 8509 #ifdef TCP_OFFLOAD 8510 sx_destroy(&t4_uld_list_lock); 8511 #endif 8512 sx_destroy(&t4_list_lock); 8513 t4_sge_modunload(); 8514 loaded = 0; 8515 } else { 8516 rc = EBUSY; 8517 loaded++; /* undo earlier decrement */ 8518 } 8519 } 8520 done_unload: 8521 sx_xunlock(&mlu); 8522 break; 8523 } 8524 8525 return (rc); 8526 } 8527 8528 static devclass_t t4_devclass, t5_devclass; 8529 static devclass_t cxgbe_devclass, cxl_devclass; 8530 8531 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 8532 MODULE_VERSION(t4nex, 1); 8533 MODULE_DEPEND(t4nex, firmware, 1, 1, 1); 8534 #ifdef DEV_NETMAP 8535 MODULE_DEPEND(t4nex, netmap, 1, 1, 1); 8536 #endif /* DEV_NETMAP */ 8537 8538 8539 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 8540 MODULE_VERSION(t5nex, 1); 8541 MODULE_DEPEND(t5nex, firmware, 1, 1, 1); 8542 #ifdef DEV_NETMAP 8543 MODULE_DEPEND(t5nex, netmap, 1, 1, 1); 8544 #endif /* DEV_NETMAP */ 8545 8546 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 8547 MODULE_VERSION(cxgbe, 1); 8548 8549 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 8550 MODULE_VERSION(cxl, 1); 8551