1 /*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_inet.h" 32 #include "opt_inet6.h" 33 34 #include <sys/param.h> 35 #include <sys/conf.h> 36 #include <sys/priv.h> 37 #include <sys/kernel.h> 38 #include <sys/bus.h> 39 #include <sys/module.h> 40 #include <sys/malloc.h> 41 #include <sys/queue.h> 42 #include <sys/taskqueue.h> 43 #include <sys/pciio.h> 44 #include <dev/pci/pcireg.h> 45 #include <dev/pci/pcivar.h> 46 #include <dev/pci/pci_private.h> 47 #include <sys/firmware.h> 48 #include <sys/sbuf.h> 49 #include <sys/smp.h> 50 #include <sys/socket.h> 51 #include <sys/sockio.h> 52 #include <sys/sysctl.h> 53 #include <net/ethernet.h> 54 #include <net/if.h> 55 #include <net/if_types.h> 56 #include <net/if_dl.h> 57 #include <net/if_vlan_var.h> 58 #if defined(__i386__) || defined(__amd64__) 59 #include <vm/vm.h> 60 #include <vm/pmap.h> 61 #endif 62 63 #include "common/common.h" 64 #include "common/t4_msg.h" 65 #include "common/t4_regs.h" 66 #include "common/t4_regs_values.h" 67 #include "t4_ioctl.h" 68 #include "t4_l2t.h" 69 #include "t4_mp_ring.h" 70 71 /* T4 bus driver interface */ 72 static int t4_probe(device_t); 73 static int t4_attach(device_t); 74 static int t4_detach(device_t); 75 static device_method_t t4_methods[] = { 76 DEVMETHOD(device_probe, t4_probe), 77 DEVMETHOD(device_attach, t4_attach), 78 DEVMETHOD(device_detach, t4_detach), 79 80 DEVMETHOD_END 81 }; 82 static driver_t t4_driver = { 83 "t4nex", 84 t4_methods, 85 sizeof(struct adapter) 86 }; 87 88 89 /* T4 port (cxgbe) interface */ 90 static int cxgbe_probe(device_t); 91 static int cxgbe_attach(device_t); 92 static int cxgbe_detach(device_t); 93 static device_method_t cxgbe_methods[] = { 94 DEVMETHOD(device_probe, cxgbe_probe), 95 DEVMETHOD(device_attach, cxgbe_attach), 96 DEVMETHOD(device_detach, cxgbe_detach), 97 { 0, 0 } 98 }; 99 static driver_t cxgbe_driver = { 100 "cxgbe", 101 cxgbe_methods, 102 sizeof(struct port_info) 103 }; 104 105 static d_ioctl_t t4_ioctl; 106 static d_open_t t4_open; 107 static d_close_t t4_close; 108 109 static struct cdevsw t4_cdevsw = { 110 .d_version = D_VERSION, 111 .d_flags = 0, 112 .d_open = t4_open, 113 .d_close = t4_close, 114 .d_ioctl = t4_ioctl, 115 .d_name = "t4nex", 116 }; 117 118 /* T5 bus driver interface */ 119 static int t5_probe(device_t); 120 static device_method_t t5_methods[] = { 121 DEVMETHOD(device_probe, t5_probe), 122 DEVMETHOD(device_attach, t4_attach), 123 DEVMETHOD(device_detach, t4_detach), 124 125 DEVMETHOD_END 126 }; 127 static driver_t t5_driver = { 128 "t5nex", 129 t5_methods, 130 sizeof(struct adapter) 131 }; 132 133 134 /* T5 port (cxl) interface */ 135 static driver_t cxl_driver = { 136 "cxl", 137 cxgbe_methods, 138 sizeof(struct port_info) 139 }; 140 141 static struct cdevsw t5_cdevsw = { 142 .d_version = D_VERSION, 143 .d_flags = 0, 144 .d_open = t4_open, 145 .d_close = t4_close, 146 .d_ioctl = t4_ioctl, 147 .d_name = "t5nex", 148 }; 149 150 /* ifnet + media interface */ 151 static void cxgbe_init(void *); 152 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 153 static int cxgbe_transmit(struct ifnet *, struct mbuf *); 154 static void cxgbe_qflush(struct ifnet *); 155 static uint64_t cxgbe_get_counter(struct ifnet *, ift_counter); 156 static int cxgbe_media_change(struct ifnet *); 157 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 158 159 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 160 161 /* 162 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 163 * then ADAPTER_LOCK, then t4_uld_list_lock. 164 */ 165 static struct sx t4_list_lock; 166 SLIST_HEAD(, adapter) t4_list; 167 #ifdef TCP_OFFLOAD 168 static struct sx t4_uld_list_lock; 169 SLIST_HEAD(, uld_info) t4_uld_list; 170 #endif 171 172 /* 173 * Tunables. See tweak_tunables() too. 174 * 175 * Each tunable is set to a default value here if it's known at compile-time. 176 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 177 * provide a reasonable default when the driver is loaded. 178 * 179 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 180 * T5 are under hw.cxl. 181 */ 182 183 /* 184 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 185 */ 186 #define NTXQ_10G 16 187 static int t4_ntxq10g = -1; 188 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 189 190 #define NRXQ_10G 8 191 static int t4_nrxq10g = -1; 192 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 193 194 #define NTXQ_1G 4 195 static int t4_ntxq1g = -1; 196 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 197 198 #define NRXQ_1G 2 199 static int t4_nrxq1g = -1; 200 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 201 202 static int t4_rsrv_noflowq = 0; 203 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq); 204 205 #ifdef TCP_OFFLOAD 206 #define NOFLDTXQ_10G 8 207 static int t4_nofldtxq10g = -1; 208 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 209 210 #define NOFLDRXQ_10G 2 211 static int t4_nofldrxq10g = -1; 212 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 213 214 #define NOFLDTXQ_1G 2 215 static int t4_nofldtxq1g = -1; 216 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 217 218 #define NOFLDRXQ_1G 1 219 static int t4_nofldrxq1g = -1; 220 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 221 #endif 222 223 #ifdef DEV_NETMAP 224 #define NNMTXQ_10G 2 225 static int t4_nnmtxq10g = -1; 226 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g); 227 228 #define NNMRXQ_10G 2 229 static int t4_nnmrxq10g = -1; 230 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g); 231 232 #define NNMTXQ_1G 1 233 static int t4_nnmtxq1g = -1; 234 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g); 235 236 #define NNMRXQ_1G 1 237 static int t4_nnmrxq1g = -1; 238 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g); 239 #endif 240 241 /* 242 * Holdoff parameters for 10G and 1G ports. 243 */ 244 #define TMR_IDX_10G 1 245 static int t4_tmr_idx_10g = TMR_IDX_10G; 246 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 247 248 #define PKTC_IDX_10G (-1) 249 static int t4_pktc_idx_10g = PKTC_IDX_10G; 250 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 251 252 #define TMR_IDX_1G 1 253 static int t4_tmr_idx_1g = TMR_IDX_1G; 254 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 255 256 #define PKTC_IDX_1G (-1) 257 static int t4_pktc_idx_1g = PKTC_IDX_1G; 258 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 259 260 /* 261 * Size (# of entries) of each tx and rx queue. 262 */ 263 static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 264 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 265 266 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 267 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 268 269 /* 270 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 271 */ 272 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 273 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 274 275 /* 276 * Configuration file. 277 */ 278 #define DEFAULT_CF "default" 279 #define FLASH_CF "flash" 280 #define UWIRE_CF "uwire" 281 #define FPGA_CF "fpga" 282 static char t4_cfg_file[32] = DEFAULT_CF; 283 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 284 285 /* 286 * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively). 287 * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them. 288 * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water 289 * mark or when signalled to do so, 0 to never emit PAUSE. 290 */ 291 static int t4_pause_settings = PAUSE_TX | PAUSE_RX; 292 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings); 293 294 /* 295 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 296 * encouraged respectively). 297 */ 298 static unsigned int t4_fw_install = 1; 299 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 300 301 /* 302 * ASIC features that will be used. Disable the ones you don't want so that the 303 * chip resources aren't wasted on features that will not be used. 304 */ 305 static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 306 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 307 308 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 309 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 310 311 static int t4_toecaps_allowed = -1; 312 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 313 314 static int t4_rdmacaps_allowed = 0; 315 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 316 317 static int t4_iscsicaps_allowed = 0; 318 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 319 320 static int t4_fcoecaps_allowed = 0; 321 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 322 323 static int t5_write_combine = 0; 324 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 325 326 struct intrs_and_queues { 327 uint16_t intr_type; /* INTx, MSI, or MSI-X */ 328 uint16_t nirq; /* Total # of vectors */ 329 uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */ 330 uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */ 331 uint16_t ntxq10g; /* # of NIC txq's for each 10G port */ 332 uint16_t nrxq10g; /* # of NIC rxq's for each 10G port */ 333 uint16_t ntxq1g; /* # of NIC txq's for each 1G port */ 334 uint16_t nrxq1g; /* # of NIC rxq's for each 1G port */ 335 uint16_t rsrv_noflowq; /* Flag whether to reserve queue 0 */ 336 #ifdef TCP_OFFLOAD 337 uint16_t nofldtxq10g; /* # of TOE txq's for each 10G port */ 338 uint16_t nofldrxq10g; /* # of TOE rxq's for each 10G port */ 339 uint16_t nofldtxq1g; /* # of TOE txq's for each 1G port */ 340 uint16_t nofldrxq1g; /* # of TOE rxq's for each 1G port */ 341 #endif 342 #ifdef DEV_NETMAP 343 uint16_t nnmtxq10g; /* # of netmap txq's for each 10G port */ 344 uint16_t nnmrxq10g; /* # of netmap rxq's for each 10G port */ 345 uint16_t nnmtxq1g; /* # of netmap txq's for each 1G port */ 346 uint16_t nnmrxq1g; /* # of netmap rxq's for each 1G port */ 347 #endif 348 }; 349 350 struct filter_entry { 351 uint32_t valid:1; /* filter allocated and valid */ 352 uint32_t locked:1; /* filter is administratively locked */ 353 uint32_t pending:1; /* filter action is pending firmware reply */ 354 uint32_t smtidx:8; /* Source MAC Table index for smac */ 355 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 356 357 struct t4_filter_specification fs; 358 }; 359 360 static int map_bars_0_and_4(struct adapter *); 361 static int map_bar_2(struct adapter *); 362 static void setup_memwin(struct adapter *); 363 static int validate_mem_range(struct adapter *, uint32_t, int); 364 static int fwmtype_to_hwmtype(int); 365 static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 366 uint32_t *); 367 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 368 static uint32_t position_memwin(struct adapter *, int, uint32_t); 369 static int cfg_itype_and_nqueues(struct adapter *, int, int, 370 struct intrs_and_queues *); 371 static int prep_firmware(struct adapter *); 372 static int partition_resources(struct adapter *, const struct firmware *, 373 const char *); 374 static int get_params__pre_init(struct adapter *); 375 static int get_params__post_init(struct adapter *); 376 static int set_params__post_init(struct adapter *); 377 static void t4_set_desc(struct adapter *); 378 static void build_medialist(struct port_info *, struct ifmedia *); 379 static int cxgbe_init_synchronized(struct port_info *); 380 static int cxgbe_uninit_synchronized(struct port_info *); 381 static int setup_intr_handlers(struct adapter *); 382 static void quiesce_txq(struct adapter *, struct sge_txq *); 383 static void quiesce_wrq(struct adapter *, struct sge_wrq *); 384 static void quiesce_iq(struct adapter *, struct sge_iq *); 385 static void quiesce_fl(struct adapter *, struct sge_fl *); 386 static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 387 driver_intr_t *, void *, char *); 388 static int t4_free_irq(struct adapter *, struct irq *); 389 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 390 unsigned int); 391 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 392 static void cxgbe_refresh_stats(struct adapter *, struct port_info *); 393 static void cxgbe_tick(void *); 394 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 395 static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 396 struct mbuf *); 397 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 398 static int fw_msg_not_handled(struct adapter *, const __be64 *); 399 static int t4_sysctls(struct adapter *); 400 static int cxgbe_sysctls(struct port_info *); 401 static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 402 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 403 static int sysctl_btphy(SYSCTL_HANDLER_ARGS); 404 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); 405 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 406 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 407 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 408 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 409 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS); 410 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 411 static int sysctl_temperature(SYSCTL_HANDLER_ARGS); 412 #ifdef SBUF_DRAIN 413 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 414 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 415 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 416 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); 417 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); 418 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 419 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 420 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 421 static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 422 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 423 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 424 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 425 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); 426 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 427 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); 428 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 429 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 430 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 431 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 432 static int sysctl_tids(SYSCTL_HANDLER_ARGS); 433 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 434 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 435 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 436 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 437 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 438 #endif 439 static uint32_t fconf_to_mode(uint32_t); 440 static uint32_t mode_to_fconf(uint32_t); 441 static uint32_t fspec_to_fconf(struct t4_filter_specification *); 442 static int get_filter_mode(struct adapter *, uint32_t *); 443 static int set_filter_mode(struct adapter *, uint32_t); 444 static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 445 static int get_filter(struct adapter *, struct t4_filter *); 446 static int set_filter(struct adapter *, struct t4_filter *); 447 static int del_filter(struct adapter *, struct t4_filter *); 448 static void clear_filter(struct filter_entry *); 449 static int set_filter_wr(struct adapter *, int); 450 static int del_filter_wr(struct adapter *, int); 451 static int get_sge_context(struct adapter *, struct t4_sge_context *); 452 static int load_fw(struct adapter *, struct t4_data *); 453 static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 454 static int read_i2c(struct adapter *, struct t4_i2c_data *); 455 static int set_sched_class(struct adapter *, struct t4_sched_params *); 456 static int set_sched_queue(struct adapter *, struct t4_sched_queue *); 457 #ifdef TCP_OFFLOAD 458 static int toe_capability(struct port_info *, int); 459 #endif 460 static int mod_event(module_t, int, void *); 461 462 struct { 463 uint16_t device; 464 char *desc; 465 } t4_pciids[] = { 466 {0xa000, "Chelsio Terminator 4 FPGA"}, 467 {0x4400, "Chelsio T440-dbg"}, 468 {0x4401, "Chelsio T420-CR"}, 469 {0x4402, "Chelsio T422-CR"}, 470 {0x4403, "Chelsio T440-CR"}, 471 {0x4404, "Chelsio T420-BCH"}, 472 {0x4405, "Chelsio T440-BCH"}, 473 {0x4406, "Chelsio T440-CH"}, 474 {0x4407, "Chelsio T420-SO"}, 475 {0x4408, "Chelsio T420-CX"}, 476 {0x4409, "Chelsio T420-BT"}, 477 {0x440a, "Chelsio T404-BT"}, 478 {0x440e, "Chelsio T440-LP-CR"}, 479 }, t5_pciids[] = { 480 {0xb000, "Chelsio Terminator 5 FPGA"}, 481 {0x5400, "Chelsio T580-dbg"}, 482 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ 483 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ 484 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ 485 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ 486 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ 487 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ 488 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ 489 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ 490 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 491 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ 492 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ 493 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ 494 {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */ 495 #ifdef notyet 496 {0x5404, "Chelsio T520-BCH"}, 497 {0x5405, "Chelsio T540-BCH"}, 498 {0x5406, "Chelsio T540-CH"}, 499 {0x5408, "Chelsio T520-CX"}, 500 {0x540b, "Chelsio B520-SR"}, 501 {0x540c, "Chelsio B504-BT"}, 502 {0x540f, "Chelsio Amsterdam"}, 503 {0x5413, "Chelsio T580-CHR"}, 504 #endif 505 }; 506 507 #ifdef TCP_OFFLOAD 508 /* 509 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 510 * exactly the same for both rxq and ofld_rxq. 511 */ 512 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 513 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 514 #endif 515 516 /* No easy way to include t4_msg.h before adapter.h so we check this way */ 517 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 518 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 519 520 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); 521 522 static int 523 t4_probe(device_t dev) 524 { 525 int i; 526 uint16_t v = pci_get_vendor(dev); 527 uint16_t d = pci_get_device(dev); 528 uint8_t f = pci_get_function(dev); 529 530 if (v != PCI_VENDOR_ID_CHELSIO) 531 return (ENXIO); 532 533 /* Attach only to PF0 of the FPGA */ 534 if (d == 0xa000 && f != 0) 535 return (ENXIO); 536 537 for (i = 0; i < nitems(t4_pciids); i++) { 538 if (d == t4_pciids[i].device) { 539 device_set_desc(dev, t4_pciids[i].desc); 540 return (BUS_PROBE_DEFAULT); 541 } 542 } 543 544 return (ENXIO); 545 } 546 547 static int 548 t5_probe(device_t dev) 549 { 550 int i; 551 uint16_t v = pci_get_vendor(dev); 552 uint16_t d = pci_get_device(dev); 553 uint8_t f = pci_get_function(dev); 554 555 if (v != PCI_VENDOR_ID_CHELSIO) 556 return (ENXIO); 557 558 /* Attach only to PF0 of the FPGA */ 559 if (d == 0xb000 && f != 0) 560 return (ENXIO); 561 562 for (i = 0; i < nitems(t5_pciids); i++) { 563 if (d == t5_pciids[i].device) { 564 device_set_desc(dev, t5_pciids[i].desc); 565 return (BUS_PROBE_DEFAULT); 566 } 567 } 568 569 return (ENXIO); 570 } 571 572 static int 573 t4_attach(device_t dev) 574 { 575 struct adapter *sc; 576 int rc = 0, i, n10g, n1g, rqidx, tqidx; 577 struct intrs_and_queues iaq; 578 struct sge *s; 579 #ifdef TCP_OFFLOAD 580 int ofld_rqidx, ofld_tqidx; 581 #endif 582 #ifdef DEV_NETMAP 583 int nm_rqidx, nm_tqidx; 584 #endif 585 586 sc = device_get_softc(dev); 587 sc->dev = dev; 588 589 pci_enable_busmaster(dev); 590 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 591 uint32_t v; 592 593 pci_set_max_read_req(dev, 4096); 594 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 595 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 596 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 597 598 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5); 599 } 600 601 sc->traceq = -1; 602 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); 603 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", 604 device_get_nameunit(dev)); 605 606 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 607 device_get_nameunit(dev)); 608 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 609 sx_xlock(&t4_list_lock); 610 SLIST_INSERT_HEAD(&t4_list, sc, link); 611 sx_xunlock(&t4_list_lock); 612 613 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 614 TAILQ_INIT(&sc->sfl); 615 callout_init(&sc->sfl_callout, 1); 616 617 mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF); 618 619 rc = map_bars_0_and_4(sc); 620 if (rc != 0) 621 goto done; /* error message displayed already */ 622 623 /* 624 * This is the real PF# to which we're attaching. Works from within PCI 625 * passthrough environments too, where pci_get_function() could return a 626 * different PF# depending on the passthrough configuration. We need to 627 * use the real PF# in all our communication with the firmware. 628 */ 629 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 630 sc->mbox = sc->pf; 631 632 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 633 sc->an_handler = an_not_handled; 634 for (i = 0; i < nitems(sc->cpl_handler); i++) 635 sc->cpl_handler[i] = cpl_not_handled; 636 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 637 sc->fw_msg_handler[i] = fw_msg_not_handled; 638 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 639 t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt); 640 t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt); 641 t4_init_sge_cpl_handlers(sc); 642 643 /* Prepare the adapter for operation */ 644 rc = -t4_prep_adapter(sc); 645 if (rc != 0) { 646 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 647 goto done; 648 } 649 650 /* 651 * Do this really early, with the memory windows set up even before the 652 * character device. The userland tool's register i/o and mem read 653 * will work even in "recovery mode". 654 */ 655 setup_memwin(sc); 656 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 657 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 658 device_get_nameunit(dev)); 659 if (sc->cdev == NULL) 660 device_printf(dev, "failed to create nexus char device.\n"); 661 else 662 sc->cdev->si_drv1 = sc; 663 664 /* Go no further if recovery mode has been requested. */ 665 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 666 device_printf(dev, "recovery mode.\n"); 667 goto done; 668 } 669 670 #if defined(__i386__) 671 if ((cpu_feature & CPUID_CX8) == 0) { 672 device_printf(dev, "64 bit atomics not available.\n"); 673 rc = ENOTSUP; 674 goto done; 675 } 676 #endif 677 678 /* Prepare the firmware for operation */ 679 rc = prep_firmware(sc); 680 if (rc != 0) 681 goto done; /* error message displayed already */ 682 683 rc = get_params__post_init(sc); 684 if (rc != 0) 685 goto done; /* error message displayed already */ 686 687 rc = set_params__post_init(sc); 688 if (rc != 0) 689 goto done; /* error message displayed already */ 690 691 rc = map_bar_2(sc); 692 if (rc != 0) 693 goto done; /* error message displayed already */ 694 695 rc = t4_create_dma_tag(sc); 696 if (rc != 0) 697 goto done; /* error message displayed already */ 698 699 /* 700 * First pass over all the ports - allocate VIs and initialize some 701 * basic parameters like mac address, port type, etc. We also figure 702 * out whether a port is 10G or 1G and use that information when 703 * calculating how many interrupts to attempt to allocate. 704 */ 705 n10g = n1g = 0; 706 for_each_port(sc, i) { 707 struct port_info *pi; 708 709 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 710 sc->port[i] = pi; 711 712 /* These must be set before t4_port_init */ 713 pi->adapter = sc; 714 pi->port_id = i; 715 716 /* Allocate the vi and initialize parameters like mac addr */ 717 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 718 if (rc != 0) { 719 device_printf(dev, "unable to initialize port %d: %d\n", 720 i, rc); 721 free(pi, M_CXGBE); 722 sc->port[i] = NULL; 723 goto done; 724 } 725 726 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX); 727 pi->link_cfg.requested_fc |= t4_pause_settings; 728 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX); 729 pi->link_cfg.fc |= t4_pause_settings; 730 731 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 732 if (rc != 0) { 733 device_printf(dev, "port %d l1cfg failed: %d\n", i, rc); 734 free(pi, M_CXGBE); 735 sc->port[i] = NULL; 736 goto done; 737 } 738 739 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 740 device_get_nameunit(dev), i); 741 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 742 sc->chan_map[pi->tx_chan] = i; 743 744 if (is_10G_port(pi) || is_40G_port(pi)) { 745 n10g++; 746 pi->tmr_idx = t4_tmr_idx_10g; 747 pi->pktc_idx = t4_pktc_idx_10g; 748 } else { 749 n1g++; 750 pi->tmr_idx = t4_tmr_idx_1g; 751 pi->pktc_idx = t4_pktc_idx_1g; 752 } 753 754 pi->xact_addr_filt = -1; 755 pi->linkdnrc = -1; 756 757 pi->qsize_rxq = t4_qsize_rxq; 758 pi->qsize_txq = t4_qsize_txq; 759 760 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 761 if (pi->dev == NULL) { 762 device_printf(dev, 763 "failed to add device for port %d.\n", i); 764 rc = ENXIO; 765 goto done; 766 } 767 device_set_softc(pi->dev, pi); 768 } 769 770 /* 771 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 772 */ 773 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 774 if (rc != 0) 775 goto done; /* error message displayed already */ 776 777 sc->intr_type = iaq.intr_type; 778 sc->intr_count = iaq.nirq; 779 780 s = &sc->sge; 781 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 782 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 783 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 784 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 785 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 786 #ifdef TCP_OFFLOAD 787 if (is_offload(sc)) { 788 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 789 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 790 s->neq += s->nofldtxq + s->nofldrxq; 791 s->niq += s->nofldrxq; 792 793 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 794 M_CXGBE, M_ZERO | M_WAITOK); 795 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 796 M_CXGBE, M_ZERO | M_WAITOK); 797 } 798 #endif 799 #ifdef DEV_NETMAP 800 s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g; 801 s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g; 802 s->neq += s->nnmtxq + s->nnmrxq; 803 s->niq += s->nnmrxq; 804 805 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq), 806 M_CXGBE, M_ZERO | M_WAITOK); 807 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq), 808 M_CXGBE, M_ZERO | M_WAITOK); 809 #endif 810 811 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 812 M_ZERO | M_WAITOK); 813 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 814 M_ZERO | M_WAITOK); 815 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 816 M_ZERO | M_WAITOK); 817 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 818 M_ZERO | M_WAITOK); 819 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 820 M_ZERO | M_WAITOK); 821 822 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 823 M_ZERO | M_WAITOK); 824 825 t4_init_l2t(sc, M_WAITOK); 826 827 /* 828 * Second pass over the ports. This time we know the number of rx and 829 * tx queues that each port should get. 830 */ 831 rqidx = tqidx = 0; 832 #ifdef TCP_OFFLOAD 833 ofld_rqidx = ofld_tqidx = 0; 834 #endif 835 #ifdef DEV_NETMAP 836 nm_rqidx = nm_tqidx = 0; 837 #endif 838 for_each_port(sc, i) { 839 struct port_info *pi = sc->port[i]; 840 841 if (pi == NULL) 842 continue; 843 844 pi->first_rxq = rqidx; 845 pi->first_txq = tqidx; 846 if (is_10G_port(pi) || is_40G_port(pi)) { 847 pi->flags |= iaq.intr_flags_10g; 848 pi->nrxq = iaq.nrxq10g; 849 pi->ntxq = iaq.ntxq10g; 850 } else { 851 pi->flags |= iaq.intr_flags_1g; 852 pi->nrxq = iaq.nrxq1g; 853 pi->ntxq = iaq.ntxq1g; 854 } 855 856 if (pi->ntxq > 1) 857 pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0; 858 else 859 pi->rsrv_noflowq = 0; 860 861 rqidx += pi->nrxq; 862 tqidx += pi->ntxq; 863 #ifdef TCP_OFFLOAD 864 if (is_offload(sc)) { 865 pi->first_ofld_rxq = ofld_rqidx; 866 pi->first_ofld_txq = ofld_tqidx; 867 if (is_10G_port(pi) || is_40G_port(pi)) { 868 pi->nofldrxq = iaq.nofldrxq10g; 869 pi->nofldtxq = iaq.nofldtxq10g; 870 } else { 871 pi->nofldrxq = iaq.nofldrxq1g; 872 pi->nofldtxq = iaq.nofldtxq1g; 873 } 874 ofld_rqidx += pi->nofldrxq; 875 ofld_tqidx += pi->nofldtxq; 876 } 877 #endif 878 #ifdef DEV_NETMAP 879 pi->first_nm_rxq = nm_rqidx; 880 pi->first_nm_txq = nm_tqidx; 881 if (is_10G_port(pi) || is_40G_port(pi)) { 882 pi->nnmrxq = iaq.nnmrxq10g; 883 pi->nnmtxq = iaq.nnmtxq10g; 884 } else { 885 pi->nnmrxq = iaq.nnmrxq1g; 886 pi->nnmtxq = iaq.nnmtxq1g; 887 } 888 nm_rqidx += pi->nnmrxq; 889 nm_tqidx += pi->nnmtxq; 890 #endif 891 } 892 893 rc = setup_intr_handlers(sc); 894 if (rc != 0) { 895 device_printf(dev, 896 "failed to setup interrupt handlers: %d\n", rc); 897 goto done; 898 } 899 900 rc = bus_generic_attach(dev); 901 if (rc != 0) { 902 device_printf(dev, 903 "failed to attach all child ports: %d\n", rc); 904 goto done; 905 } 906 907 device_printf(dev, 908 "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 909 sc->params.pci.speed, sc->params.pci.width, sc->params.nports, 910 sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" : 911 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 912 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 913 914 t4_set_desc(sc); 915 916 done: 917 if (rc != 0 && sc->cdev) { 918 /* cdev was created and so cxgbetool works; recover that way. */ 919 device_printf(dev, 920 "error during attach, adapter is now in recovery mode.\n"); 921 rc = 0; 922 } 923 924 if (rc != 0) 925 t4_detach(dev); 926 else 927 t4_sysctls(sc); 928 929 return (rc); 930 } 931 932 /* 933 * Idempotent 934 */ 935 static int 936 t4_detach(device_t dev) 937 { 938 struct adapter *sc; 939 struct port_info *pi; 940 int i, rc; 941 942 sc = device_get_softc(dev); 943 944 if (sc->flags & FULL_INIT_DONE) 945 t4_intr_disable(sc); 946 947 if (sc->cdev) { 948 destroy_dev(sc->cdev); 949 sc->cdev = NULL; 950 } 951 952 rc = bus_generic_detach(dev); 953 if (rc) { 954 device_printf(dev, 955 "failed to detach child devices: %d\n", rc); 956 return (rc); 957 } 958 959 for (i = 0; i < sc->intr_count; i++) 960 t4_free_irq(sc, &sc->irq[i]); 961 962 for (i = 0; i < MAX_NPORTS; i++) { 963 pi = sc->port[i]; 964 if (pi) { 965 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid); 966 if (pi->dev) 967 device_delete_child(dev, pi->dev); 968 969 mtx_destroy(&pi->pi_lock); 970 free(pi, M_CXGBE); 971 } 972 } 973 974 if (sc->flags & FULL_INIT_DONE) 975 adapter_full_uninit(sc); 976 977 if (sc->flags & FW_OK) 978 t4_fw_bye(sc, sc->mbox); 979 980 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 981 pci_release_msi(dev); 982 983 if (sc->regs_res) 984 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 985 sc->regs_res); 986 987 if (sc->udbs_res) 988 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 989 sc->udbs_res); 990 991 if (sc->msix_res) 992 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 993 sc->msix_res); 994 995 if (sc->l2t) 996 t4_free_l2t(sc->l2t); 997 998 #ifdef TCP_OFFLOAD 999 free(sc->sge.ofld_rxq, M_CXGBE); 1000 free(sc->sge.ofld_txq, M_CXGBE); 1001 #endif 1002 #ifdef DEV_NETMAP 1003 free(sc->sge.nm_rxq, M_CXGBE); 1004 free(sc->sge.nm_txq, M_CXGBE); 1005 #endif 1006 free(sc->irq, M_CXGBE); 1007 free(sc->sge.rxq, M_CXGBE); 1008 free(sc->sge.txq, M_CXGBE); 1009 free(sc->sge.ctrlq, M_CXGBE); 1010 free(sc->sge.iqmap, M_CXGBE); 1011 free(sc->sge.eqmap, M_CXGBE); 1012 free(sc->tids.ftid_tab, M_CXGBE); 1013 t4_destroy_dma_tag(sc); 1014 if (mtx_initialized(&sc->sc_lock)) { 1015 sx_xlock(&t4_list_lock); 1016 SLIST_REMOVE(&t4_list, sc, adapter, link); 1017 sx_xunlock(&t4_list_lock); 1018 mtx_destroy(&sc->sc_lock); 1019 } 1020 1021 if (mtx_initialized(&sc->tids.ftid_lock)) 1022 mtx_destroy(&sc->tids.ftid_lock); 1023 if (mtx_initialized(&sc->sfl_lock)) 1024 mtx_destroy(&sc->sfl_lock); 1025 if (mtx_initialized(&sc->ifp_lock)) 1026 mtx_destroy(&sc->ifp_lock); 1027 if (mtx_initialized(&sc->regwin_lock)) 1028 mtx_destroy(&sc->regwin_lock); 1029 1030 bzero(sc, sizeof(*sc)); 1031 1032 return (0); 1033 } 1034 1035 static int 1036 cxgbe_probe(device_t dev) 1037 { 1038 char buf[128]; 1039 struct port_info *pi = device_get_softc(dev); 1040 1041 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 1042 device_set_desc_copy(dev, buf); 1043 1044 return (BUS_PROBE_DEFAULT); 1045 } 1046 1047 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 1048 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 1049 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS) 1050 #define T4_CAP_ENABLE (T4_CAP) 1051 1052 static int 1053 cxgbe_attach(device_t dev) 1054 { 1055 struct port_info *pi = device_get_softc(dev); 1056 struct ifnet *ifp; 1057 char *s; 1058 int n, o; 1059 1060 /* Allocate an ifnet and set it up */ 1061 ifp = if_alloc(IFT_ETHER); 1062 if (ifp == NULL) { 1063 device_printf(dev, "Cannot allocate ifnet\n"); 1064 return (ENOMEM); 1065 } 1066 pi->ifp = ifp; 1067 ifp->if_softc = pi; 1068 1069 callout_init(&pi->tick, 1); 1070 1071 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1072 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1073 1074 ifp->if_init = cxgbe_init; 1075 ifp->if_ioctl = cxgbe_ioctl; 1076 ifp->if_transmit = cxgbe_transmit; 1077 ifp->if_qflush = cxgbe_qflush; 1078 ifp->if_get_counter = cxgbe_get_counter; 1079 1080 ifp->if_capabilities = T4_CAP; 1081 #ifdef TCP_OFFLOAD 1082 if (is_offload(pi->adapter)) 1083 ifp->if_capabilities |= IFCAP_TOE; 1084 #endif 1085 ifp->if_capenable = T4_CAP_ENABLE; 1086 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1087 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1088 1089 ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1090 ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS; 1091 ifp->if_hw_tsomaxsegsize = 65536; 1092 1093 /* Initialize ifmedia for this port */ 1094 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 1095 cxgbe_media_status); 1096 build_medialist(pi, &pi->media); 1097 1098 pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 1099 EVENTHANDLER_PRI_ANY); 1100 1101 ether_ifattach(ifp, pi->hw_addr); 1102 1103 n = 128; 1104 s = malloc(n, M_CXGBE, M_WAITOK); 1105 o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq); 1106 MPASS(n > o); 1107 #ifdef TCP_OFFLOAD 1108 if (is_offload(pi->adapter)) { 1109 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)", 1110 pi->nofldtxq, pi->nofldrxq); 1111 MPASS(n > o); 1112 } 1113 #endif 1114 #ifdef DEV_NETMAP 1115 o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq, 1116 pi->nnmrxq); 1117 MPASS(n > o); 1118 #endif 1119 device_printf(dev, "%s\n", s); 1120 free(s, M_CXGBE); 1121 1122 #ifdef DEV_NETMAP 1123 /* nm_media handled here to keep implementation private to this file */ 1124 ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change, 1125 cxgbe_media_status); 1126 build_medialist(pi, &pi->nm_media); 1127 create_netmap_ifnet(pi); /* logs errors it something fails */ 1128 #endif 1129 cxgbe_sysctls(pi); 1130 1131 return (0); 1132 } 1133 1134 static int 1135 cxgbe_detach(device_t dev) 1136 { 1137 struct port_info *pi = device_get_softc(dev); 1138 struct adapter *sc = pi->adapter; 1139 struct ifnet *ifp = pi->ifp; 1140 1141 /* Tell if_ioctl and if_init that the port is going away */ 1142 ADAPTER_LOCK(sc); 1143 SET_DOOMED(pi); 1144 wakeup(&sc->flags); 1145 while (IS_BUSY(sc)) 1146 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 1147 SET_BUSY(sc); 1148 #ifdef INVARIANTS 1149 sc->last_op = "t4detach"; 1150 sc->last_op_thr = curthread; 1151 #endif 1152 ADAPTER_UNLOCK(sc); 1153 1154 if (pi->flags & HAS_TRACEQ) { 1155 sc->traceq = -1; /* cloner should not create ifnet */ 1156 t4_tracer_port_detach(sc); 1157 } 1158 1159 if (pi->vlan_c) 1160 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c); 1161 1162 PORT_LOCK(pi); 1163 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1164 callout_stop(&pi->tick); 1165 PORT_UNLOCK(pi); 1166 callout_drain(&pi->tick); 1167 1168 /* Let detach proceed even if these fail. */ 1169 cxgbe_uninit_synchronized(pi); 1170 port_full_uninit(pi); 1171 1172 ifmedia_removeall(&pi->media); 1173 ether_ifdetach(pi->ifp); 1174 if_free(pi->ifp); 1175 1176 #ifdef DEV_NETMAP 1177 /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */ 1178 destroy_netmap_ifnet(pi); 1179 #endif 1180 1181 ADAPTER_LOCK(sc); 1182 CLR_BUSY(sc); 1183 wakeup(&sc->flags); 1184 ADAPTER_UNLOCK(sc); 1185 1186 return (0); 1187 } 1188 1189 static void 1190 cxgbe_init(void *arg) 1191 { 1192 struct port_info *pi = arg; 1193 struct adapter *sc = pi->adapter; 1194 1195 if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0) 1196 return; 1197 cxgbe_init_synchronized(pi); 1198 end_synchronized_op(sc, 0); 1199 } 1200 1201 static int 1202 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1203 { 1204 int rc = 0, mtu, flags, can_sleep; 1205 struct port_info *pi = ifp->if_softc; 1206 struct adapter *sc = pi->adapter; 1207 struct ifreq *ifr = (struct ifreq *)data; 1208 uint32_t mask; 1209 1210 switch (cmd) { 1211 case SIOCSIFMTU: 1212 mtu = ifr->ifr_mtu; 1213 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1214 return (EINVAL); 1215 1216 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu"); 1217 if (rc) 1218 return (rc); 1219 ifp->if_mtu = mtu; 1220 if (pi->flags & PORT_INIT_DONE) { 1221 t4_update_fl_bufsize(ifp); 1222 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1223 rc = update_mac_settings(ifp, XGMAC_MTU); 1224 } 1225 end_synchronized_op(sc, 0); 1226 break; 1227 1228 case SIOCSIFFLAGS: 1229 can_sleep = 0; 1230 redo_sifflags: 1231 rc = begin_synchronized_op(sc, pi, 1232 can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg"); 1233 if (rc) 1234 return (rc); 1235 1236 if (ifp->if_flags & IFF_UP) { 1237 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1238 flags = pi->if_flags; 1239 if ((ifp->if_flags ^ flags) & 1240 (IFF_PROMISC | IFF_ALLMULTI)) { 1241 if (can_sleep == 1) { 1242 end_synchronized_op(sc, 0); 1243 can_sleep = 0; 1244 goto redo_sifflags; 1245 } 1246 rc = update_mac_settings(ifp, 1247 XGMAC_PROMISC | XGMAC_ALLMULTI); 1248 } 1249 } else { 1250 if (can_sleep == 0) { 1251 end_synchronized_op(sc, LOCK_HELD); 1252 can_sleep = 1; 1253 goto redo_sifflags; 1254 } 1255 rc = cxgbe_init_synchronized(pi); 1256 } 1257 pi->if_flags = ifp->if_flags; 1258 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1259 if (can_sleep == 0) { 1260 end_synchronized_op(sc, LOCK_HELD); 1261 can_sleep = 1; 1262 goto redo_sifflags; 1263 } 1264 rc = cxgbe_uninit_synchronized(pi); 1265 } 1266 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD); 1267 break; 1268 1269 case SIOCADDMULTI: 1270 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1271 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi"); 1272 if (rc) 1273 return (rc); 1274 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1275 rc = update_mac_settings(ifp, XGMAC_MCADDRS); 1276 end_synchronized_op(sc, LOCK_HELD); 1277 break; 1278 1279 case SIOCSIFCAP: 1280 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap"); 1281 if (rc) 1282 return (rc); 1283 1284 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1285 if (mask & IFCAP_TXCSUM) { 1286 ifp->if_capenable ^= IFCAP_TXCSUM; 1287 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1288 1289 if (IFCAP_TSO4 & ifp->if_capenable && 1290 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1291 ifp->if_capenable &= ~IFCAP_TSO4; 1292 if_printf(ifp, 1293 "tso4 disabled due to -txcsum.\n"); 1294 } 1295 } 1296 if (mask & IFCAP_TXCSUM_IPV6) { 1297 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1298 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1299 1300 if (IFCAP_TSO6 & ifp->if_capenable && 1301 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1302 ifp->if_capenable &= ~IFCAP_TSO6; 1303 if_printf(ifp, 1304 "tso6 disabled due to -txcsum6.\n"); 1305 } 1306 } 1307 if (mask & IFCAP_RXCSUM) 1308 ifp->if_capenable ^= IFCAP_RXCSUM; 1309 if (mask & IFCAP_RXCSUM_IPV6) 1310 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1311 1312 /* 1313 * Note that we leave CSUM_TSO alone (it is always set). The 1314 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1315 * sending a TSO request our way, so it's sufficient to toggle 1316 * IFCAP_TSOx only. 1317 */ 1318 if (mask & IFCAP_TSO4) { 1319 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1320 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1321 if_printf(ifp, "enable txcsum first.\n"); 1322 rc = EAGAIN; 1323 goto fail; 1324 } 1325 ifp->if_capenable ^= IFCAP_TSO4; 1326 } 1327 if (mask & IFCAP_TSO6) { 1328 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1329 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1330 if_printf(ifp, "enable txcsum6 first.\n"); 1331 rc = EAGAIN; 1332 goto fail; 1333 } 1334 ifp->if_capenable ^= IFCAP_TSO6; 1335 } 1336 if (mask & IFCAP_LRO) { 1337 #if defined(INET) || defined(INET6) 1338 int i; 1339 struct sge_rxq *rxq; 1340 1341 ifp->if_capenable ^= IFCAP_LRO; 1342 for_each_rxq(pi, i, rxq) { 1343 if (ifp->if_capenable & IFCAP_LRO) 1344 rxq->iq.flags |= IQ_LRO_ENABLED; 1345 else 1346 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1347 } 1348 #endif 1349 } 1350 #ifdef TCP_OFFLOAD 1351 if (mask & IFCAP_TOE) { 1352 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1353 1354 rc = toe_capability(pi, enable); 1355 if (rc != 0) 1356 goto fail; 1357 1358 ifp->if_capenable ^= mask; 1359 } 1360 #endif 1361 if (mask & IFCAP_VLAN_HWTAGGING) { 1362 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1363 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1364 rc = update_mac_settings(ifp, XGMAC_VLANEX); 1365 } 1366 if (mask & IFCAP_VLAN_MTU) { 1367 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1368 1369 /* Need to find out how to disable auto-mtu-inflation */ 1370 } 1371 if (mask & IFCAP_VLAN_HWTSO) 1372 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1373 if (mask & IFCAP_VLAN_HWCSUM) 1374 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1375 1376 #ifdef VLAN_CAPABILITIES 1377 VLAN_CAPABILITIES(ifp); 1378 #endif 1379 fail: 1380 end_synchronized_op(sc, 0); 1381 break; 1382 1383 case SIOCSIFMEDIA: 1384 case SIOCGIFMEDIA: 1385 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1386 break; 1387 1388 case SIOCGI2C: { 1389 struct ifi2creq i2c; 1390 1391 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); 1392 if (rc != 0) 1393 break; 1394 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) { 1395 rc = EPERM; 1396 break; 1397 } 1398 if (i2c.len > sizeof(i2c.data)) { 1399 rc = EINVAL; 1400 break; 1401 } 1402 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4i2c"); 1403 if (rc) 1404 return (rc); 1405 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr, 1406 i2c.offset, i2c.len, &i2c.data[0]); 1407 end_synchronized_op(sc, 0); 1408 if (rc == 0) 1409 rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); 1410 break; 1411 } 1412 1413 default: 1414 rc = ether_ioctl(ifp, cmd, data); 1415 } 1416 1417 return (rc); 1418 } 1419 1420 static int 1421 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1422 { 1423 struct port_info *pi = ifp->if_softc; 1424 struct adapter *sc = pi->adapter; 1425 struct sge_txq *txq; 1426 void *items[1]; 1427 int rc; 1428 1429 M_ASSERTPKTHDR(m); 1430 MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */ 1431 1432 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1433 m_freem(m); 1434 return (ENETDOWN); 1435 } 1436 1437 rc = parse_pkt(&m); 1438 if (__predict_false(rc != 0)) { 1439 MPASS(m == NULL); /* was freed already */ 1440 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */ 1441 return (rc); 1442 } 1443 1444 /* Select a txq. */ 1445 txq = &sc->sge.txq[pi->first_txq]; 1446 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 1447 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) + 1448 pi->rsrv_noflowq); 1449 1450 items[0] = m; 1451 rc = mp_ring_enqueue(txq->r, items, 1, 4096); 1452 if (__predict_false(rc != 0)) 1453 m_freem(m); 1454 1455 return (rc); 1456 } 1457 1458 static void 1459 cxgbe_qflush(struct ifnet *ifp) 1460 { 1461 struct port_info *pi = ifp->if_softc; 1462 struct sge_txq *txq; 1463 int i; 1464 1465 /* queues do not exist if !PORT_INIT_DONE. */ 1466 if (pi->flags & PORT_INIT_DONE) { 1467 for_each_txq(pi, i, txq) { 1468 TXQ_LOCK(txq); 1469 txq->eq.flags &= ~EQ_ENABLED; 1470 TXQ_UNLOCK(txq); 1471 while (!mp_ring_is_idle(txq->r)) { 1472 mp_ring_check_drainage(txq->r, 0); 1473 pause("qflush", 1); 1474 } 1475 } 1476 } 1477 if_qflush(ifp); 1478 } 1479 1480 static uint64_t 1481 cxgbe_get_counter(struct ifnet *ifp, ift_counter c) 1482 { 1483 struct port_info *pi = ifp->if_softc; 1484 struct adapter *sc = pi->adapter; 1485 struct port_stats *s = &pi->stats; 1486 1487 cxgbe_refresh_stats(sc, pi); 1488 1489 switch (c) { 1490 case IFCOUNTER_IPACKETS: 1491 return (s->rx_frames - s->rx_pause); 1492 1493 case IFCOUNTER_IERRORS: 1494 return (s->rx_jabber + s->rx_runt + s->rx_too_long + 1495 s->rx_fcs_err + s->rx_len_err); 1496 1497 case IFCOUNTER_OPACKETS: 1498 return (s->tx_frames - s->tx_pause); 1499 1500 case IFCOUNTER_OERRORS: 1501 return (s->tx_error_frames); 1502 1503 case IFCOUNTER_IBYTES: 1504 return (s->rx_octets - s->rx_pause * 64); 1505 1506 case IFCOUNTER_OBYTES: 1507 return (s->tx_octets - s->tx_pause * 64); 1508 1509 case IFCOUNTER_IMCASTS: 1510 return (s->rx_mcast_frames - s->rx_pause); 1511 1512 case IFCOUNTER_OMCASTS: 1513 return (s->tx_mcast_frames - s->tx_pause); 1514 1515 case IFCOUNTER_IQDROPS: 1516 return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 1517 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 1518 s->rx_trunc3 + pi->tnl_cong_drops); 1519 1520 case IFCOUNTER_OQDROPS: { 1521 uint64_t drops; 1522 1523 drops = s->tx_drop; 1524 if (pi->flags & PORT_INIT_DONE) { 1525 int i; 1526 struct sge_txq *txq; 1527 1528 for_each_txq(pi, i, txq) 1529 drops += counter_u64_fetch(txq->r->drops); 1530 } 1531 1532 return (drops); 1533 1534 } 1535 1536 default: 1537 return (if_get_counter_default(ifp, c)); 1538 } 1539 } 1540 1541 static int 1542 cxgbe_media_change(struct ifnet *ifp) 1543 { 1544 struct port_info *pi = ifp->if_softc; 1545 1546 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1547 1548 return (EOPNOTSUPP); 1549 } 1550 1551 static void 1552 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1553 { 1554 struct port_info *pi = ifp->if_softc; 1555 struct ifmedia *media = NULL; 1556 struct ifmedia_entry *cur; 1557 int speed = pi->link_cfg.speed; 1558 1559 if (ifp == pi->ifp) 1560 media = &pi->media; 1561 #ifdef DEV_NETMAP 1562 else if (ifp == pi->nm_ifp) 1563 media = &pi->nm_media; 1564 #endif 1565 MPASS(media != NULL); 1566 1567 cur = media->ifm_cur; 1568 1569 ifmr->ifm_status = IFM_AVALID; 1570 if (!pi->link_cfg.link_ok) 1571 return; 1572 1573 ifmr->ifm_status |= IFM_ACTIVE; 1574 1575 /* active and current will differ iff current media is autoselect. */ 1576 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1577 return; 1578 1579 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1580 if (speed == SPEED_10000) 1581 ifmr->ifm_active |= IFM_10G_T; 1582 else if (speed == SPEED_1000) 1583 ifmr->ifm_active |= IFM_1000_T; 1584 else if (speed == SPEED_100) 1585 ifmr->ifm_active |= IFM_100_TX; 1586 else if (speed == SPEED_10) 1587 ifmr->ifm_active |= IFM_10_T; 1588 else 1589 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1590 speed)); 1591 } 1592 1593 void 1594 t4_fatal_err(struct adapter *sc) 1595 { 1596 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1597 t4_intr_disable(sc); 1598 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1599 device_get_nameunit(sc->dev)); 1600 } 1601 1602 static int 1603 map_bars_0_and_4(struct adapter *sc) 1604 { 1605 sc->regs_rid = PCIR_BAR(0); 1606 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1607 &sc->regs_rid, RF_ACTIVE); 1608 if (sc->regs_res == NULL) { 1609 device_printf(sc->dev, "cannot map registers.\n"); 1610 return (ENXIO); 1611 } 1612 sc->bt = rman_get_bustag(sc->regs_res); 1613 sc->bh = rman_get_bushandle(sc->regs_res); 1614 sc->mmio_len = rman_get_size(sc->regs_res); 1615 setbit(&sc->doorbells, DOORBELL_KDB); 1616 1617 sc->msix_rid = PCIR_BAR(4); 1618 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1619 &sc->msix_rid, RF_ACTIVE); 1620 if (sc->msix_res == NULL) { 1621 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1622 return (ENXIO); 1623 } 1624 1625 return (0); 1626 } 1627 1628 static int 1629 map_bar_2(struct adapter *sc) 1630 { 1631 1632 /* 1633 * T4: only iWARP driver uses the userspace doorbells. There is no need 1634 * to map it if RDMA is disabled. 1635 */ 1636 if (is_t4(sc) && sc->rdmacaps == 0) 1637 return (0); 1638 1639 sc->udbs_rid = PCIR_BAR(2); 1640 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1641 &sc->udbs_rid, RF_ACTIVE); 1642 if (sc->udbs_res == NULL) { 1643 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1644 return (ENXIO); 1645 } 1646 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1647 1648 if (is_t5(sc)) { 1649 setbit(&sc->doorbells, DOORBELL_UDB); 1650 #if defined(__i386__) || defined(__amd64__) 1651 if (t5_write_combine) { 1652 int rc; 1653 1654 /* 1655 * Enable write combining on BAR2. This is the 1656 * userspace doorbell BAR and is split into 128B 1657 * (UDBS_SEG_SIZE) doorbell regions, each associated 1658 * with an egress queue. The first 64B has the doorbell 1659 * and the second 64B can be used to submit a tx work 1660 * request with an implicit doorbell. 1661 */ 1662 1663 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1664 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1665 if (rc == 0) { 1666 clrbit(&sc->doorbells, DOORBELL_UDB); 1667 setbit(&sc->doorbells, DOORBELL_WCWR); 1668 setbit(&sc->doorbells, DOORBELL_UDBWC); 1669 } else { 1670 device_printf(sc->dev, 1671 "couldn't enable write combining: %d\n", 1672 rc); 1673 } 1674 1675 t4_write_reg(sc, A_SGE_STAT_CFG, 1676 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1677 } 1678 #endif 1679 } 1680 1681 return (0); 1682 } 1683 1684 static const struct memwin t4_memwin[] = { 1685 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1686 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1687 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1688 }; 1689 1690 static const struct memwin t5_memwin[] = { 1691 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1692 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1693 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1694 }; 1695 1696 static void 1697 setup_memwin(struct adapter *sc) 1698 { 1699 const struct memwin *mw; 1700 int i, n; 1701 uint32_t bar0; 1702 1703 if (is_t4(sc)) { 1704 /* 1705 * Read low 32b of bar0 indirectly via the hardware backdoor 1706 * mechanism. Works from within PCI passthrough environments 1707 * too, where rman_get_start() can return a different value. We 1708 * need to program the T4 memory window decoders with the actual 1709 * addresses that will be coming across the PCIe link. 1710 */ 1711 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1712 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1713 1714 mw = &t4_memwin[0]; 1715 n = nitems(t4_memwin); 1716 } else { 1717 /* T5 uses the relative offset inside the PCIe BAR */ 1718 bar0 = 0; 1719 1720 mw = &t5_memwin[0]; 1721 n = nitems(t5_memwin); 1722 } 1723 1724 for (i = 0; i < n; i++, mw++) { 1725 t4_write_reg(sc, 1726 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 1727 (mw->base + bar0) | V_BIR(0) | 1728 V_WINDOW(ilog2(mw->aperture) - 10)); 1729 } 1730 1731 /* flush */ 1732 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1733 } 1734 1735 /* 1736 * Verify that the memory range specified by the addr/len pair is valid and lies 1737 * entirely within a single region (EDCx or MCx). 1738 */ 1739 static int 1740 validate_mem_range(struct adapter *sc, uint32_t addr, int len) 1741 { 1742 uint32_t em, addr_len, maddr, mlen; 1743 1744 /* Memory can only be accessed in naturally aligned 4 byte units */ 1745 if (addr & 3 || len & 3 || len == 0) 1746 return (EINVAL); 1747 1748 /* Enabled memories */ 1749 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1750 if (em & F_EDRAM0_ENABLE) { 1751 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1752 maddr = G_EDRAM0_BASE(addr_len) << 20; 1753 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1754 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1755 addr + len <= maddr + mlen) 1756 return (0); 1757 } 1758 if (em & F_EDRAM1_ENABLE) { 1759 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1760 maddr = G_EDRAM1_BASE(addr_len) << 20; 1761 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1762 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1763 addr + len <= maddr + mlen) 1764 return (0); 1765 } 1766 if (em & F_EXT_MEM_ENABLE) { 1767 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1768 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1769 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1770 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1771 addr + len <= maddr + mlen) 1772 return (0); 1773 } 1774 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 1775 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1776 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1777 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1778 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1779 addr + len <= maddr + mlen) 1780 return (0); 1781 } 1782 1783 return (EFAULT); 1784 } 1785 1786 static int 1787 fwmtype_to_hwmtype(int mtype) 1788 { 1789 1790 switch (mtype) { 1791 case FW_MEMTYPE_EDC0: 1792 return (MEM_EDC0); 1793 case FW_MEMTYPE_EDC1: 1794 return (MEM_EDC1); 1795 case FW_MEMTYPE_EXTMEM: 1796 return (MEM_MC0); 1797 case FW_MEMTYPE_EXTMEM1: 1798 return (MEM_MC1); 1799 default: 1800 panic("%s: cannot translate fw mtype %d.", __func__, mtype); 1801 } 1802 } 1803 1804 /* 1805 * Verify that the memory range specified by the memtype/offset/len pair is 1806 * valid and lies entirely within the memtype specified. The global address of 1807 * the start of the range is returned in addr. 1808 */ 1809 static int 1810 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 1811 uint32_t *addr) 1812 { 1813 uint32_t em, addr_len, maddr, mlen; 1814 1815 /* Memory can only be accessed in naturally aligned 4 byte units */ 1816 if (off & 3 || len & 3 || len == 0) 1817 return (EINVAL); 1818 1819 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1820 switch (fwmtype_to_hwmtype(mtype)) { 1821 case MEM_EDC0: 1822 if (!(em & F_EDRAM0_ENABLE)) 1823 return (EINVAL); 1824 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1825 maddr = G_EDRAM0_BASE(addr_len) << 20; 1826 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1827 break; 1828 case MEM_EDC1: 1829 if (!(em & F_EDRAM1_ENABLE)) 1830 return (EINVAL); 1831 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1832 maddr = G_EDRAM1_BASE(addr_len) << 20; 1833 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1834 break; 1835 case MEM_MC: 1836 if (!(em & F_EXT_MEM_ENABLE)) 1837 return (EINVAL); 1838 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1839 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1840 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1841 break; 1842 case MEM_MC1: 1843 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 1844 return (EINVAL); 1845 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1846 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1847 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1848 break; 1849 default: 1850 return (EINVAL); 1851 } 1852 1853 if (mlen > 0 && off < mlen && off + len <= mlen) { 1854 *addr = maddr + off; /* global address */ 1855 return (0); 1856 } 1857 1858 return (EFAULT); 1859 } 1860 1861 static void 1862 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 1863 { 1864 const struct memwin *mw; 1865 1866 if (is_t4(sc)) { 1867 KASSERT(win >= 0 && win < nitems(t4_memwin), 1868 ("%s: incorrect memwin# (%d)", __func__, win)); 1869 mw = &t4_memwin[win]; 1870 } else { 1871 KASSERT(win >= 0 && win < nitems(t5_memwin), 1872 ("%s: incorrect memwin# (%d)", __func__, win)); 1873 mw = &t5_memwin[win]; 1874 } 1875 1876 if (base != NULL) 1877 *base = mw->base; 1878 if (aperture != NULL) 1879 *aperture = mw->aperture; 1880 } 1881 1882 /* 1883 * Positions the memory window such that it can be used to access the specified 1884 * address in the chip's address space. The return value is the offset of addr 1885 * from the start of the window. 1886 */ 1887 static uint32_t 1888 position_memwin(struct adapter *sc, int n, uint32_t addr) 1889 { 1890 uint32_t start, pf; 1891 uint32_t reg; 1892 1893 KASSERT(n >= 0 && n <= 3, 1894 ("%s: invalid window %d.", __func__, n)); 1895 KASSERT((addr & 3) == 0, 1896 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 1897 1898 if (is_t4(sc)) { 1899 pf = 0; 1900 start = addr & ~0xf; /* start must be 16B aligned */ 1901 } else { 1902 pf = V_PFNUM(sc->pf); 1903 start = addr & ~0x7f; /* start must be 128B aligned */ 1904 } 1905 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 1906 1907 t4_write_reg(sc, reg, start | pf); 1908 t4_read_reg(sc, reg); 1909 1910 return (addr - start); 1911 } 1912 1913 static int 1914 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1915 struct intrs_and_queues *iaq) 1916 { 1917 int rc, itype, navail, nrxq10g, nrxq1g, n; 1918 int nofldrxq10g = 0, nofldrxq1g = 0; 1919 int nnmrxq10g = 0, nnmrxq1g = 0; 1920 1921 bzero(iaq, sizeof(*iaq)); 1922 1923 iaq->ntxq10g = t4_ntxq10g; 1924 iaq->ntxq1g = t4_ntxq1g; 1925 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1926 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1927 iaq->rsrv_noflowq = t4_rsrv_noflowq; 1928 #ifdef TCP_OFFLOAD 1929 if (is_offload(sc)) { 1930 iaq->nofldtxq10g = t4_nofldtxq10g; 1931 iaq->nofldtxq1g = t4_nofldtxq1g; 1932 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1933 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1934 } 1935 #endif 1936 #ifdef DEV_NETMAP 1937 iaq->nnmtxq10g = t4_nnmtxq10g; 1938 iaq->nnmtxq1g = t4_nnmtxq1g; 1939 iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g; 1940 iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g; 1941 #endif 1942 1943 for (itype = INTR_MSIX; itype; itype >>= 1) { 1944 1945 if ((itype & t4_intr_types) == 0) 1946 continue; /* not allowed */ 1947 1948 if (itype == INTR_MSIX) 1949 navail = pci_msix_count(sc->dev); 1950 else if (itype == INTR_MSI) 1951 navail = pci_msi_count(sc->dev); 1952 else 1953 navail = 1; 1954 restart: 1955 if (navail == 0) 1956 continue; 1957 1958 iaq->intr_type = itype; 1959 iaq->intr_flags_10g = 0; 1960 iaq->intr_flags_1g = 0; 1961 1962 /* 1963 * Best option: an interrupt vector for errors, one for the 1964 * firmware event queue, and one for every rxq (NIC, TOE, and 1965 * netmap). 1966 */ 1967 iaq->nirq = T4_EXTRA_INTR; 1968 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g); 1969 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g); 1970 if (iaq->nirq <= navail && 1971 (itype != INTR_MSI || powerof2(iaq->nirq))) { 1972 iaq->intr_flags_10g = INTR_ALL; 1973 iaq->intr_flags_1g = INTR_ALL; 1974 goto allocate; 1975 } 1976 1977 /* 1978 * Second best option: a vector for errors, one for the firmware 1979 * event queue, and vectors for either all the NIC rx queues or 1980 * all the TOE rx queues. The queues that don't get vectors 1981 * will forward their interrupts to those that do. 1982 * 1983 * Note: netmap rx queues cannot be created early and so they 1984 * can't be setup to receive forwarded interrupts for others. 1985 */ 1986 iaq->nirq = T4_EXTRA_INTR; 1987 if (nrxq10g >= nofldrxq10g) { 1988 iaq->intr_flags_10g = INTR_RXQ; 1989 iaq->nirq += n10g * nrxq10g; 1990 #ifdef DEV_NETMAP 1991 iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g); 1992 #endif 1993 } else { 1994 iaq->intr_flags_10g = INTR_OFLD_RXQ; 1995 iaq->nirq += n10g * nofldrxq10g; 1996 #ifdef DEV_NETMAP 1997 iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g); 1998 #endif 1999 } 2000 if (nrxq1g >= nofldrxq1g) { 2001 iaq->intr_flags_1g = INTR_RXQ; 2002 iaq->nirq += n1g * nrxq1g; 2003 #ifdef DEV_NETMAP 2004 iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g); 2005 #endif 2006 } else { 2007 iaq->intr_flags_1g = INTR_OFLD_RXQ; 2008 iaq->nirq += n1g * nofldrxq1g; 2009 #ifdef DEV_NETMAP 2010 iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g); 2011 #endif 2012 } 2013 if (iaq->nirq <= navail && 2014 (itype != INTR_MSI || powerof2(iaq->nirq))) 2015 goto allocate; 2016 2017 /* 2018 * Next best option: an interrupt vector for errors, one for the 2019 * firmware event queue, and at least one per port. At this 2020 * point we know we'll have to downsize nrxq and/or nofldrxq 2021 * and/or nnmrxq to fit what's available to us. 2022 */ 2023 iaq->nirq = T4_EXTRA_INTR; 2024 iaq->nirq += n10g + n1g; 2025 if (iaq->nirq <= navail) { 2026 int leftover = navail - iaq->nirq; 2027 2028 if (n10g > 0) { 2029 int target = max(nrxq10g, nofldrxq10g); 2030 2031 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ? 2032 INTR_RXQ : INTR_OFLD_RXQ; 2033 2034 n = 1; 2035 while (n < target && leftover >= n10g) { 2036 leftover -= n10g; 2037 iaq->nirq += n10g; 2038 n++; 2039 } 2040 iaq->nrxq10g = min(n, nrxq10g); 2041 #ifdef TCP_OFFLOAD 2042 iaq->nofldrxq10g = min(n, nofldrxq10g); 2043 #endif 2044 #ifdef DEV_NETMAP 2045 iaq->nnmrxq10g = min(n, nnmrxq10g); 2046 #endif 2047 } 2048 2049 if (n1g > 0) { 2050 int target = max(nrxq1g, nofldrxq1g); 2051 2052 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ? 2053 INTR_RXQ : INTR_OFLD_RXQ; 2054 2055 n = 1; 2056 while (n < target && leftover >= n1g) { 2057 leftover -= n1g; 2058 iaq->nirq += n1g; 2059 n++; 2060 } 2061 iaq->nrxq1g = min(n, nrxq1g); 2062 #ifdef TCP_OFFLOAD 2063 iaq->nofldrxq1g = min(n, nofldrxq1g); 2064 #endif 2065 #ifdef DEV_NETMAP 2066 iaq->nnmrxq1g = min(n, nnmrxq1g); 2067 #endif 2068 } 2069 2070 if (itype != INTR_MSI || powerof2(iaq->nirq)) 2071 goto allocate; 2072 } 2073 2074 /* 2075 * Least desirable option: one interrupt vector for everything. 2076 */ 2077 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 2078 iaq->intr_flags_10g = iaq->intr_flags_1g = 0; 2079 #ifdef TCP_OFFLOAD 2080 if (is_offload(sc)) 2081 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 2082 #endif 2083 #ifdef DEV_NETMAP 2084 iaq->nnmrxq10g = iaq->nnmrxq1g = 1; 2085 #endif 2086 2087 allocate: 2088 navail = iaq->nirq; 2089 rc = 0; 2090 if (itype == INTR_MSIX) 2091 rc = pci_alloc_msix(sc->dev, &navail); 2092 else if (itype == INTR_MSI) 2093 rc = pci_alloc_msi(sc->dev, &navail); 2094 2095 if (rc == 0) { 2096 if (navail == iaq->nirq) 2097 return (0); 2098 2099 /* 2100 * Didn't get the number requested. Use whatever number 2101 * the kernel is willing to allocate (it's in navail). 2102 */ 2103 device_printf(sc->dev, "fewer vectors than requested, " 2104 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 2105 itype, iaq->nirq, navail); 2106 pci_release_msi(sc->dev); 2107 goto restart; 2108 } 2109 2110 device_printf(sc->dev, 2111 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 2112 itype, rc, iaq->nirq, navail); 2113 } 2114 2115 device_printf(sc->dev, 2116 "failed to find a usable interrupt type. " 2117 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 2118 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 2119 2120 return (ENXIO); 2121 } 2122 2123 #define FW_VERSION(chip) ( \ 2124 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ 2125 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ 2126 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ 2127 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) 2128 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) 2129 2130 struct fw_info { 2131 uint8_t chip; 2132 char *kld_name; 2133 char *fw_mod_name; 2134 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 2135 } fw_info[] = { 2136 { 2137 .chip = CHELSIO_T4, 2138 .kld_name = "t4fw_cfg", 2139 .fw_mod_name = "t4fw", 2140 .fw_hdr = { 2141 .chip = FW_HDR_CHIP_T4, 2142 .fw_ver = htobe32_const(FW_VERSION(T4)), 2143 .intfver_nic = FW_INTFVER(T4, NIC), 2144 .intfver_vnic = FW_INTFVER(T4, VNIC), 2145 .intfver_ofld = FW_INTFVER(T4, OFLD), 2146 .intfver_ri = FW_INTFVER(T4, RI), 2147 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 2148 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 2149 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 2150 .intfver_fcoe = FW_INTFVER(T4, FCOE), 2151 }, 2152 }, { 2153 .chip = CHELSIO_T5, 2154 .kld_name = "t5fw_cfg", 2155 .fw_mod_name = "t5fw", 2156 .fw_hdr = { 2157 .chip = FW_HDR_CHIP_T5, 2158 .fw_ver = htobe32_const(FW_VERSION(T5)), 2159 .intfver_nic = FW_INTFVER(T5, NIC), 2160 .intfver_vnic = FW_INTFVER(T5, VNIC), 2161 .intfver_ofld = FW_INTFVER(T5, OFLD), 2162 .intfver_ri = FW_INTFVER(T5, RI), 2163 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 2164 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 2165 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 2166 .intfver_fcoe = FW_INTFVER(T5, FCOE), 2167 }, 2168 } 2169 }; 2170 2171 static struct fw_info * 2172 find_fw_info(int chip) 2173 { 2174 int i; 2175 2176 for (i = 0; i < nitems(fw_info); i++) { 2177 if (fw_info[i].chip == chip) 2178 return (&fw_info[i]); 2179 } 2180 return (NULL); 2181 } 2182 2183 /* 2184 * Is the given firmware API compatible with the one the driver was compiled 2185 * with? 2186 */ 2187 static int 2188 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 2189 { 2190 2191 /* short circuit if it's the exact same firmware version */ 2192 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 2193 return (1); 2194 2195 /* 2196 * XXX: Is this too conservative? Perhaps I should limit this to the 2197 * features that are supported in the driver. 2198 */ 2199 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 2200 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 2201 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 2202 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 2203 return (1); 2204 #undef SAME_INTF 2205 2206 return (0); 2207 } 2208 2209 /* 2210 * The firmware in the KLD is usable, but should it be installed? This routine 2211 * explains itself in detail if it indicates the KLD firmware should be 2212 * installed. 2213 */ 2214 static int 2215 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 2216 { 2217 const char *reason; 2218 2219 if (!card_fw_usable) { 2220 reason = "incompatible or unusable"; 2221 goto install; 2222 } 2223 2224 if (k > c) { 2225 reason = "older than the version bundled with this driver"; 2226 goto install; 2227 } 2228 2229 if (t4_fw_install == 2 && k != c) { 2230 reason = "different than the version bundled with this driver"; 2231 goto install; 2232 } 2233 2234 return (0); 2235 2236 install: 2237 if (t4_fw_install == 0) { 2238 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2239 "but the driver is prohibited from installing a different " 2240 "firmware on the card.\n", 2241 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2242 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 2243 2244 return (0); 2245 } 2246 2247 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2248 "installing firmware %u.%u.%u.%u on card.\n", 2249 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2250 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 2251 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2252 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2253 2254 return (1); 2255 } 2256 /* 2257 * Establish contact with the firmware and determine if we are the master driver 2258 * or not, and whether we are responsible for chip initialization. 2259 */ 2260 static int 2261 prep_firmware(struct adapter *sc) 2262 { 2263 const struct firmware *fw = NULL, *default_cfg; 2264 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 2265 enum dev_state state; 2266 struct fw_info *fw_info; 2267 struct fw_hdr *card_fw; /* fw on the card */ 2268 const struct fw_hdr *kld_fw; /* fw in the KLD */ 2269 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 2270 against */ 2271 2272 /* Contact firmware. */ 2273 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 2274 if (rc < 0 || state == DEV_STATE_ERR) { 2275 rc = -rc; 2276 device_printf(sc->dev, 2277 "failed to connect to the firmware: %d, %d.\n", rc, state); 2278 return (rc); 2279 } 2280 pf = rc; 2281 if (pf == sc->mbox) 2282 sc->flags |= MASTER_PF; 2283 else if (state == DEV_STATE_UNINIT) { 2284 /* 2285 * We didn't get to be the master so we definitely won't be 2286 * configuring the chip. It's a bug if someone else hasn't 2287 * configured it already. 2288 */ 2289 device_printf(sc->dev, "couldn't be master(%d), " 2290 "device not already initialized either(%d).\n", rc, state); 2291 return (EDOOFUS); 2292 } 2293 2294 /* This is the firmware whose headers the driver was compiled against */ 2295 fw_info = find_fw_info(chip_id(sc)); 2296 if (fw_info == NULL) { 2297 device_printf(sc->dev, 2298 "unable to look up firmware information for chip %d.\n", 2299 chip_id(sc)); 2300 return (EINVAL); 2301 } 2302 drv_fw = &fw_info->fw_hdr; 2303 2304 /* 2305 * The firmware KLD contains many modules. The KLD name is also the 2306 * name of the module that contains the default config file. 2307 */ 2308 default_cfg = firmware_get(fw_info->kld_name); 2309 2310 /* Read the header of the firmware on the card */ 2311 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 2312 rc = -t4_read_flash(sc, FLASH_FW_START, 2313 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 2314 if (rc == 0) 2315 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 2316 else { 2317 device_printf(sc->dev, 2318 "Unable to read card's firmware header: %d\n", rc); 2319 card_fw_usable = 0; 2320 } 2321 2322 /* This is the firmware in the KLD */ 2323 fw = firmware_get(fw_info->fw_mod_name); 2324 if (fw != NULL) { 2325 kld_fw = (const void *)fw->data; 2326 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 2327 } else { 2328 kld_fw = NULL; 2329 kld_fw_usable = 0; 2330 } 2331 2332 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2333 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) { 2334 /* 2335 * Common case: the firmware on the card is an exact match and 2336 * the KLD is an exact match too, or the KLD is 2337 * absent/incompatible. Note that t4_fw_install = 2 is ignored 2338 * here -- use cxgbetool loadfw if you want to reinstall the 2339 * same firmware as the one on the card. 2340 */ 2341 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2342 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2343 be32toh(card_fw->fw_ver))) { 2344 2345 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 2346 if (rc != 0) { 2347 device_printf(sc->dev, 2348 "failed to install firmware: %d\n", rc); 2349 goto done; 2350 } 2351 2352 /* Installed successfully, update the cached header too. */ 2353 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2354 card_fw_usable = 1; 2355 need_fw_reset = 0; /* already reset as part of load_fw */ 2356 } 2357 2358 if (!card_fw_usable) { 2359 uint32_t d, c, k; 2360 2361 d = ntohl(drv_fw->fw_ver); 2362 c = ntohl(card_fw->fw_ver); 2363 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2364 2365 device_printf(sc->dev, "Cannot find a usable firmware: " 2366 "fw_install %d, chip state %d, " 2367 "driver compiled with %d.%d.%d.%d, " 2368 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2369 t4_fw_install, state, 2370 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2371 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2372 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2373 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2374 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2375 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2376 rc = EINVAL; 2377 goto done; 2378 } 2379 2380 /* We're using whatever's on the card and it's known to be good. */ 2381 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2382 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2383 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2384 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2385 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2386 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2387 t4_get_tp_version(sc, &sc->params.tp_vers); 2388 2389 /* Reset device */ 2390 if (need_fw_reset && 2391 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2392 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2393 if (rc != ETIMEDOUT && rc != EIO) 2394 t4_fw_bye(sc, sc->mbox); 2395 goto done; 2396 } 2397 sc->flags |= FW_OK; 2398 2399 rc = get_params__pre_init(sc); 2400 if (rc != 0) 2401 goto done; /* error message displayed already */ 2402 2403 /* Partition adapter resources as specified in the config file. */ 2404 if (state == DEV_STATE_UNINIT) { 2405 2406 KASSERT(sc->flags & MASTER_PF, 2407 ("%s: trying to change chip settings when not master.", 2408 __func__)); 2409 2410 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2411 if (rc != 0) 2412 goto done; /* error message displayed already */ 2413 2414 t4_tweak_chip_settings(sc); 2415 2416 /* get basic stuff going */ 2417 rc = -t4_fw_initialize(sc, sc->mbox); 2418 if (rc != 0) { 2419 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2420 goto done; 2421 } 2422 } else { 2423 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2424 sc->cfcsum = 0; 2425 } 2426 2427 done: 2428 free(card_fw, M_CXGBE); 2429 if (fw != NULL) 2430 firmware_put(fw, FIRMWARE_UNLOAD); 2431 if (default_cfg != NULL) 2432 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2433 2434 return (rc); 2435 } 2436 2437 #define FW_PARAM_DEV(param) \ 2438 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2439 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2440 #define FW_PARAM_PFVF(param) \ 2441 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2442 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2443 2444 /* 2445 * Partition chip resources for use between various PFs, VFs, etc. 2446 */ 2447 static int 2448 partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2449 const char *name_prefix) 2450 { 2451 const struct firmware *cfg = NULL; 2452 int rc = 0; 2453 struct fw_caps_config_cmd caps; 2454 uint32_t mtype, moff, finicsum, cfcsum; 2455 2456 /* 2457 * Figure out what configuration file to use. Pick the default config 2458 * file for the card if the user hasn't specified one explicitly. 2459 */ 2460 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2461 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2462 /* Card specific overrides go here. */ 2463 if (pci_get_device(sc->dev) == 0x440a) 2464 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2465 if (is_fpga(sc)) 2466 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2467 } 2468 2469 /* 2470 * We need to load another module if the profile is anything except 2471 * "default" or "flash". 2472 */ 2473 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2474 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2475 char s[32]; 2476 2477 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2478 cfg = firmware_get(s); 2479 if (cfg == NULL) { 2480 if (default_cfg != NULL) { 2481 device_printf(sc->dev, 2482 "unable to load module \"%s\" for " 2483 "configuration profile \"%s\", will use " 2484 "the default config file instead.\n", 2485 s, sc->cfg_file); 2486 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2487 "%s", DEFAULT_CF); 2488 } else { 2489 device_printf(sc->dev, 2490 "unable to load module \"%s\" for " 2491 "configuration profile \"%s\", will use " 2492 "the config file on the card's flash " 2493 "instead.\n", s, sc->cfg_file); 2494 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2495 "%s", FLASH_CF); 2496 } 2497 } 2498 } 2499 2500 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2501 default_cfg == NULL) { 2502 device_printf(sc->dev, 2503 "default config file not available, will use the config " 2504 "file on the card's flash instead.\n"); 2505 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2506 } 2507 2508 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2509 u_int cflen, i, n; 2510 const uint32_t *cfdata; 2511 uint32_t param, val, addr, off, mw_base, mw_aperture; 2512 2513 KASSERT(cfg != NULL || default_cfg != NULL, 2514 ("%s: no config to upload", __func__)); 2515 2516 /* 2517 * Ask the firmware where it wants us to upload the config file. 2518 */ 2519 param = FW_PARAM_DEV(CF); 2520 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2521 if (rc != 0) { 2522 /* No support for config file? Shouldn't happen. */ 2523 device_printf(sc->dev, 2524 "failed to query config file location: %d.\n", rc); 2525 goto done; 2526 } 2527 mtype = G_FW_PARAMS_PARAM_Y(val); 2528 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2529 2530 /* 2531 * XXX: sheer laziness. We deliberately added 4 bytes of 2532 * useless stuffing/comments at the end of the config file so 2533 * it's ok to simply throw away the last remaining bytes when 2534 * the config file is not an exact multiple of 4. This also 2535 * helps with the validate_mt_off_len check. 2536 */ 2537 if (cfg != NULL) { 2538 cflen = cfg->datasize & ~3; 2539 cfdata = cfg->data; 2540 } else { 2541 cflen = default_cfg->datasize & ~3; 2542 cfdata = default_cfg->data; 2543 } 2544 2545 if (cflen > FLASH_CFG_MAX_SIZE) { 2546 device_printf(sc->dev, 2547 "config file too long (%d, max allowed is %d). " 2548 "Will try to use the config on the card, if any.\n", 2549 cflen, FLASH_CFG_MAX_SIZE); 2550 goto use_config_on_flash; 2551 } 2552 2553 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2554 if (rc != 0) { 2555 device_printf(sc->dev, 2556 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2557 "Will try to use the config on the card, if any.\n", 2558 __func__, mtype, moff, cflen, rc); 2559 goto use_config_on_flash; 2560 } 2561 2562 memwin_info(sc, 2, &mw_base, &mw_aperture); 2563 while (cflen) { 2564 off = position_memwin(sc, 2, addr); 2565 n = min(cflen, mw_aperture - off); 2566 for (i = 0; i < n; i += 4) 2567 t4_write_reg(sc, mw_base + off + i, *cfdata++); 2568 cflen -= n; 2569 addr += n; 2570 } 2571 } else { 2572 use_config_on_flash: 2573 mtype = FW_MEMTYPE_FLASH; 2574 moff = t4_flash_cfg_addr(sc); 2575 } 2576 2577 bzero(&caps, sizeof(caps)); 2578 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2579 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2580 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2581 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2582 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2583 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2584 if (rc != 0) { 2585 device_printf(sc->dev, 2586 "failed to pre-process config file: %d " 2587 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2588 goto done; 2589 } 2590 2591 finicsum = be32toh(caps.finicsum); 2592 cfcsum = be32toh(caps.cfcsum); 2593 if (finicsum != cfcsum) { 2594 device_printf(sc->dev, 2595 "WARNING: config file checksum mismatch: %08x %08x\n", 2596 finicsum, cfcsum); 2597 } 2598 sc->cfcsum = cfcsum; 2599 2600 #define LIMIT_CAPS(x) do { \ 2601 caps.x &= htobe16(t4_##x##_allowed); \ 2602 } while (0) 2603 2604 /* 2605 * Let the firmware know what features will (not) be used so it can tune 2606 * things accordingly. 2607 */ 2608 LIMIT_CAPS(linkcaps); 2609 LIMIT_CAPS(niccaps); 2610 LIMIT_CAPS(toecaps); 2611 LIMIT_CAPS(rdmacaps); 2612 LIMIT_CAPS(iscsicaps); 2613 LIMIT_CAPS(fcoecaps); 2614 #undef LIMIT_CAPS 2615 2616 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2617 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2618 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2619 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2620 if (rc != 0) { 2621 device_printf(sc->dev, 2622 "failed to process config file: %d.\n", rc); 2623 } 2624 done: 2625 if (cfg != NULL) 2626 firmware_put(cfg, FIRMWARE_UNLOAD); 2627 return (rc); 2628 } 2629 2630 /* 2631 * Retrieve parameters that are needed (or nice to have) very early. 2632 */ 2633 static int 2634 get_params__pre_init(struct adapter *sc) 2635 { 2636 int rc; 2637 uint32_t param[2], val[2]; 2638 struct fw_devlog_cmd cmd; 2639 struct devlog_params *dlog = &sc->params.devlog; 2640 2641 param[0] = FW_PARAM_DEV(PORTVEC); 2642 param[1] = FW_PARAM_DEV(CCLK); 2643 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2644 if (rc != 0) { 2645 device_printf(sc->dev, 2646 "failed to query parameters (pre_init): %d.\n", rc); 2647 return (rc); 2648 } 2649 2650 sc->params.portvec = val[0]; 2651 sc->params.nports = bitcount32(val[0]); 2652 sc->params.vpd.cclk = val[1]; 2653 2654 /* Read device log parameters. */ 2655 bzero(&cmd, sizeof(cmd)); 2656 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2657 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2658 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2659 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2660 if (rc != 0) { 2661 device_printf(sc->dev, 2662 "failed to get devlog parameters: %d.\n", rc); 2663 bzero(dlog, sizeof (*dlog)); 2664 rc = 0; /* devlog isn't critical for device operation */ 2665 } else { 2666 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2667 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2668 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2669 dlog->size = be32toh(cmd.memsize_devlog); 2670 } 2671 2672 return (rc); 2673 } 2674 2675 /* 2676 * Retrieve various parameters that are of interest to the driver. The device 2677 * has been initialized by the firmware at this point. 2678 */ 2679 static int 2680 get_params__post_init(struct adapter *sc) 2681 { 2682 int rc; 2683 uint32_t param[7], val[7]; 2684 struct fw_caps_config_cmd caps; 2685 2686 param[0] = FW_PARAM_PFVF(IQFLINT_START); 2687 param[1] = FW_PARAM_PFVF(EQ_START); 2688 param[2] = FW_PARAM_PFVF(FILTER_START); 2689 param[3] = FW_PARAM_PFVF(FILTER_END); 2690 param[4] = FW_PARAM_PFVF(L2T_START); 2691 param[5] = FW_PARAM_PFVF(L2T_END); 2692 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2693 if (rc != 0) { 2694 device_printf(sc->dev, 2695 "failed to query parameters (post_init): %d.\n", rc); 2696 return (rc); 2697 } 2698 2699 sc->sge.iq_start = val[0]; 2700 sc->sge.eq_start = val[1]; 2701 sc->tids.ftid_base = val[2]; 2702 sc->tids.nftids = val[3] - val[2] + 1; 2703 sc->params.ftid_min = val[2]; 2704 sc->params.ftid_max = val[3]; 2705 sc->vres.l2t.start = val[4]; 2706 sc->vres.l2t.size = val[5] - val[4] + 1; 2707 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2708 ("%s: L2 table size (%u) larger than expected (%u)", 2709 __func__, sc->vres.l2t.size, L2T_SIZE)); 2710 2711 /* get capabilites */ 2712 bzero(&caps, sizeof(caps)); 2713 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2714 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2715 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2716 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2717 if (rc != 0) { 2718 device_printf(sc->dev, 2719 "failed to get card capabilities: %d.\n", rc); 2720 return (rc); 2721 } 2722 2723 #define READ_CAPS(x) do { \ 2724 sc->x = htobe16(caps.x); \ 2725 } while (0) 2726 READ_CAPS(linkcaps); 2727 READ_CAPS(niccaps); 2728 READ_CAPS(toecaps); 2729 READ_CAPS(rdmacaps); 2730 READ_CAPS(iscsicaps); 2731 READ_CAPS(fcoecaps); 2732 2733 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { 2734 param[0] = FW_PARAM_PFVF(ETHOFLD_START); 2735 param[1] = FW_PARAM_PFVF(ETHOFLD_END); 2736 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2737 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); 2738 if (rc != 0) { 2739 device_printf(sc->dev, 2740 "failed to query NIC parameters: %d.\n", rc); 2741 return (rc); 2742 } 2743 sc->tids.etid_base = val[0]; 2744 sc->params.etid_min = val[0]; 2745 sc->tids.netids = val[1] - val[0] + 1; 2746 sc->params.netids = sc->tids.netids; 2747 sc->params.eo_wr_cred = val[2]; 2748 sc->params.ethoffload = 1; 2749 } 2750 2751 if (sc->toecaps) { 2752 /* query offload-related parameters */ 2753 param[0] = FW_PARAM_DEV(NTID); 2754 param[1] = FW_PARAM_PFVF(SERVER_START); 2755 param[2] = FW_PARAM_PFVF(SERVER_END); 2756 param[3] = FW_PARAM_PFVF(TDDP_START); 2757 param[4] = FW_PARAM_PFVF(TDDP_END); 2758 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2759 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2760 if (rc != 0) { 2761 device_printf(sc->dev, 2762 "failed to query TOE parameters: %d.\n", rc); 2763 return (rc); 2764 } 2765 sc->tids.ntids = val[0]; 2766 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 2767 sc->tids.stid_base = val[1]; 2768 sc->tids.nstids = val[2] - val[1] + 1; 2769 sc->vres.ddp.start = val[3]; 2770 sc->vres.ddp.size = val[4] - val[3] + 1; 2771 sc->params.ofldq_wr_cred = val[5]; 2772 sc->params.offload = 1; 2773 } 2774 if (sc->rdmacaps) { 2775 param[0] = FW_PARAM_PFVF(STAG_START); 2776 param[1] = FW_PARAM_PFVF(STAG_END); 2777 param[2] = FW_PARAM_PFVF(RQ_START); 2778 param[3] = FW_PARAM_PFVF(RQ_END); 2779 param[4] = FW_PARAM_PFVF(PBL_START); 2780 param[5] = FW_PARAM_PFVF(PBL_END); 2781 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2782 if (rc != 0) { 2783 device_printf(sc->dev, 2784 "failed to query RDMA parameters(1): %d.\n", rc); 2785 return (rc); 2786 } 2787 sc->vres.stag.start = val[0]; 2788 sc->vres.stag.size = val[1] - val[0] + 1; 2789 sc->vres.rq.start = val[2]; 2790 sc->vres.rq.size = val[3] - val[2] + 1; 2791 sc->vres.pbl.start = val[4]; 2792 sc->vres.pbl.size = val[5] - val[4] + 1; 2793 2794 param[0] = FW_PARAM_PFVF(SQRQ_START); 2795 param[1] = FW_PARAM_PFVF(SQRQ_END); 2796 param[2] = FW_PARAM_PFVF(CQ_START); 2797 param[3] = FW_PARAM_PFVF(CQ_END); 2798 param[4] = FW_PARAM_PFVF(OCQ_START); 2799 param[5] = FW_PARAM_PFVF(OCQ_END); 2800 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2801 if (rc != 0) { 2802 device_printf(sc->dev, 2803 "failed to query RDMA parameters(2): %d.\n", rc); 2804 return (rc); 2805 } 2806 sc->vres.qp.start = val[0]; 2807 sc->vres.qp.size = val[1] - val[0] + 1; 2808 sc->vres.cq.start = val[2]; 2809 sc->vres.cq.size = val[3] - val[2] + 1; 2810 sc->vres.ocq.start = val[4]; 2811 sc->vres.ocq.size = val[5] - val[4] + 1; 2812 } 2813 if (sc->iscsicaps) { 2814 param[0] = FW_PARAM_PFVF(ISCSI_START); 2815 param[1] = FW_PARAM_PFVF(ISCSI_END); 2816 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2817 if (rc != 0) { 2818 device_printf(sc->dev, 2819 "failed to query iSCSI parameters: %d.\n", rc); 2820 return (rc); 2821 } 2822 sc->vres.iscsi.start = val[0]; 2823 sc->vres.iscsi.size = val[1] - val[0] + 1; 2824 } 2825 2826 /* 2827 * We've got the params we wanted to query via the firmware. Now grab 2828 * some others directly from the chip. 2829 */ 2830 rc = t4_read_chip_settings(sc); 2831 2832 return (rc); 2833 } 2834 2835 static int 2836 set_params__post_init(struct adapter *sc) 2837 { 2838 uint32_t param, val; 2839 2840 /* ask for encapsulated CPLs */ 2841 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 2842 val = 1; 2843 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2844 2845 return (0); 2846 } 2847 2848 #undef FW_PARAM_PFVF 2849 #undef FW_PARAM_DEV 2850 2851 static void 2852 t4_set_desc(struct adapter *sc) 2853 { 2854 char buf[128]; 2855 struct adapter_params *p = &sc->params; 2856 2857 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, " 2858 "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "", 2859 chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec); 2860 2861 device_set_desc_copy(sc->dev, buf); 2862 } 2863 2864 static void 2865 build_medialist(struct port_info *pi, struct ifmedia *media) 2866 { 2867 int m; 2868 2869 PORT_LOCK(pi); 2870 2871 ifmedia_removeall(media); 2872 2873 m = IFM_ETHER | IFM_FDX; 2874 2875 switch(pi->port_type) { 2876 case FW_PORT_TYPE_BT_XFI: 2877 case FW_PORT_TYPE_BT_XAUI: 2878 ifmedia_add(media, m | IFM_10G_T, 0, NULL); 2879 /* fall through */ 2880 2881 case FW_PORT_TYPE_BT_SGMII: 2882 ifmedia_add(media, m | IFM_1000_T, 0, NULL); 2883 ifmedia_add(media, m | IFM_100_TX, 0, NULL); 2884 ifmedia_add(media, IFM_ETHER | IFM_AUTO, 0, NULL); 2885 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2886 break; 2887 2888 case FW_PORT_TYPE_CX4: 2889 ifmedia_add(media, m | IFM_10G_CX4, 0, NULL); 2890 ifmedia_set(media, m | IFM_10G_CX4); 2891 break; 2892 2893 case FW_PORT_TYPE_QSFP_10G: 2894 case FW_PORT_TYPE_SFP: 2895 case FW_PORT_TYPE_FIBER_XFI: 2896 case FW_PORT_TYPE_FIBER_XAUI: 2897 switch (pi->mod_type) { 2898 2899 case FW_PORT_MOD_TYPE_LR: 2900 ifmedia_add(media, m | IFM_10G_LR, 0, NULL); 2901 ifmedia_set(media, m | IFM_10G_LR); 2902 break; 2903 2904 case FW_PORT_MOD_TYPE_SR: 2905 ifmedia_add(media, m | IFM_10G_SR, 0, NULL); 2906 ifmedia_set(media, m | IFM_10G_SR); 2907 break; 2908 2909 case FW_PORT_MOD_TYPE_LRM: 2910 ifmedia_add(media, m | IFM_10G_LRM, 0, NULL); 2911 ifmedia_set(media, m | IFM_10G_LRM); 2912 break; 2913 2914 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2915 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2916 ifmedia_add(media, m | IFM_10G_TWINAX, 0, NULL); 2917 ifmedia_set(media, m | IFM_10G_TWINAX); 2918 break; 2919 2920 case FW_PORT_MOD_TYPE_NONE: 2921 m &= ~IFM_FDX; 2922 ifmedia_add(media, m | IFM_NONE, 0, NULL); 2923 ifmedia_set(media, m | IFM_NONE); 2924 break; 2925 2926 case FW_PORT_MOD_TYPE_NA: 2927 case FW_PORT_MOD_TYPE_ER: 2928 default: 2929 device_printf(pi->dev, 2930 "unknown port_type (%d), mod_type (%d)\n", 2931 pi->port_type, pi->mod_type); 2932 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 2933 ifmedia_set(media, m | IFM_UNKNOWN); 2934 break; 2935 } 2936 break; 2937 2938 case FW_PORT_TYPE_QSFP: 2939 switch (pi->mod_type) { 2940 2941 case FW_PORT_MOD_TYPE_LR: 2942 ifmedia_add(media, m | IFM_40G_LR4, 0, NULL); 2943 ifmedia_set(media, m | IFM_40G_LR4); 2944 break; 2945 2946 case FW_PORT_MOD_TYPE_SR: 2947 ifmedia_add(media, m | IFM_40G_SR4, 0, NULL); 2948 ifmedia_set(media, m | IFM_40G_SR4); 2949 break; 2950 2951 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2952 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2953 ifmedia_add(media, m | IFM_40G_CR4, 0, NULL); 2954 ifmedia_set(media, m | IFM_40G_CR4); 2955 break; 2956 2957 case FW_PORT_MOD_TYPE_NONE: 2958 m &= ~IFM_FDX; 2959 ifmedia_add(media, m | IFM_NONE, 0, NULL); 2960 ifmedia_set(media, m | IFM_NONE); 2961 break; 2962 2963 default: 2964 device_printf(pi->dev, 2965 "unknown port_type (%d), mod_type (%d)\n", 2966 pi->port_type, pi->mod_type); 2967 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 2968 ifmedia_set(media, m | IFM_UNKNOWN); 2969 break; 2970 } 2971 break; 2972 2973 default: 2974 device_printf(pi->dev, 2975 "unknown port_type (%d), mod_type (%d)\n", pi->port_type, 2976 pi->mod_type); 2977 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 2978 ifmedia_set(media, m | IFM_UNKNOWN); 2979 break; 2980 } 2981 2982 PORT_UNLOCK(pi); 2983 } 2984 2985 #define FW_MAC_EXACT_CHUNK 7 2986 2987 /* 2988 * Program the port's XGMAC based on parameters in ifnet. The caller also 2989 * indicates which parameters should be programmed (the rest are left alone). 2990 */ 2991 int 2992 update_mac_settings(struct ifnet *ifp, int flags) 2993 { 2994 int rc = 0; 2995 struct port_info *pi = ifp->if_softc; 2996 struct adapter *sc = pi->adapter; 2997 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 2998 uint16_t viid = 0xffff; 2999 int16_t *xact_addr_filt = NULL; 3000 3001 ASSERT_SYNCHRONIZED_OP(sc); 3002 KASSERT(flags, ("%s: not told what to update.", __func__)); 3003 3004 if (ifp == pi->ifp) { 3005 viid = pi->viid; 3006 xact_addr_filt = &pi->xact_addr_filt; 3007 } 3008 #ifdef DEV_NETMAP 3009 else if (ifp == pi->nm_ifp) { 3010 viid = pi->nm_viid; 3011 xact_addr_filt = &pi->nm_xact_addr_filt; 3012 } 3013 #endif 3014 if (flags & XGMAC_MTU) 3015 mtu = ifp->if_mtu; 3016 3017 if (flags & XGMAC_PROMISC) 3018 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 3019 3020 if (flags & XGMAC_ALLMULTI) 3021 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 3022 3023 if (flags & XGMAC_VLANEX) 3024 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 3025 3026 if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) { 3027 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti, 3028 1, vlanex, false); 3029 if (rc) { 3030 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, 3031 rc); 3032 return (rc); 3033 } 3034 } 3035 3036 if (flags & XGMAC_UCADDR) { 3037 uint8_t ucaddr[ETHER_ADDR_LEN]; 3038 3039 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 3040 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr, 3041 true, true); 3042 if (rc < 0) { 3043 rc = -rc; 3044 if_printf(ifp, "change_mac failed: %d\n", rc); 3045 return (rc); 3046 } else { 3047 *xact_addr_filt = rc; 3048 rc = 0; 3049 } 3050 } 3051 3052 if (flags & XGMAC_MCADDRS) { 3053 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 3054 int del = 1; 3055 uint64_t hash = 0; 3056 struct ifmultiaddr *ifma; 3057 int i = 0, j; 3058 3059 if_maddr_rlock(ifp); 3060 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3061 if (ifma->ifma_addr->sa_family != AF_LINK) 3062 continue; 3063 mcaddr[i] = 3064 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 3065 MPASS(ETHER_IS_MULTICAST(mcaddr[i])); 3066 i++; 3067 3068 if (i == FW_MAC_EXACT_CHUNK) { 3069 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, 3070 i, mcaddr, NULL, &hash, 0); 3071 if (rc < 0) { 3072 rc = -rc; 3073 for (j = 0; j < i; j++) { 3074 if_printf(ifp, 3075 "failed to add mc address" 3076 " %02x:%02x:%02x:" 3077 "%02x:%02x:%02x rc=%d\n", 3078 mcaddr[j][0], mcaddr[j][1], 3079 mcaddr[j][2], mcaddr[j][3], 3080 mcaddr[j][4], mcaddr[j][5], 3081 rc); 3082 } 3083 goto mcfail; 3084 } 3085 del = 0; 3086 i = 0; 3087 } 3088 } 3089 if (i > 0) { 3090 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i, 3091 mcaddr, NULL, &hash, 0); 3092 if (rc < 0) { 3093 rc = -rc; 3094 for (j = 0; j < i; j++) { 3095 if_printf(ifp, 3096 "failed to add mc address" 3097 " %02x:%02x:%02x:" 3098 "%02x:%02x:%02x rc=%d\n", 3099 mcaddr[j][0], mcaddr[j][1], 3100 mcaddr[j][2], mcaddr[j][3], 3101 mcaddr[j][4], mcaddr[j][5], 3102 rc); 3103 } 3104 goto mcfail; 3105 } 3106 } 3107 3108 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0); 3109 if (rc != 0) 3110 if_printf(ifp, "failed to set mc address hash: %d", rc); 3111 mcfail: 3112 if_maddr_runlock(ifp); 3113 } 3114 3115 return (rc); 3116 } 3117 3118 /* 3119 * {begin|end}_synchronized_op must be called from the same thread. 3120 */ 3121 int 3122 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags, 3123 char *wmesg) 3124 { 3125 int rc, pri; 3126 3127 #ifdef WITNESS 3128 /* the caller thinks it's ok to sleep, but is it really? */ 3129 if (flags & SLEEP_OK) 3130 pause("t4slptst", 1); 3131 #endif 3132 3133 if (INTR_OK) 3134 pri = PCATCH; 3135 else 3136 pri = 0; 3137 3138 ADAPTER_LOCK(sc); 3139 for (;;) { 3140 3141 if (pi && IS_DOOMED(pi)) { 3142 rc = ENXIO; 3143 goto done; 3144 } 3145 3146 if (!IS_BUSY(sc)) { 3147 rc = 0; 3148 break; 3149 } 3150 3151 if (!(flags & SLEEP_OK)) { 3152 rc = EBUSY; 3153 goto done; 3154 } 3155 3156 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 3157 rc = EINTR; 3158 goto done; 3159 } 3160 } 3161 3162 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 3163 SET_BUSY(sc); 3164 #ifdef INVARIANTS 3165 sc->last_op = wmesg; 3166 sc->last_op_thr = curthread; 3167 #endif 3168 3169 done: 3170 if (!(flags & HOLD_LOCK) || rc) 3171 ADAPTER_UNLOCK(sc); 3172 3173 return (rc); 3174 } 3175 3176 /* 3177 * {begin|end}_synchronized_op must be called from the same thread. 3178 */ 3179 void 3180 end_synchronized_op(struct adapter *sc, int flags) 3181 { 3182 3183 if (flags & LOCK_HELD) 3184 ADAPTER_LOCK_ASSERT_OWNED(sc); 3185 else 3186 ADAPTER_LOCK(sc); 3187 3188 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 3189 CLR_BUSY(sc); 3190 wakeup(&sc->flags); 3191 ADAPTER_UNLOCK(sc); 3192 } 3193 3194 static int 3195 cxgbe_init_synchronized(struct port_info *pi) 3196 { 3197 struct adapter *sc = pi->adapter; 3198 struct ifnet *ifp = pi->ifp; 3199 int rc = 0, i; 3200 struct sge_txq *txq; 3201 3202 ASSERT_SYNCHRONIZED_OP(sc); 3203 3204 if (isset(&sc->open_device_map, pi->port_id)) { 3205 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 3206 ("mismatch between open_device_map and if_drv_flags")); 3207 return (0); /* already running */ 3208 } 3209 3210 if (!(sc->flags & FULL_INIT_DONE) && 3211 ((rc = adapter_full_init(sc)) != 0)) 3212 return (rc); /* error message displayed already */ 3213 3214 if (!(pi->flags & PORT_INIT_DONE) && 3215 ((rc = port_full_init(pi)) != 0)) 3216 return (rc); /* error message displayed already */ 3217 3218 rc = update_mac_settings(ifp, XGMAC_ALL); 3219 if (rc) 3220 goto done; /* error message displayed already */ 3221 3222 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 3223 if (rc != 0) { 3224 if_printf(ifp, "enable_vi failed: %d\n", rc); 3225 goto done; 3226 } 3227 3228 /* 3229 * Can't fail from this point onwards. Review cxgbe_uninit_synchronized 3230 * if this changes. 3231 */ 3232 3233 for_each_txq(pi, i, txq) { 3234 TXQ_LOCK(txq); 3235 txq->eq.flags |= EQ_ENABLED; 3236 TXQ_UNLOCK(txq); 3237 } 3238 3239 /* 3240 * The first iq of the first port to come up is used for tracing. 3241 */ 3242 if (sc->traceq < 0) { 3243 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id; 3244 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : 3245 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | 3246 V_QUEUENUMBER(sc->traceq)); 3247 pi->flags |= HAS_TRACEQ; 3248 } 3249 3250 /* all ok */ 3251 setbit(&sc->open_device_map, pi->port_id); 3252 PORT_LOCK(pi); 3253 ifp->if_drv_flags |= IFF_DRV_RUNNING; 3254 PORT_UNLOCK(pi); 3255 3256 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 3257 done: 3258 if (rc != 0) 3259 cxgbe_uninit_synchronized(pi); 3260 3261 return (rc); 3262 } 3263 3264 /* 3265 * Idempotent. 3266 */ 3267 static int 3268 cxgbe_uninit_synchronized(struct port_info *pi) 3269 { 3270 struct adapter *sc = pi->adapter; 3271 struct ifnet *ifp = pi->ifp; 3272 int rc, i; 3273 struct sge_txq *txq; 3274 3275 ASSERT_SYNCHRONIZED_OP(sc); 3276 3277 if (!(pi->flags & PORT_INIT_DONE)) { 3278 KASSERT(!(ifp->if_drv_flags & IFF_DRV_RUNNING), 3279 ("uninited port is running")); 3280 return (0); 3281 } 3282 3283 /* 3284 * Disable the VI so that all its data in either direction is discarded 3285 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 3286 * tick) intact as the TP can deliver negative advice or data that it's 3287 * holding in its RAM (for an offloaded connection) even after the VI is 3288 * disabled. 3289 */ 3290 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 3291 if (rc) { 3292 if_printf(ifp, "disable_vi failed: %d\n", rc); 3293 return (rc); 3294 } 3295 3296 for_each_txq(pi, i, txq) { 3297 TXQ_LOCK(txq); 3298 txq->eq.flags &= ~EQ_ENABLED; 3299 TXQ_UNLOCK(txq); 3300 } 3301 3302 clrbit(&sc->open_device_map, pi->port_id); 3303 PORT_LOCK(pi); 3304 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3305 PORT_UNLOCK(pi); 3306 3307 pi->link_cfg.link_ok = 0; 3308 pi->link_cfg.speed = 0; 3309 pi->linkdnrc = -1; 3310 t4_os_link_changed(sc, pi->port_id, 0, -1); 3311 3312 return (0); 3313 } 3314 3315 /* 3316 * It is ok for this function to fail midway and return right away. t4_detach 3317 * will walk the entire sc->irq list and clean up whatever is valid. 3318 */ 3319 static int 3320 setup_intr_handlers(struct adapter *sc) 3321 { 3322 int rc, rid, p, q; 3323 char s[8]; 3324 struct irq *irq; 3325 struct port_info *pi; 3326 struct sge_rxq *rxq; 3327 #ifdef TCP_OFFLOAD 3328 struct sge_ofld_rxq *ofld_rxq; 3329 #endif 3330 #ifdef DEV_NETMAP 3331 struct sge_nm_rxq *nm_rxq; 3332 #endif 3333 3334 /* 3335 * Setup interrupts. 3336 */ 3337 irq = &sc->irq[0]; 3338 rid = sc->intr_type == INTR_INTX ? 0 : 1; 3339 if (sc->intr_count == 1) 3340 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all")); 3341 3342 /* Multiple interrupts. */ 3343 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 3344 ("%s: too few intr.", __func__)); 3345 3346 /* The first one is always error intr */ 3347 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 3348 if (rc != 0) 3349 return (rc); 3350 irq++; 3351 rid++; 3352 3353 /* The second one is always the firmware event queue */ 3354 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt"); 3355 if (rc != 0) 3356 return (rc); 3357 irq++; 3358 rid++; 3359 3360 for_each_port(sc, p) { 3361 pi = sc->port[p]; 3362 3363 if (pi->flags & INTR_RXQ) { 3364 for_each_rxq(pi, q, rxq) { 3365 snprintf(s, sizeof(s), "%d.%d", p, q); 3366 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq, 3367 s); 3368 if (rc != 0) 3369 return (rc); 3370 irq++; 3371 rid++; 3372 } 3373 } 3374 #ifdef TCP_OFFLOAD 3375 if (pi->flags & INTR_OFLD_RXQ) { 3376 for_each_ofld_rxq(pi, q, ofld_rxq) { 3377 snprintf(s, sizeof(s), "%d,%d", p, q); 3378 rc = t4_alloc_irq(sc, irq, rid, t4_intr, 3379 ofld_rxq, s); 3380 if (rc != 0) 3381 return (rc); 3382 irq++; 3383 rid++; 3384 } 3385 } 3386 #endif 3387 #ifdef DEV_NETMAP 3388 if (pi->flags & INTR_NM_RXQ) { 3389 for_each_nm_rxq(pi, q, nm_rxq) { 3390 snprintf(s, sizeof(s), "%d-%d", p, q); 3391 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr, 3392 nm_rxq, s); 3393 if (rc != 0) 3394 return (rc); 3395 irq++; 3396 rid++; 3397 } 3398 } 3399 #endif 3400 } 3401 MPASS(irq == &sc->irq[sc->intr_count]); 3402 3403 return (0); 3404 } 3405 3406 int 3407 adapter_full_init(struct adapter *sc) 3408 { 3409 int rc, i; 3410 3411 ASSERT_SYNCHRONIZED_OP(sc); 3412 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3413 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 3414 ("%s: FULL_INIT_DONE already", __func__)); 3415 3416 /* 3417 * queues that belong to the adapter (not any particular port). 3418 */ 3419 rc = t4_setup_adapter_queues(sc); 3420 if (rc != 0) 3421 goto done; 3422 3423 for (i = 0; i < nitems(sc->tq); i++) { 3424 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 3425 taskqueue_thread_enqueue, &sc->tq[i]); 3426 if (sc->tq[i] == NULL) { 3427 device_printf(sc->dev, 3428 "failed to allocate task queue %d\n", i); 3429 rc = ENOMEM; 3430 goto done; 3431 } 3432 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3433 device_get_nameunit(sc->dev), i); 3434 } 3435 3436 t4_intr_enable(sc); 3437 sc->flags |= FULL_INIT_DONE; 3438 done: 3439 if (rc != 0) 3440 adapter_full_uninit(sc); 3441 3442 return (rc); 3443 } 3444 3445 int 3446 adapter_full_uninit(struct adapter *sc) 3447 { 3448 int i; 3449 3450 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3451 3452 t4_teardown_adapter_queues(sc); 3453 3454 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3455 taskqueue_free(sc->tq[i]); 3456 sc->tq[i] = NULL; 3457 } 3458 3459 sc->flags &= ~FULL_INIT_DONE; 3460 3461 return (0); 3462 } 3463 3464 int 3465 port_full_init(struct port_info *pi) 3466 { 3467 struct adapter *sc = pi->adapter; 3468 struct ifnet *ifp = pi->ifp; 3469 uint16_t *rss; 3470 struct sge_rxq *rxq; 3471 int rc, i, j; 3472 3473 ASSERT_SYNCHRONIZED_OP(sc); 3474 KASSERT((pi->flags & PORT_INIT_DONE) == 0, 3475 ("%s: PORT_INIT_DONE already", __func__)); 3476 3477 sysctl_ctx_init(&pi->ctx); 3478 pi->flags |= PORT_SYSCTL_CTX; 3479 3480 /* 3481 * Allocate tx/rx/fl queues for this port. 3482 */ 3483 rc = t4_setup_port_queues(pi); 3484 if (rc != 0) 3485 goto done; /* error message displayed already */ 3486 3487 /* 3488 * Setup RSS for this port. Save a copy of the RSS table for later use. 3489 */ 3490 rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); 3491 for (i = 0; i < pi->rss_size;) { 3492 for_each_rxq(pi, j, rxq) { 3493 rss[i++] = rxq->iq.abs_id; 3494 if (i == pi->rss_size) 3495 break; 3496 } 3497 } 3498 3499 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss, 3500 pi->rss_size); 3501 if (rc != 0) { 3502 if_printf(ifp, "rss_config failed: %d\n", rc); 3503 goto done; 3504 } 3505 3506 pi->rss = rss; 3507 pi->flags |= PORT_INIT_DONE; 3508 done: 3509 if (rc != 0) 3510 port_full_uninit(pi); 3511 3512 return (rc); 3513 } 3514 3515 /* 3516 * Idempotent. 3517 */ 3518 int 3519 port_full_uninit(struct port_info *pi) 3520 { 3521 struct adapter *sc = pi->adapter; 3522 int i; 3523 struct sge_rxq *rxq; 3524 struct sge_txq *txq; 3525 #ifdef TCP_OFFLOAD 3526 struct sge_ofld_rxq *ofld_rxq; 3527 struct sge_wrq *ofld_txq; 3528 #endif 3529 3530 if (pi->flags & PORT_INIT_DONE) { 3531 3532 /* Need to quiesce queues. */ 3533 3534 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]); 3535 3536 for_each_txq(pi, i, txq) { 3537 quiesce_txq(sc, txq); 3538 } 3539 3540 #ifdef TCP_OFFLOAD 3541 for_each_ofld_txq(pi, i, ofld_txq) { 3542 quiesce_wrq(sc, ofld_txq); 3543 } 3544 #endif 3545 3546 for_each_rxq(pi, i, rxq) { 3547 quiesce_iq(sc, &rxq->iq); 3548 quiesce_fl(sc, &rxq->fl); 3549 } 3550 3551 #ifdef TCP_OFFLOAD 3552 for_each_ofld_rxq(pi, i, ofld_rxq) { 3553 quiesce_iq(sc, &ofld_rxq->iq); 3554 quiesce_fl(sc, &ofld_rxq->fl); 3555 } 3556 #endif 3557 free(pi->rss, M_CXGBE); 3558 } 3559 3560 t4_teardown_port_queues(pi); 3561 pi->flags &= ~PORT_INIT_DONE; 3562 3563 return (0); 3564 } 3565 3566 static void 3567 quiesce_txq(struct adapter *sc, struct sge_txq *txq) 3568 { 3569 struct sge_eq *eq = &txq->eq; 3570 struct sge_qstat *spg = (void *)&eq->desc[eq->sidx]; 3571 3572 (void) sc; /* unused */ 3573 3574 #ifdef INVARIANTS 3575 TXQ_LOCK(txq); 3576 MPASS((eq->flags & EQ_ENABLED) == 0); 3577 TXQ_UNLOCK(txq); 3578 #endif 3579 3580 /* Wait for the mp_ring to empty. */ 3581 while (!mp_ring_is_idle(txq->r)) { 3582 mp_ring_check_drainage(txq->r, 0); 3583 pause("rquiesce", 1); 3584 } 3585 3586 /* Then wait for the hardware to finish. */ 3587 while (spg->cidx != htobe16(eq->pidx)) 3588 pause("equiesce", 1); 3589 3590 /* Finally, wait for the driver to reclaim all descriptors. */ 3591 while (eq->cidx != eq->pidx) 3592 pause("dquiesce", 1); 3593 } 3594 3595 static void 3596 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq) 3597 { 3598 3599 /* XXXTX */ 3600 } 3601 3602 static void 3603 quiesce_iq(struct adapter *sc, struct sge_iq *iq) 3604 { 3605 (void) sc; /* unused */ 3606 3607 /* Synchronize with the interrupt handler */ 3608 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 3609 pause("iqfree", 1); 3610 } 3611 3612 static void 3613 quiesce_fl(struct adapter *sc, struct sge_fl *fl) 3614 { 3615 mtx_lock(&sc->sfl_lock); 3616 FL_LOCK(fl); 3617 fl->flags |= FL_DOOMED; 3618 FL_UNLOCK(fl); 3619 mtx_unlock(&sc->sfl_lock); 3620 3621 callout_drain(&sc->sfl_callout); 3622 KASSERT((fl->flags & FL_STARVING) == 0, 3623 ("%s: still starving", __func__)); 3624 } 3625 3626 static int 3627 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 3628 driver_intr_t *handler, void *arg, char *name) 3629 { 3630 int rc; 3631 3632 irq->rid = rid; 3633 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 3634 RF_SHAREABLE | RF_ACTIVE); 3635 if (irq->res == NULL) { 3636 device_printf(sc->dev, 3637 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 3638 return (ENOMEM); 3639 } 3640 3641 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 3642 NULL, handler, arg, &irq->tag); 3643 if (rc != 0) { 3644 device_printf(sc->dev, 3645 "failed to setup interrupt for rid %d, name %s: %d\n", 3646 rid, name, rc); 3647 } else if (name) 3648 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 3649 3650 return (rc); 3651 } 3652 3653 static int 3654 t4_free_irq(struct adapter *sc, struct irq *irq) 3655 { 3656 if (irq->tag) 3657 bus_teardown_intr(sc->dev, irq->res, irq->tag); 3658 if (irq->res) 3659 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 3660 3661 bzero(irq, sizeof(*irq)); 3662 3663 return (0); 3664 } 3665 3666 static void 3667 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 3668 unsigned int end) 3669 { 3670 uint32_t *p = (uint32_t *)(buf + start); 3671 3672 for ( ; start <= end; start += sizeof(uint32_t)) 3673 *p++ = t4_read_reg(sc, start); 3674 } 3675 3676 static void 3677 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 3678 { 3679 int i, n; 3680 const unsigned int *reg_ranges; 3681 static const unsigned int t4_reg_ranges[] = { 3682 0x1008, 0x1108, 3683 0x1180, 0x11b4, 3684 0x11fc, 0x123c, 3685 0x1300, 0x173c, 3686 0x1800, 0x18fc, 3687 0x3000, 0x30d8, 3688 0x30e0, 0x5924, 3689 0x5960, 0x59d4, 3690 0x5a00, 0x5af8, 3691 0x6000, 0x6098, 3692 0x6100, 0x6150, 3693 0x6200, 0x6208, 3694 0x6240, 0x6248, 3695 0x6280, 0x6338, 3696 0x6370, 0x638c, 3697 0x6400, 0x643c, 3698 0x6500, 0x6524, 3699 0x6a00, 0x6a38, 3700 0x6a60, 0x6a78, 3701 0x6b00, 0x6b84, 3702 0x6bf0, 0x6c84, 3703 0x6cf0, 0x6d84, 3704 0x6df0, 0x6e84, 3705 0x6ef0, 0x6f84, 3706 0x6ff0, 0x7084, 3707 0x70f0, 0x7184, 3708 0x71f0, 0x7284, 3709 0x72f0, 0x7384, 3710 0x73f0, 0x7450, 3711 0x7500, 0x7530, 3712 0x7600, 0x761c, 3713 0x7680, 0x76cc, 3714 0x7700, 0x7798, 3715 0x77c0, 0x77fc, 3716 0x7900, 0x79fc, 3717 0x7b00, 0x7c38, 3718 0x7d00, 0x7efc, 3719 0x8dc0, 0x8e1c, 3720 0x8e30, 0x8e78, 3721 0x8ea0, 0x8f6c, 3722 0x8fc0, 0x9074, 3723 0x90fc, 0x90fc, 3724 0x9400, 0x9458, 3725 0x9600, 0x96bc, 3726 0x9800, 0x9808, 3727 0x9820, 0x983c, 3728 0x9850, 0x9864, 3729 0x9c00, 0x9c6c, 3730 0x9c80, 0x9cec, 3731 0x9d00, 0x9d6c, 3732 0x9d80, 0x9dec, 3733 0x9e00, 0x9e6c, 3734 0x9e80, 0x9eec, 3735 0x9f00, 0x9f6c, 3736 0x9f80, 0x9fec, 3737 0xd004, 0xd03c, 3738 0xdfc0, 0xdfe0, 3739 0xe000, 0xea7c, 3740 0xf000, 0x11110, 3741 0x11118, 0x11190, 3742 0x19040, 0x1906c, 3743 0x19078, 0x19080, 3744 0x1908c, 0x19124, 3745 0x19150, 0x191b0, 3746 0x191d0, 0x191e8, 3747 0x19238, 0x1924c, 3748 0x193f8, 0x19474, 3749 0x19490, 0x194f8, 3750 0x19800, 0x19f30, 3751 0x1a000, 0x1a06c, 3752 0x1a0b0, 0x1a120, 3753 0x1a128, 0x1a138, 3754 0x1a190, 0x1a1c4, 3755 0x1a1fc, 0x1a1fc, 3756 0x1e040, 0x1e04c, 3757 0x1e284, 0x1e28c, 3758 0x1e2c0, 0x1e2c0, 3759 0x1e2e0, 0x1e2e0, 3760 0x1e300, 0x1e384, 3761 0x1e3c0, 0x1e3c8, 3762 0x1e440, 0x1e44c, 3763 0x1e684, 0x1e68c, 3764 0x1e6c0, 0x1e6c0, 3765 0x1e6e0, 0x1e6e0, 3766 0x1e700, 0x1e784, 3767 0x1e7c0, 0x1e7c8, 3768 0x1e840, 0x1e84c, 3769 0x1ea84, 0x1ea8c, 3770 0x1eac0, 0x1eac0, 3771 0x1eae0, 0x1eae0, 3772 0x1eb00, 0x1eb84, 3773 0x1ebc0, 0x1ebc8, 3774 0x1ec40, 0x1ec4c, 3775 0x1ee84, 0x1ee8c, 3776 0x1eec0, 0x1eec0, 3777 0x1eee0, 0x1eee0, 3778 0x1ef00, 0x1ef84, 3779 0x1efc0, 0x1efc8, 3780 0x1f040, 0x1f04c, 3781 0x1f284, 0x1f28c, 3782 0x1f2c0, 0x1f2c0, 3783 0x1f2e0, 0x1f2e0, 3784 0x1f300, 0x1f384, 3785 0x1f3c0, 0x1f3c8, 3786 0x1f440, 0x1f44c, 3787 0x1f684, 0x1f68c, 3788 0x1f6c0, 0x1f6c0, 3789 0x1f6e0, 0x1f6e0, 3790 0x1f700, 0x1f784, 3791 0x1f7c0, 0x1f7c8, 3792 0x1f840, 0x1f84c, 3793 0x1fa84, 0x1fa8c, 3794 0x1fac0, 0x1fac0, 3795 0x1fae0, 0x1fae0, 3796 0x1fb00, 0x1fb84, 3797 0x1fbc0, 0x1fbc8, 3798 0x1fc40, 0x1fc4c, 3799 0x1fe84, 0x1fe8c, 3800 0x1fec0, 0x1fec0, 3801 0x1fee0, 0x1fee0, 3802 0x1ff00, 0x1ff84, 3803 0x1ffc0, 0x1ffc8, 3804 0x20000, 0x2002c, 3805 0x20100, 0x2013c, 3806 0x20190, 0x201c8, 3807 0x20200, 0x20318, 3808 0x20400, 0x20528, 3809 0x20540, 0x20614, 3810 0x21000, 0x21040, 3811 0x2104c, 0x21060, 3812 0x210c0, 0x210ec, 3813 0x21200, 0x21268, 3814 0x21270, 0x21284, 3815 0x212fc, 0x21388, 3816 0x21400, 0x21404, 3817 0x21500, 0x21518, 3818 0x2152c, 0x2153c, 3819 0x21550, 0x21554, 3820 0x21600, 0x21600, 3821 0x21608, 0x21628, 3822 0x21630, 0x2163c, 3823 0x21700, 0x2171c, 3824 0x21780, 0x2178c, 3825 0x21800, 0x21c38, 3826 0x21c80, 0x21d7c, 3827 0x21e00, 0x21e04, 3828 0x22000, 0x2202c, 3829 0x22100, 0x2213c, 3830 0x22190, 0x221c8, 3831 0x22200, 0x22318, 3832 0x22400, 0x22528, 3833 0x22540, 0x22614, 3834 0x23000, 0x23040, 3835 0x2304c, 0x23060, 3836 0x230c0, 0x230ec, 3837 0x23200, 0x23268, 3838 0x23270, 0x23284, 3839 0x232fc, 0x23388, 3840 0x23400, 0x23404, 3841 0x23500, 0x23518, 3842 0x2352c, 0x2353c, 3843 0x23550, 0x23554, 3844 0x23600, 0x23600, 3845 0x23608, 0x23628, 3846 0x23630, 0x2363c, 3847 0x23700, 0x2371c, 3848 0x23780, 0x2378c, 3849 0x23800, 0x23c38, 3850 0x23c80, 0x23d7c, 3851 0x23e00, 0x23e04, 3852 0x24000, 0x2402c, 3853 0x24100, 0x2413c, 3854 0x24190, 0x241c8, 3855 0x24200, 0x24318, 3856 0x24400, 0x24528, 3857 0x24540, 0x24614, 3858 0x25000, 0x25040, 3859 0x2504c, 0x25060, 3860 0x250c0, 0x250ec, 3861 0x25200, 0x25268, 3862 0x25270, 0x25284, 3863 0x252fc, 0x25388, 3864 0x25400, 0x25404, 3865 0x25500, 0x25518, 3866 0x2552c, 0x2553c, 3867 0x25550, 0x25554, 3868 0x25600, 0x25600, 3869 0x25608, 0x25628, 3870 0x25630, 0x2563c, 3871 0x25700, 0x2571c, 3872 0x25780, 0x2578c, 3873 0x25800, 0x25c38, 3874 0x25c80, 0x25d7c, 3875 0x25e00, 0x25e04, 3876 0x26000, 0x2602c, 3877 0x26100, 0x2613c, 3878 0x26190, 0x261c8, 3879 0x26200, 0x26318, 3880 0x26400, 0x26528, 3881 0x26540, 0x26614, 3882 0x27000, 0x27040, 3883 0x2704c, 0x27060, 3884 0x270c0, 0x270ec, 3885 0x27200, 0x27268, 3886 0x27270, 0x27284, 3887 0x272fc, 0x27388, 3888 0x27400, 0x27404, 3889 0x27500, 0x27518, 3890 0x2752c, 0x2753c, 3891 0x27550, 0x27554, 3892 0x27600, 0x27600, 3893 0x27608, 0x27628, 3894 0x27630, 0x2763c, 3895 0x27700, 0x2771c, 3896 0x27780, 0x2778c, 3897 0x27800, 0x27c38, 3898 0x27c80, 0x27d7c, 3899 0x27e00, 0x27e04 3900 }; 3901 static const unsigned int t5_reg_ranges[] = { 3902 0x1008, 0x1148, 3903 0x1180, 0x11b4, 3904 0x11fc, 0x123c, 3905 0x1280, 0x173c, 3906 0x1800, 0x18fc, 3907 0x3000, 0x3028, 3908 0x3060, 0x30d8, 3909 0x30e0, 0x30fc, 3910 0x3140, 0x357c, 3911 0x35a8, 0x35cc, 3912 0x35ec, 0x35ec, 3913 0x3600, 0x5624, 3914 0x56cc, 0x575c, 3915 0x580c, 0x5814, 3916 0x5890, 0x58bc, 3917 0x5940, 0x59dc, 3918 0x59fc, 0x5a18, 3919 0x5a60, 0x5a9c, 3920 0x5b94, 0x5bfc, 3921 0x6000, 0x6040, 3922 0x6058, 0x614c, 3923 0x7700, 0x7798, 3924 0x77c0, 0x78fc, 3925 0x7b00, 0x7c54, 3926 0x7d00, 0x7efc, 3927 0x8dc0, 0x8de0, 3928 0x8df8, 0x8e84, 3929 0x8ea0, 0x8f84, 3930 0x8fc0, 0x90f8, 3931 0x9400, 0x9470, 3932 0x9600, 0x96f4, 3933 0x9800, 0x9808, 3934 0x9820, 0x983c, 3935 0x9850, 0x9864, 3936 0x9c00, 0x9c6c, 3937 0x9c80, 0x9cec, 3938 0x9d00, 0x9d6c, 3939 0x9d80, 0x9dec, 3940 0x9e00, 0x9e6c, 3941 0x9e80, 0x9eec, 3942 0x9f00, 0x9f6c, 3943 0x9f80, 0xa020, 3944 0xd004, 0xd03c, 3945 0xdfc0, 0xdfe0, 3946 0xe000, 0x11088, 3947 0x1109c, 0x11110, 3948 0x11118, 0x1117c, 3949 0x11190, 0x11204, 3950 0x19040, 0x1906c, 3951 0x19078, 0x19080, 3952 0x1908c, 0x19124, 3953 0x19150, 0x191b0, 3954 0x191d0, 0x191e8, 3955 0x19238, 0x19290, 3956 0x193f8, 0x19474, 3957 0x19490, 0x194cc, 3958 0x194f0, 0x194f8, 3959 0x19c00, 0x19c60, 3960 0x19c94, 0x19e10, 3961 0x19e50, 0x19f34, 3962 0x19f40, 0x19f50, 3963 0x19f90, 0x19fe4, 3964 0x1a000, 0x1a06c, 3965 0x1a0b0, 0x1a120, 3966 0x1a128, 0x1a138, 3967 0x1a190, 0x1a1c4, 3968 0x1a1fc, 0x1a1fc, 3969 0x1e008, 0x1e00c, 3970 0x1e040, 0x1e04c, 3971 0x1e284, 0x1e290, 3972 0x1e2c0, 0x1e2c0, 3973 0x1e2e0, 0x1e2e0, 3974 0x1e300, 0x1e384, 3975 0x1e3c0, 0x1e3c8, 3976 0x1e408, 0x1e40c, 3977 0x1e440, 0x1e44c, 3978 0x1e684, 0x1e690, 3979 0x1e6c0, 0x1e6c0, 3980 0x1e6e0, 0x1e6e0, 3981 0x1e700, 0x1e784, 3982 0x1e7c0, 0x1e7c8, 3983 0x1e808, 0x1e80c, 3984 0x1e840, 0x1e84c, 3985 0x1ea84, 0x1ea90, 3986 0x1eac0, 0x1eac0, 3987 0x1eae0, 0x1eae0, 3988 0x1eb00, 0x1eb84, 3989 0x1ebc0, 0x1ebc8, 3990 0x1ec08, 0x1ec0c, 3991 0x1ec40, 0x1ec4c, 3992 0x1ee84, 0x1ee90, 3993 0x1eec0, 0x1eec0, 3994 0x1eee0, 0x1eee0, 3995 0x1ef00, 0x1ef84, 3996 0x1efc0, 0x1efc8, 3997 0x1f008, 0x1f00c, 3998 0x1f040, 0x1f04c, 3999 0x1f284, 0x1f290, 4000 0x1f2c0, 0x1f2c0, 4001 0x1f2e0, 0x1f2e0, 4002 0x1f300, 0x1f384, 4003 0x1f3c0, 0x1f3c8, 4004 0x1f408, 0x1f40c, 4005 0x1f440, 0x1f44c, 4006 0x1f684, 0x1f690, 4007 0x1f6c0, 0x1f6c0, 4008 0x1f6e0, 0x1f6e0, 4009 0x1f700, 0x1f784, 4010 0x1f7c0, 0x1f7c8, 4011 0x1f808, 0x1f80c, 4012 0x1f840, 0x1f84c, 4013 0x1fa84, 0x1fa90, 4014 0x1fac0, 0x1fac0, 4015 0x1fae0, 0x1fae0, 4016 0x1fb00, 0x1fb84, 4017 0x1fbc0, 0x1fbc8, 4018 0x1fc08, 0x1fc0c, 4019 0x1fc40, 0x1fc4c, 4020 0x1fe84, 0x1fe90, 4021 0x1fec0, 0x1fec0, 4022 0x1fee0, 0x1fee0, 4023 0x1ff00, 0x1ff84, 4024 0x1ffc0, 0x1ffc8, 4025 0x30000, 0x30030, 4026 0x30100, 0x30144, 4027 0x30190, 0x301d0, 4028 0x30200, 0x30318, 4029 0x30400, 0x3052c, 4030 0x30540, 0x3061c, 4031 0x30800, 0x30834, 4032 0x308c0, 0x30908, 4033 0x30910, 0x309ac, 4034 0x30a00, 0x30a2c, 4035 0x30a44, 0x30a50, 4036 0x30a74, 0x30c24, 4037 0x30d00, 0x30d00, 4038 0x30d08, 0x30d14, 4039 0x30d1c, 0x30d20, 4040 0x30d3c, 0x30d50, 4041 0x31200, 0x3120c, 4042 0x31220, 0x31220, 4043 0x31240, 0x31240, 4044 0x31600, 0x3160c, 4045 0x31a00, 0x31a1c, 4046 0x31e00, 0x31e20, 4047 0x31e38, 0x31e3c, 4048 0x31e80, 0x31e80, 4049 0x31e88, 0x31ea8, 4050 0x31eb0, 0x31eb4, 4051 0x31ec8, 0x31ed4, 4052 0x31fb8, 0x32004, 4053 0x32200, 0x32200, 4054 0x32208, 0x32240, 4055 0x32248, 0x32280, 4056 0x32288, 0x322c0, 4057 0x322c8, 0x322fc, 4058 0x32600, 0x32630, 4059 0x32a00, 0x32abc, 4060 0x32b00, 0x32b70, 4061 0x33000, 0x33048, 4062 0x33060, 0x3309c, 4063 0x330f0, 0x33148, 4064 0x33160, 0x3319c, 4065 0x331f0, 0x332e4, 4066 0x332f8, 0x333e4, 4067 0x333f8, 0x33448, 4068 0x33460, 0x3349c, 4069 0x334f0, 0x33548, 4070 0x33560, 0x3359c, 4071 0x335f0, 0x336e4, 4072 0x336f8, 0x337e4, 4073 0x337f8, 0x337fc, 4074 0x33814, 0x33814, 4075 0x3382c, 0x3382c, 4076 0x33880, 0x3388c, 4077 0x338e8, 0x338ec, 4078 0x33900, 0x33948, 4079 0x33960, 0x3399c, 4080 0x339f0, 0x33ae4, 4081 0x33af8, 0x33b10, 4082 0x33b28, 0x33b28, 4083 0x33b3c, 0x33b50, 4084 0x33bf0, 0x33c10, 4085 0x33c28, 0x33c28, 4086 0x33c3c, 0x33c50, 4087 0x33cf0, 0x33cfc, 4088 0x34000, 0x34030, 4089 0x34100, 0x34144, 4090 0x34190, 0x341d0, 4091 0x34200, 0x34318, 4092 0x34400, 0x3452c, 4093 0x34540, 0x3461c, 4094 0x34800, 0x34834, 4095 0x348c0, 0x34908, 4096 0x34910, 0x349ac, 4097 0x34a00, 0x34a2c, 4098 0x34a44, 0x34a50, 4099 0x34a74, 0x34c24, 4100 0x34d00, 0x34d00, 4101 0x34d08, 0x34d14, 4102 0x34d1c, 0x34d20, 4103 0x34d3c, 0x34d50, 4104 0x35200, 0x3520c, 4105 0x35220, 0x35220, 4106 0x35240, 0x35240, 4107 0x35600, 0x3560c, 4108 0x35a00, 0x35a1c, 4109 0x35e00, 0x35e20, 4110 0x35e38, 0x35e3c, 4111 0x35e80, 0x35e80, 4112 0x35e88, 0x35ea8, 4113 0x35eb0, 0x35eb4, 4114 0x35ec8, 0x35ed4, 4115 0x35fb8, 0x36004, 4116 0x36200, 0x36200, 4117 0x36208, 0x36240, 4118 0x36248, 0x36280, 4119 0x36288, 0x362c0, 4120 0x362c8, 0x362fc, 4121 0x36600, 0x36630, 4122 0x36a00, 0x36abc, 4123 0x36b00, 0x36b70, 4124 0x37000, 0x37048, 4125 0x37060, 0x3709c, 4126 0x370f0, 0x37148, 4127 0x37160, 0x3719c, 4128 0x371f0, 0x372e4, 4129 0x372f8, 0x373e4, 4130 0x373f8, 0x37448, 4131 0x37460, 0x3749c, 4132 0x374f0, 0x37548, 4133 0x37560, 0x3759c, 4134 0x375f0, 0x376e4, 4135 0x376f8, 0x377e4, 4136 0x377f8, 0x377fc, 4137 0x37814, 0x37814, 4138 0x3782c, 0x3782c, 4139 0x37880, 0x3788c, 4140 0x378e8, 0x378ec, 4141 0x37900, 0x37948, 4142 0x37960, 0x3799c, 4143 0x379f0, 0x37ae4, 4144 0x37af8, 0x37b10, 4145 0x37b28, 0x37b28, 4146 0x37b3c, 0x37b50, 4147 0x37bf0, 0x37c10, 4148 0x37c28, 0x37c28, 4149 0x37c3c, 0x37c50, 4150 0x37cf0, 0x37cfc, 4151 0x38000, 0x38030, 4152 0x38100, 0x38144, 4153 0x38190, 0x381d0, 4154 0x38200, 0x38318, 4155 0x38400, 0x3852c, 4156 0x38540, 0x3861c, 4157 0x38800, 0x38834, 4158 0x388c0, 0x38908, 4159 0x38910, 0x389ac, 4160 0x38a00, 0x38a2c, 4161 0x38a44, 0x38a50, 4162 0x38a74, 0x38c24, 4163 0x38d00, 0x38d00, 4164 0x38d08, 0x38d14, 4165 0x38d1c, 0x38d20, 4166 0x38d3c, 0x38d50, 4167 0x39200, 0x3920c, 4168 0x39220, 0x39220, 4169 0x39240, 0x39240, 4170 0x39600, 0x3960c, 4171 0x39a00, 0x39a1c, 4172 0x39e00, 0x39e20, 4173 0x39e38, 0x39e3c, 4174 0x39e80, 0x39e80, 4175 0x39e88, 0x39ea8, 4176 0x39eb0, 0x39eb4, 4177 0x39ec8, 0x39ed4, 4178 0x39fb8, 0x3a004, 4179 0x3a200, 0x3a200, 4180 0x3a208, 0x3a240, 4181 0x3a248, 0x3a280, 4182 0x3a288, 0x3a2c0, 4183 0x3a2c8, 0x3a2fc, 4184 0x3a600, 0x3a630, 4185 0x3aa00, 0x3aabc, 4186 0x3ab00, 0x3ab70, 4187 0x3b000, 0x3b048, 4188 0x3b060, 0x3b09c, 4189 0x3b0f0, 0x3b148, 4190 0x3b160, 0x3b19c, 4191 0x3b1f0, 0x3b2e4, 4192 0x3b2f8, 0x3b3e4, 4193 0x3b3f8, 0x3b448, 4194 0x3b460, 0x3b49c, 4195 0x3b4f0, 0x3b548, 4196 0x3b560, 0x3b59c, 4197 0x3b5f0, 0x3b6e4, 4198 0x3b6f8, 0x3b7e4, 4199 0x3b7f8, 0x3b7fc, 4200 0x3b814, 0x3b814, 4201 0x3b82c, 0x3b82c, 4202 0x3b880, 0x3b88c, 4203 0x3b8e8, 0x3b8ec, 4204 0x3b900, 0x3b948, 4205 0x3b960, 0x3b99c, 4206 0x3b9f0, 0x3bae4, 4207 0x3baf8, 0x3bb10, 4208 0x3bb28, 0x3bb28, 4209 0x3bb3c, 0x3bb50, 4210 0x3bbf0, 0x3bc10, 4211 0x3bc28, 0x3bc28, 4212 0x3bc3c, 0x3bc50, 4213 0x3bcf0, 0x3bcfc, 4214 0x3c000, 0x3c030, 4215 0x3c100, 0x3c144, 4216 0x3c190, 0x3c1d0, 4217 0x3c200, 0x3c318, 4218 0x3c400, 0x3c52c, 4219 0x3c540, 0x3c61c, 4220 0x3c800, 0x3c834, 4221 0x3c8c0, 0x3c908, 4222 0x3c910, 0x3c9ac, 4223 0x3ca00, 0x3ca2c, 4224 0x3ca44, 0x3ca50, 4225 0x3ca74, 0x3cc24, 4226 0x3cd00, 0x3cd00, 4227 0x3cd08, 0x3cd14, 4228 0x3cd1c, 0x3cd20, 4229 0x3cd3c, 0x3cd50, 4230 0x3d200, 0x3d20c, 4231 0x3d220, 0x3d220, 4232 0x3d240, 0x3d240, 4233 0x3d600, 0x3d60c, 4234 0x3da00, 0x3da1c, 4235 0x3de00, 0x3de20, 4236 0x3de38, 0x3de3c, 4237 0x3de80, 0x3de80, 4238 0x3de88, 0x3dea8, 4239 0x3deb0, 0x3deb4, 4240 0x3dec8, 0x3ded4, 4241 0x3dfb8, 0x3e004, 4242 0x3e200, 0x3e200, 4243 0x3e208, 0x3e240, 4244 0x3e248, 0x3e280, 4245 0x3e288, 0x3e2c0, 4246 0x3e2c8, 0x3e2fc, 4247 0x3e600, 0x3e630, 4248 0x3ea00, 0x3eabc, 4249 0x3eb00, 0x3eb70, 4250 0x3f000, 0x3f048, 4251 0x3f060, 0x3f09c, 4252 0x3f0f0, 0x3f148, 4253 0x3f160, 0x3f19c, 4254 0x3f1f0, 0x3f2e4, 4255 0x3f2f8, 0x3f3e4, 4256 0x3f3f8, 0x3f448, 4257 0x3f460, 0x3f49c, 4258 0x3f4f0, 0x3f548, 4259 0x3f560, 0x3f59c, 4260 0x3f5f0, 0x3f6e4, 4261 0x3f6f8, 0x3f7e4, 4262 0x3f7f8, 0x3f7fc, 4263 0x3f814, 0x3f814, 4264 0x3f82c, 0x3f82c, 4265 0x3f880, 0x3f88c, 4266 0x3f8e8, 0x3f8ec, 4267 0x3f900, 0x3f948, 4268 0x3f960, 0x3f99c, 4269 0x3f9f0, 0x3fae4, 4270 0x3faf8, 0x3fb10, 4271 0x3fb28, 0x3fb28, 4272 0x3fb3c, 0x3fb50, 4273 0x3fbf0, 0x3fc10, 4274 0x3fc28, 0x3fc28, 4275 0x3fc3c, 0x3fc50, 4276 0x3fcf0, 0x3fcfc, 4277 0x40000, 0x4000c, 4278 0x40040, 0x40068, 4279 0x4007c, 0x40144, 4280 0x40180, 0x4018c, 4281 0x40200, 0x40298, 4282 0x402ac, 0x4033c, 4283 0x403f8, 0x403fc, 4284 0x41304, 0x413c4, 4285 0x41400, 0x4141c, 4286 0x41480, 0x414d0, 4287 0x44000, 0x44078, 4288 0x440c0, 0x44278, 4289 0x442c0, 0x44478, 4290 0x444c0, 0x44678, 4291 0x446c0, 0x44878, 4292 0x448c0, 0x449fc, 4293 0x45000, 0x45068, 4294 0x45080, 0x45084, 4295 0x450a0, 0x450b0, 4296 0x45200, 0x45268, 4297 0x45280, 0x45284, 4298 0x452a0, 0x452b0, 4299 0x460c0, 0x460e4, 4300 0x47000, 0x4708c, 4301 0x47200, 0x47250, 4302 0x47400, 0x47420, 4303 0x47600, 0x47618, 4304 0x47800, 0x47814, 4305 0x48000, 0x4800c, 4306 0x48040, 0x48068, 4307 0x4807c, 0x48144, 4308 0x48180, 0x4818c, 4309 0x48200, 0x48298, 4310 0x482ac, 0x4833c, 4311 0x483f8, 0x483fc, 4312 0x49304, 0x493c4, 4313 0x49400, 0x4941c, 4314 0x49480, 0x494d0, 4315 0x4c000, 0x4c078, 4316 0x4c0c0, 0x4c278, 4317 0x4c2c0, 0x4c478, 4318 0x4c4c0, 0x4c678, 4319 0x4c6c0, 0x4c878, 4320 0x4c8c0, 0x4c9fc, 4321 0x4d000, 0x4d068, 4322 0x4d080, 0x4d084, 4323 0x4d0a0, 0x4d0b0, 4324 0x4d200, 0x4d268, 4325 0x4d280, 0x4d284, 4326 0x4d2a0, 0x4d2b0, 4327 0x4e0c0, 0x4e0e4, 4328 0x4f000, 0x4f08c, 4329 0x4f200, 0x4f250, 4330 0x4f400, 0x4f420, 4331 0x4f600, 0x4f618, 4332 0x4f800, 0x4f814, 4333 0x50000, 0x500cc, 4334 0x50400, 0x50400, 4335 0x50800, 0x508cc, 4336 0x50c00, 0x50c00, 4337 0x51000, 0x5101c, 4338 0x51300, 0x51308, 4339 }; 4340 4341 if (is_t4(sc)) { 4342 reg_ranges = &t4_reg_ranges[0]; 4343 n = nitems(t4_reg_ranges); 4344 } else { 4345 reg_ranges = &t5_reg_ranges[0]; 4346 n = nitems(t5_reg_ranges); 4347 } 4348 4349 regs->version = chip_id(sc) | chip_rev(sc) << 10; 4350 for (i = 0; i < n; i += 2) 4351 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 4352 } 4353 4354 static void 4355 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi) 4356 { 4357 int i; 4358 u_int v, tnl_cong_drops; 4359 struct timeval tv; 4360 const struct timeval interval = {0, 250000}; /* 250ms */ 4361 4362 getmicrotime(&tv); 4363 timevalsub(&tv, &interval); 4364 if (timevalcmp(&tv, &pi->last_refreshed, <)) 4365 return; 4366 4367 tnl_cong_drops = 0; 4368 t4_get_port_stats(sc, pi->tx_chan, &pi->stats); 4369 for (i = 0; i < NCHAN; i++) { 4370 if (pi->rx_chan_map & (1 << i)) { 4371 mtx_lock(&sc->regwin_lock); 4372 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 4373 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 4374 mtx_unlock(&sc->regwin_lock); 4375 tnl_cong_drops += v; 4376 } 4377 } 4378 pi->tnl_cong_drops = tnl_cong_drops; 4379 getmicrotime(&pi->last_refreshed); 4380 } 4381 4382 static void 4383 cxgbe_tick(void *arg) 4384 { 4385 struct port_info *pi = arg; 4386 struct adapter *sc = pi->adapter; 4387 struct ifnet *ifp = pi->ifp; 4388 4389 PORT_LOCK(pi); 4390 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 4391 PORT_UNLOCK(pi); 4392 return; /* without scheduling another callout */ 4393 } 4394 4395 cxgbe_refresh_stats(sc, pi); 4396 4397 callout_schedule(&pi->tick, hz); 4398 PORT_UNLOCK(pi); 4399 } 4400 4401 static void 4402 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 4403 { 4404 struct ifnet *vlan; 4405 4406 if (arg != ifp || ifp->if_type != IFT_ETHER) 4407 return; 4408 4409 vlan = VLAN_DEVAT(ifp, vid); 4410 VLAN_SETCOOKIE(vlan, ifp); 4411 } 4412 4413 static int 4414 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 4415 { 4416 4417 #ifdef INVARIANTS 4418 panic("%s: opcode 0x%02x on iq %p with payload %p", 4419 __func__, rss->opcode, iq, m); 4420 #else 4421 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 4422 __func__, rss->opcode, iq, m); 4423 m_freem(m); 4424 #endif 4425 return (EDOOFUS); 4426 } 4427 4428 int 4429 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 4430 { 4431 uintptr_t *loc, new; 4432 4433 if (opcode >= nitems(sc->cpl_handler)) 4434 return (EINVAL); 4435 4436 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 4437 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 4438 atomic_store_rel_ptr(loc, new); 4439 4440 return (0); 4441 } 4442 4443 static int 4444 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 4445 { 4446 4447 #ifdef INVARIANTS 4448 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 4449 #else 4450 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 4451 __func__, iq, ctrl); 4452 #endif 4453 return (EDOOFUS); 4454 } 4455 4456 int 4457 t4_register_an_handler(struct adapter *sc, an_handler_t h) 4458 { 4459 uintptr_t *loc, new; 4460 4461 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 4462 loc = (uintptr_t *) &sc->an_handler; 4463 atomic_store_rel_ptr(loc, new); 4464 4465 return (0); 4466 } 4467 4468 static int 4469 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 4470 { 4471 const struct cpl_fw6_msg *cpl = 4472 __containerof(rpl, struct cpl_fw6_msg, data[0]); 4473 4474 #ifdef INVARIANTS 4475 panic("%s: fw_msg type %d", __func__, cpl->type); 4476 #else 4477 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 4478 #endif 4479 return (EDOOFUS); 4480 } 4481 4482 int 4483 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 4484 { 4485 uintptr_t *loc, new; 4486 4487 if (type >= nitems(sc->fw_msg_handler)) 4488 return (EINVAL); 4489 4490 /* 4491 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 4492 * handler dispatch table. Reject any attempt to install a handler for 4493 * this subtype. 4494 */ 4495 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 4496 return (EINVAL); 4497 4498 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 4499 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 4500 atomic_store_rel_ptr(loc, new); 4501 4502 return (0); 4503 } 4504 4505 static int 4506 t4_sysctls(struct adapter *sc) 4507 { 4508 struct sysctl_ctx_list *ctx; 4509 struct sysctl_oid *oid; 4510 struct sysctl_oid_list *children, *c0; 4511 static char *caps[] = { 4512 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 4513 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL" /* caps[1] niccaps */ 4514 "\6HASHFILTER\7ETHOFLD", 4515 "\20\1TOE", /* caps[2] toecaps */ 4516 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 4517 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 4518 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 4519 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 4520 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 4521 "\4PO_INITIAOR\5PO_TARGET" 4522 }; 4523 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 4524 4525 ctx = device_get_sysctl_ctx(sc->dev); 4526 4527 /* 4528 * dev.t4nex.X. 4529 */ 4530 oid = device_get_sysctl_tree(sc->dev); 4531 c0 = children = SYSCTL_CHILDREN(oid); 4532 4533 sc->sc_do_rxcopy = 1; 4534 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, 4535 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); 4536 4537 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 4538 sc->params.nports, "# of ports"); 4539 4540 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 4541 NULL, chip_rev(sc), "chip hardware revision"); 4542 4543 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 4544 CTLFLAG_RD, sc->fw_version, 0, "firmware version"); 4545 4546 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 4547 CTLFLAG_RD, sc->cfg_file, 0, "configuration file"); 4548 4549 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 4550 sc->cfcsum, "config file checksum"); 4551 4552 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 4553 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 4554 sysctl_bitfield, "A", "available doorbells"); 4555 4556 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 4557 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 4558 sysctl_bitfield, "A", "available link capabilities"); 4559 4560 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 4561 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 4562 sysctl_bitfield, "A", "available NIC capabilities"); 4563 4564 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 4565 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 4566 sysctl_bitfield, "A", "available TCP offload capabilities"); 4567 4568 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 4569 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 4570 sysctl_bitfield, "A", "available RDMA capabilities"); 4571 4572 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 4573 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 4574 sysctl_bitfield, "A", "available iSCSI capabilities"); 4575 4576 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 4577 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 4578 sysctl_bitfield, "A", "available FCoE capabilities"); 4579 4580 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 4581 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 4582 4583 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 4584 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 4585 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 4586 "interrupt holdoff timer values (us)"); 4587 4588 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 4589 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 4590 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 4591 "interrupt holdoff packet counter values"); 4592 4593 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, 4594 NULL, sc->tids.nftids, "number of filters"); 4595 4596 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | 4597 CTLFLAG_RD, sc, 0, sysctl_temperature, "I", 4598 "chip temperature (in Celsius)"); 4599 4600 t4_sge_sysctls(sc, ctx, children); 4601 4602 sc->lro_timeout = 100; 4603 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, 4604 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); 4605 4606 #ifdef SBUF_DRAIN 4607 /* 4608 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 4609 */ 4610 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 4611 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 4612 "logs and miscellaneous information"); 4613 children = SYSCTL_CHILDREN(oid); 4614 4615 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 4616 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4617 sysctl_cctrl, "A", "congestion control"); 4618 4619 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 4620 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4621 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 4622 4623 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 4624 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 4625 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 4626 4627 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 4628 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 4629 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 4630 4631 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 4632 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 4633 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 4634 4635 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 4636 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 4637 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 4638 4639 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 4640 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 4641 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 4642 4643 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 4644 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4645 sysctl_cim_la, "A", "CIM logic analyzer"); 4646 4647 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", 4648 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4649 sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); 4650 4651 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 4652 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 4653 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 4654 4655 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 4656 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 4657 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 4658 4659 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 4660 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 4661 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 4662 4663 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 4664 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 4665 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 4666 4667 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 4668 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 4669 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 4670 4671 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 4672 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 4673 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 4674 4675 if (is_t5(sc)) { 4676 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 4677 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 4678 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 4679 4680 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 4681 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 4682 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 4683 } 4684 4685 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", 4686 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4687 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); 4688 4689 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 4690 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4691 sysctl_cim_qcfg, "A", "CIM queue configuration"); 4692 4693 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 4694 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4695 sysctl_cpl_stats, "A", "CPL statistics"); 4696 4697 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 4698 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4699 sysctl_ddp_stats, "A", "non-TCP DDP statistics"); 4700 4701 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 4702 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4703 sysctl_devlog, "A", "firmware's device log"); 4704 4705 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 4706 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4707 sysctl_fcoe_stats, "A", "FCoE statistics"); 4708 4709 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 4710 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4711 sysctl_hw_sched, "A", "hardware scheduler "); 4712 4713 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 4714 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4715 sysctl_l2t, "A", "hardware L2 table"); 4716 4717 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 4718 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4719 sysctl_lb_stats, "A", "loopback statistics"); 4720 4721 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 4722 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4723 sysctl_meminfo, "A", "memory regions"); 4724 4725 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", 4726 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4727 sysctl_mps_tcam, "A", "MPS TCAM entries"); 4728 4729 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 4730 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4731 sysctl_path_mtus, "A", "path MTUs"); 4732 4733 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 4734 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4735 sysctl_pm_stats, "A", "PM statistics"); 4736 4737 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 4738 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4739 sysctl_rdma_stats, "A", "RDMA statistics"); 4740 4741 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 4742 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4743 sysctl_tcp_stats, "A", "TCP statistics"); 4744 4745 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 4746 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4747 sysctl_tids, "A", "TID information"); 4748 4749 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 4750 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4751 sysctl_tp_err_stats, "A", "TP error statistics"); 4752 4753 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", 4754 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4755 sysctl_tp_la, "A", "TP logic analyzer"); 4756 4757 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 4758 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4759 sysctl_tx_rate, "A", "Tx rate"); 4760 4761 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", 4762 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4763 sysctl_ulprx_la, "A", "ULPRX logic analyzer"); 4764 4765 if (is_t5(sc)) { 4766 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 4767 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4768 sysctl_wcwr_stats, "A", "write combined work requests"); 4769 } 4770 #endif 4771 4772 #ifdef TCP_OFFLOAD 4773 if (is_offload(sc)) { 4774 /* 4775 * dev.t4nex.X.toe. 4776 */ 4777 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 4778 NULL, "TOE parameters"); 4779 children = SYSCTL_CHILDREN(oid); 4780 4781 sc->tt.sndbuf = 256 * 1024; 4782 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 4783 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 4784 4785 sc->tt.ddp = 0; 4786 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 4787 &sc->tt.ddp, 0, "DDP allowed"); 4788 4789 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 4790 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 4791 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 4792 4793 sc->tt.ddp_thres = 4794 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 4795 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 4796 &sc->tt.ddp_thres, 0, "DDP threshold"); 4797 4798 sc->tt.rx_coalesce = 1; 4799 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", 4800 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); 4801 4802 sc->tt.tx_align = 1; 4803 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align", 4804 CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload"); 4805 } 4806 #endif 4807 4808 4809 return (0); 4810 } 4811 4812 static int 4813 cxgbe_sysctls(struct port_info *pi) 4814 { 4815 struct sysctl_ctx_list *ctx; 4816 struct sysctl_oid *oid; 4817 struct sysctl_oid_list *children; 4818 struct adapter *sc = pi->adapter; 4819 4820 ctx = device_get_sysctl_ctx(pi->dev); 4821 4822 /* 4823 * dev.cxgbe.X. 4824 */ 4825 oid = device_get_sysctl_tree(pi->dev); 4826 children = SYSCTL_CHILDREN(oid); 4827 4828 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | 4829 CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); 4830 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { 4831 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 4832 CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I", 4833 "PHY temperature (in Celsius)"); 4834 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", 4835 CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I", 4836 "PHY firmware version"); 4837 } 4838 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 4839 &pi->nrxq, 0, "# of rx queues"); 4840 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 4841 &pi->ntxq, 0, "# of tx queues"); 4842 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 4843 &pi->first_rxq, 0, "index of first rx queue"); 4844 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 4845 &pi->first_txq, 0, "index of first tx queue"); 4846 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT | 4847 CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU", 4848 "Reserve queue 0 for non-flowid packets"); 4849 4850 #ifdef TCP_OFFLOAD 4851 if (is_offload(sc)) { 4852 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 4853 &pi->nofldrxq, 0, 4854 "# of rx queues for offloaded TCP connections"); 4855 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 4856 &pi->nofldtxq, 0, 4857 "# of tx queues for offloaded TCP connections"); 4858 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 4859 CTLFLAG_RD, &pi->first_ofld_rxq, 0, 4860 "index of first TOE rx queue"); 4861 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 4862 CTLFLAG_RD, &pi->first_ofld_txq, 0, 4863 "index of first TOE tx queue"); 4864 } 4865 #endif 4866 #ifdef DEV_NETMAP 4867 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD, 4868 &pi->nnmrxq, 0, "# of rx queues for netmap"); 4869 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD, 4870 &pi->nnmtxq, 0, "# of tx queues for netmap"); 4871 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq", 4872 CTLFLAG_RD, &pi->first_nm_rxq, 0, 4873 "index of first netmap rx queue"); 4874 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq", 4875 CTLFLAG_RD, &pi->first_nm_txq, 0, 4876 "index of first netmap tx queue"); 4877 #endif 4878 4879 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 4880 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 4881 "holdoff timer index"); 4882 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 4883 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 4884 "holdoff packet counter index"); 4885 4886 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 4887 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 4888 "rx queue size"); 4889 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 4890 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 4891 "tx queue size"); 4892 4893 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings", 4894 CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings, 4895 "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)"); 4896 4897 /* 4898 * dev.cxgbe.X.stats. 4899 */ 4900 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 4901 NULL, "port statistics"); 4902 children = SYSCTL_CHILDREN(oid); 4903 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD, 4904 &pi->tx_parse_error, 0, 4905 "# of tx packets with invalid length or # of segments"); 4906 4907 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 4908 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 4909 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ 4910 sysctl_handle_t4_reg64, "QU", desc) 4911 4912 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 4913 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 4914 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 4915 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 4916 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 4917 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 4918 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 4919 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 4920 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 4921 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 4922 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 4923 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 4924 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 4925 "# of tx frames in this range", 4926 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 4927 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 4928 "# of tx frames in this range", 4929 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 4930 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 4931 "# of tx frames in this range", 4932 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 4933 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 4934 "# of tx frames in this range", 4935 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 4936 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 4937 "# of tx frames in this range", 4938 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 4939 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 4940 "# of tx frames in this range", 4941 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 4942 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 4943 "# of tx frames in this range", 4944 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 4945 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 4946 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 4947 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 4948 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 4949 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 4950 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 4951 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 4952 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 4953 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 4954 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 4955 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 4956 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 4957 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 4958 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 4959 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 4960 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 4961 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 4962 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 4963 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 4964 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 4965 4966 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 4967 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 4968 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 4969 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 4970 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 4971 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 4972 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 4973 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 4974 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 4975 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 4976 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 4977 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 4978 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 4979 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 4980 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 4981 "# of frames received with bad FCS", 4982 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 4983 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 4984 "# of frames received with length error", 4985 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 4986 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 4987 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 4988 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 4989 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 4990 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 4991 "# of rx frames in this range", 4992 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 4993 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 4994 "# of rx frames in this range", 4995 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 4996 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 4997 "# of rx frames in this range", 4998 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 4999 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 5000 "# of rx frames in this range", 5001 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 5002 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 5003 "# of rx frames in this range", 5004 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 5005 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 5006 "# of rx frames in this range", 5007 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 5008 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 5009 "# of rx frames in this range", 5010 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 5011 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 5012 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 5013 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 5014 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 5015 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 5016 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 5017 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 5018 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 5019 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 5020 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 5021 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 5022 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 5023 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 5024 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 5025 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 5026 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 5027 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 5028 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 5029 5030 #undef SYSCTL_ADD_T4_REG64 5031 5032 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 5033 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 5034 &pi->stats.name, desc) 5035 5036 /* We get these from port_stats and they may be stale by upto 1s */ 5037 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 5038 "# drops due to buffer-group 0 overflows"); 5039 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 5040 "# drops due to buffer-group 1 overflows"); 5041 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 5042 "# drops due to buffer-group 2 overflows"); 5043 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 5044 "# drops due to buffer-group 3 overflows"); 5045 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 5046 "# of buffer-group 0 truncated packets"); 5047 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 5048 "# of buffer-group 1 truncated packets"); 5049 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 5050 "# of buffer-group 2 truncated packets"); 5051 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 5052 "# of buffer-group 3 truncated packets"); 5053 5054 #undef SYSCTL_ADD_T4_PORTSTAT 5055 5056 return (0); 5057 } 5058 5059 static int 5060 sysctl_int_array(SYSCTL_HANDLER_ARGS) 5061 { 5062 int rc, *i, space = 0; 5063 struct sbuf sb; 5064 5065 sbuf_new_for_sysctl(&sb, NULL, 64, req); 5066 for (i = arg1; arg2; arg2 -= sizeof(int), i++) { 5067 if (space) 5068 sbuf_printf(&sb, " "); 5069 sbuf_printf(&sb, "%d", *i); 5070 space = 1; 5071 } 5072 rc = sbuf_finish(&sb); 5073 sbuf_delete(&sb); 5074 return (rc); 5075 } 5076 5077 static int 5078 sysctl_bitfield(SYSCTL_HANDLER_ARGS) 5079 { 5080 int rc; 5081 struct sbuf *sb; 5082 5083 rc = sysctl_wire_old_buffer(req, 0); 5084 if (rc != 0) 5085 return(rc); 5086 5087 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5088 if (sb == NULL) 5089 return (ENOMEM); 5090 5091 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 5092 rc = sbuf_finish(sb); 5093 sbuf_delete(sb); 5094 5095 return (rc); 5096 } 5097 5098 static int 5099 sysctl_btphy(SYSCTL_HANDLER_ARGS) 5100 { 5101 struct port_info *pi = arg1; 5102 int op = arg2; 5103 struct adapter *sc = pi->adapter; 5104 u_int v; 5105 int rc; 5106 5107 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt"); 5108 if (rc) 5109 return (rc); 5110 /* XXX: magic numbers */ 5111 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, 5112 &v); 5113 end_synchronized_op(sc, 0); 5114 if (rc) 5115 return (rc); 5116 if (op == 0) 5117 v /= 256; 5118 5119 rc = sysctl_handle_int(oidp, &v, 0, req); 5120 return (rc); 5121 } 5122 5123 static int 5124 sysctl_noflowq(SYSCTL_HANDLER_ARGS) 5125 { 5126 struct port_info *pi = arg1; 5127 int rc, val; 5128 5129 val = pi->rsrv_noflowq; 5130 rc = sysctl_handle_int(oidp, &val, 0, req); 5131 if (rc != 0 || req->newptr == NULL) 5132 return (rc); 5133 5134 if ((val >= 1) && (pi->ntxq > 1)) 5135 pi->rsrv_noflowq = 1; 5136 else 5137 pi->rsrv_noflowq = 0; 5138 5139 return (rc); 5140 } 5141 5142 static int 5143 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 5144 { 5145 struct port_info *pi = arg1; 5146 struct adapter *sc = pi->adapter; 5147 int idx, rc, i; 5148 struct sge_rxq *rxq; 5149 #ifdef TCP_OFFLOAD 5150 struct sge_ofld_rxq *ofld_rxq; 5151 #endif 5152 uint8_t v; 5153 5154 idx = pi->tmr_idx; 5155 5156 rc = sysctl_handle_int(oidp, &idx, 0, req); 5157 if (rc != 0 || req->newptr == NULL) 5158 return (rc); 5159 5160 if (idx < 0 || idx >= SGE_NTIMERS) 5161 return (EINVAL); 5162 5163 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5164 "t4tmr"); 5165 if (rc) 5166 return (rc); 5167 5168 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 5169 for_each_rxq(pi, i, rxq) { 5170 #ifdef atomic_store_rel_8 5171 atomic_store_rel_8(&rxq->iq.intr_params, v); 5172 #else 5173 rxq->iq.intr_params = v; 5174 #endif 5175 } 5176 #ifdef TCP_OFFLOAD 5177 for_each_ofld_rxq(pi, i, ofld_rxq) { 5178 #ifdef atomic_store_rel_8 5179 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); 5180 #else 5181 ofld_rxq->iq.intr_params = v; 5182 #endif 5183 } 5184 #endif 5185 pi->tmr_idx = idx; 5186 5187 end_synchronized_op(sc, LOCK_HELD); 5188 return (0); 5189 } 5190 5191 static int 5192 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 5193 { 5194 struct port_info *pi = arg1; 5195 struct adapter *sc = pi->adapter; 5196 int idx, rc; 5197 5198 idx = pi->pktc_idx; 5199 5200 rc = sysctl_handle_int(oidp, &idx, 0, req); 5201 if (rc != 0 || req->newptr == NULL) 5202 return (rc); 5203 5204 if (idx < -1 || idx >= SGE_NCOUNTERS) 5205 return (EINVAL); 5206 5207 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5208 "t4pktc"); 5209 if (rc) 5210 return (rc); 5211 5212 if (pi->flags & PORT_INIT_DONE) 5213 rc = EBUSY; /* cannot be changed once the queues are created */ 5214 else 5215 pi->pktc_idx = idx; 5216 5217 end_synchronized_op(sc, LOCK_HELD); 5218 return (rc); 5219 } 5220 5221 static int 5222 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 5223 { 5224 struct port_info *pi = arg1; 5225 struct adapter *sc = pi->adapter; 5226 int qsize, rc; 5227 5228 qsize = pi->qsize_rxq; 5229 5230 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5231 if (rc != 0 || req->newptr == NULL) 5232 return (rc); 5233 5234 if (qsize < 128 || (qsize & 7)) 5235 return (EINVAL); 5236 5237 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5238 "t4rxqs"); 5239 if (rc) 5240 return (rc); 5241 5242 if (pi->flags & PORT_INIT_DONE) 5243 rc = EBUSY; /* cannot be changed once the queues are created */ 5244 else 5245 pi->qsize_rxq = qsize; 5246 5247 end_synchronized_op(sc, LOCK_HELD); 5248 return (rc); 5249 } 5250 5251 static int 5252 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 5253 { 5254 struct port_info *pi = arg1; 5255 struct adapter *sc = pi->adapter; 5256 int qsize, rc; 5257 5258 qsize = pi->qsize_txq; 5259 5260 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5261 if (rc != 0 || req->newptr == NULL) 5262 return (rc); 5263 5264 if (qsize < 128 || qsize > 65536) 5265 return (EINVAL); 5266 5267 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5268 "t4txqs"); 5269 if (rc) 5270 return (rc); 5271 5272 if (pi->flags & PORT_INIT_DONE) 5273 rc = EBUSY; /* cannot be changed once the queues are created */ 5274 else 5275 pi->qsize_txq = qsize; 5276 5277 end_synchronized_op(sc, LOCK_HELD); 5278 return (rc); 5279 } 5280 5281 static int 5282 sysctl_pause_settings(SYSCTL_HANDLER_ARGS) 5283 { 5284 struct port_info *pi = arg1; 5285 struct adapter *sc = pi->adapter; 5286 struct link_config *lc = &pi->link_cfg; 5287 int rc; 5288 5289 if (req->newptr == NULL) { 5290 struct sbuf *sb; 5291 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX"; 5292 5293 rc = sysctl_wire_old_buffer(req, 0); 5294 if (rc != 0) 5295 return(rc); 5296 5297 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5298 if (sb == NULL) 5299 return (ENOMEM); 5300 5301 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits); 5302 rc = sbuf_finish(sb); 5303 sbuf_delete(sb); 5304 } else { 5305 char s[2]; 5306 int n; 5307 5308 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX)); 5309 s[1] = 0; 5310 5311 rc = sysctl_handle_string(oidp, s, sizeof(s), req); 5312 if (rc != 0) 5313 return(rc); 5314 5315 if (s[1] != 0) 5316 return (EINVAL); 5317 if (s[0] < '0' || s[0] > '9') 5318 return (EINVAL); /* not a number */ 5319 n = s[0] - '0'; 5320 if (n & ~(PAUSE_TX | PAUSE_RX)) 5321 return (EINVAL); /* some other bit is set too */ 5322 5323 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE"); 5324 if (rc) 5325 return (rc); 5326 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) { 5327 int link_ok = lc->link_ok; 5328 5329 lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX); 5330 lc->requested_fc |= n; 5331 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc); 5332 lc->link_ok = link_ok; /* restore */ 5333 } 5334 end_synchronized_op(sc, 0); 5335 } 5336 5337 return (rc); 5338 } 5339 5340 static int 5341 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 5342 { 5343 struct adapter *sc = arg1; 5344 int reg = arg2; 5345 uint64_t val; 5346 5347 val = t4_read_reg64(sc, reg); 5348 5349 return (sysctl_handle_64(oidp, &val, 0, req)); 5350 } 5351 5352 static int 5353 sysctl_temperature(SYSCTL_HANDLER_ARGS) 5354 { 5355 struct adapter *sc = arg1; 5356 int rc, t; 5357 uint32_t param, val; 5358 5359 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); 5360 if (rc) 5361 return (rc); 5362 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 5363 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 5364 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); 5365 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 5366 end_synchronized_op(sc, 0); 5367 if (rc) 5368 return (rc); 5369 5370 /* unknown is returned as 0 but we display -1 in that case */ 5371 t = val == 0 ? -1 : val; 5372 5373 rc = sysctl_handle_int(oidp, &t, 0, req); 5374 return (rc); 5375 } 5376 5377 #ifdef SBUF_DRAIN 5378 static int 5379 sysctl_cctrl(SYSCTL_HANDLER_ARGS) 5380 { 5381 struct adapter *sc = arg1; 5382 struct sbuf *sb; 5383 int rc, i; 5384 uint16_t incr[NMTUS][NCCTRL_WIN]; 5385 static const char *dec_fac[] = { 5386 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 5387 "0.9375" 5388 }; 5389 5390 rc = sysctl_wire_old_buffer(req, 0); 5391 if (rc != 0) 5392 return (rc); 5393 5394 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5395 if (sb == NULL) 5396 return (ENOMEM); 5397 5398 t4_read_cong_tbl(sc, incr); 5399 5400 for (i = 0; i < NCCTRL_WIN; ++i) { 5401 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 5402 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 5403 incr[5][i], incr[6][i], incr[7][i]); 5404 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 5405 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 5406 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 5407 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 5408 } 5409 5410 rc = sbuf_finish(sb); 5411 sbuf_delete(sb); 5412 5413 return (rc); 5414 } 5415 5416 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 5417 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 5418 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 5419 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 5420 }; 5421 5422 static int 5423 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 5424 { 5425 struct adapter *sc = arg1; 5426 struct sbuf *sb; 5427 int rc, i, n, qid = arg2; 5428 uint32_t *buf, *p; 5429 char *qtype; 5430 u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5; 5431 5432 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 5433 ("%s: bad qid %d\n", __func__, qid)); 5434 5435 if (qid < CIM_NUM_IBQ) { 5436 /* inbound queue */ 5437 qtype = "IBQ"; 5438 n = 4 * CIM_IBQ_SIZE; 5439 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5440 rc = t4_read_cim_ibq(sc, qid, buf, n); 5441 } else { 5442 /* outbound queue */ 5443 qtype = "OBQ"; 5444 qid -= CIM_NUM_IBQ; 5445 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 5446 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5447 rc = t4_read_cim_obq(sc, qid, buf, n); 5448 } 5449 5450 if (rc < 0) { 5451 rc = -rc; 5452 goto done; 5453 } 5454 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 5455 5456 rc = sysctl_wire_old_buffer(req, 0); 5457 if (rc != 0) 5458 goto done; 5459 5460 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5461 if (sb == NULL) { 5462 rc = ENOMEM; 5463 goto done; 5464 } 5465 5466 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 5467 for (i = 0, p = buf; i < n; i += 16, p += 4) 5468 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 5469 p[2], p[3]); 5470 5471 rc = sbuf_finish(sb); 5472 sbuf_delete(sb); 5473 done: 5474 free(buf, M_CXGBE); 5475 return (rc); 5476 } 5477 5478 static int 5479 sysctl_cim_la(SYSCTL_HANDLER_ARGS) 5480 { 5481 struct adapter *sc = arg1; 5482 u_int cfg; 5483 struct sbuf *sb; 5484 uint32_t *buf, *p; 5485 int rc; 5486 5487 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 5488 if (rc != 0) 5489 return (rc); 5490 5491 rc = sysctl_wire_old_buffer(req, 0); 5492 if (rc != 0) 5493 return (rc); 5494 5495 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5496 if (sb == NULL) 5497 return (ENOMEM); 5498 5499 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 5500 M_ZERO | M_WAITOK); 5501 5502 rc = -t4_cim_read_la(sc, buf, NULL); 5503 if (rc != 0) 5504 goto done; 5505 5506 sbuf_printf(sb, "Status Data PC%s", 5507 cfg & F_UPDBGLACAPTPCONLY ? "" : 5508 " LS0Stat LS0Addr LS0Data"); 5509 5510 KASSERT((sc->params.cim_la_size & 7) == 0, 5511 ("%s: p will walk off the end of buf", __func__)); 5512 5513 for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) { 5514 if (cfg & F_UPDBGLACAPTPCONLY) { 5515 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 5516 p[6], p[7]); 5517 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 5518 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 5519 p[4] & 0xff, p[5] >> 8); 5520 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 5521 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5522 p[1] & 0xf, p[2] >> 4); 5523 } else { 5524 sbuf_printf(sb, 5525 "\n %02x %x%07x %x%07x %08x %08x " 5526 "%08x%08x%08x%08x", 5527 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5528 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 5529 p[6], p[7]); 5530 } 5531 } 5532 5533 rc = sbuf_finish(sb); 5534 sbuf_delete(sb); 5535 done: 5536 free(buf, M_CXGBE); 5537 return (rc); 5538 } 5539 5540 static int 5541 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) 5542 { 5543 struct adapter *sc = arg1; 5544 u_int i; 5545 struct sbuf *sb; 5546 uint32_t *buf, *p; 5547 int rc; 5548 5549 rc = sysctl_wire_old_buffer(req, 0); 5550 if (rc != 0) 5551 return (rc); 5552 5553 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5554 if (sb == NULL) 5555 return (ENOMEM); 5556 5557 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, 5558 M_ZERO | M_WAITOK); 5559 5560 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); 5561 p = buf; 5562 5563 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5564 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], 5565 p[1], p[0]); 5566 } 5567 5568 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); 5569 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5570 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", 5571 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, 5572 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, 5573 (p[1] >> 2) | ((p[2] & 3) << 30), 5574 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, 5575 p[0] & 1); 5576 } 5577 5578 rc = sbuf_finish(sb); 5579 sbuf_delete(sb); 5580 free(buf, M_CXGBE); 5581 return (rc); 5582 } 5583 5584 static int 5585 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) 5586 { 5587 struct adapter *sc = arg1; 5588 u_int i; 5589 struct sbuf *sb; 5590 uint32_t *buf, *p; 5591 int rc; 5592 5593 rc = sysctl_wire_old_buffer(req, 0); 5594 if (rc != 0) 5595 return (rc); 5596 5597 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5598 if (sb == NULL) 5599 return (ENOMEM); 5600 5601 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, 5602 M_ZERO | M_WAITOK); 5603 5604 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); 5605 p = buf; 5606 5607 sbuf_printf(sb, "Cntl ID DataBE Addr Data"); 5608 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5609 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", 5610 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, 5611 p[4], p[3], p[2], p[1], p[0]); 5612 } 5613 5614 sbuf_printf(sb, "\n\nCntl ID Data"); 5615 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5616 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", 5617 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); 5618 } 5619 5620 rc = sbuf_finish(sb); 5621 sbuf_delete(sb); 5622 free(buf, M_CXGBE); 5623 return (rc); 5624 } 5625 5626 static int 5627 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 5628 { 5629 struct adapter *sc = arg1; 5630 struct sbuf *sb; 5631 int rc, i; 5632 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5633 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5634 uint16_t thres[CIM_NUM_IBQ]; 5635 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 5636 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 5637 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 5638 5639 if (is_t4(sc)) { 5640 cim_num_obq = CIM_NUM_OBQ; 5641 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 5642 obq_rdaddr = A_UP_OBQ_0_REALADDR; 5643 } else { 5644 cim_num_obq = CIM_NUM_OBQ_T5; 5645 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 5646 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 5647 } 5648 nq = CIM_NUM_IBQ + cim_num_obq; 5649 5650 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 5651 if (rc == 0) 5652 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 5653 if (rc != 0) 5654 return (rc); 5655 5656 t4_read_cimq_cfg(sc, base, size, thres); 5657 5658 rc = sysctl_wire_old_buffer(req, 0); 5659 if (rc != 0) 5660 return (rc); 5661 5662 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5663 if (sb == NULL) 5664 return (ENOMEM); 5665 5666 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 5667 5668 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 5669 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 5670 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 5671 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5672 G_QUEREMFLITS(p[2]) * 16); 5673 for ( ; i < nq; i++, p += 4, wr += 2) 5674 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 5675 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 5676 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5677 G_QUEREMFLITS(p[2]) * 16); 5678 5679 rc = sbuf_finish(sb); 5680 sbuf_delete(sb); 5681 5682 return (rc); 5683 } 5684 5685 static int 5686 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 5687 { 5688 struct adapter *sc = arg1; 5689 struct sbuf *sb; 5690 int rc; 5691 struct tp_cpl_stats stats; 5692 5693 rc = sysctl_wire_old_buffer(req, 0); 5694 if (rc != 0) 5695 return (rc); 5696 5697 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5698 if (sb == NULL) 5699 return (ENOMEM); 5700 5701 t4_tp_get_cpl_stats(sc, &stats); 5702 5703 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 5704 "channel 3\n"); 5705 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 5706 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 5707 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 5708 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 5709 5710 rc = sbuf_finish(sb); 5711 sbuf_delete(sb); 5712 5713 return (rc); 5714 } 5715 5716 static int 5717 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 5718 { 5719 struct adapter *sc = arg1; 5720 struct sbuf *sb; 5721 int rc; 5722 struct tp_usm_stats stats; 5723 5724 rc = sysctl_wire_old_buffer(req, 0); 5725 if (rc != 0) 5726 return(rc); 5727 5728 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5729 if (sb == NULL) 5730 return (ENOMEM); 5731 5732 t4_get_usm_stats(sc, &stats); 5733 5734 sbuf_printf(sb, "Frames: %u\n", stats.frames); 5735 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 5736 sbuf_printf(sb, "Drops: %u", stats.drops); 5737 5738 rc = sbuf_finish(sb); 5739 sbuf_delete(sb); 5740 5741 return (rc); 5742 } 5743 5744 const char *devlog_level_strings[] = { 5745 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 5746 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 5747 [FW_DEVLOG_LEVEL_ERR] = "ERR", 5748 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 5749 [FW_DEVLOG_LEVEL_INFO] = "INFO", 5750 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 5751 }; 5752 5753 const char *devlog_facility_strings[] = { 5754 [FW_DEVLOG_FACILITY_CORE] = "CORE", 5755 [FW_DEVLOG_FACILITY_CF] = "CF", 5756 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 5757 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 5758 [FW_DEVLOG_FACILITY_RES] = "RES", 5759 [FW_DEVLOG_FACILITY_HW] = "HW", 5760 [FW_DEVLOG_FACILITY_FLR] = "FLR", 5761 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 5762 [FW_DEVLOG_FACILITY_PHY] = "PHY", 5763 [FW_DEVLOG_FACILITY_MAC] = "MAC", 5764 [FW_DEVLOG_FACILITY_PORT] = "PORT", 5765 [FW_DEVLOG_FACILITY_VI] = "VI", 5766 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 5767 [FW_DEVLOG_FACILITY_ACL] = "ACL", 5768 [FW_DEVLOG_FACILITY_TM] = "TM", 5769 [FW_DEVLOG_FACILITY_QFC] = "QFC", 5770 [FW_DEVLOG_FACILITY_DCB] = "DCB", 5771 [FW_DEVLOG_FACILITY_ETH] = "ETH", 5772 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 5773 [FW_DEVLOG_FACILITY_RI] = "RI", 5774 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 5775 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 5776 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 5777 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 5778 }; 5779 5780 static int 5781 sysctl_devlog(SYSCTL_HANDLER_ARGS) 5782 { 5783 struct adapter *sc = arg1; 5784 struct devlog_params *dparams = &sc->params.devlog; 5785 struct fw_devlog_e *buf, *e; 5786 int i, j, rc, nentries, first = 0, m; 5787 struct sbuf *sb; 5788 uint64_t ftstamp = UINT64_MAX; 5789 5790 if (dparams->start == 0) { 5791 dparams->memtype = FW_MEMTYPE_EDC0; 5792 dparams->start = 0x84000; 5793 dparams->size = 32768; 5794 } 5795 5796 nentries = dparams->size / sizeof(struct fw_devlog_e); 5797 5798 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 5799 if (buf == NULL) 5800 return (ENOMEM); 5801 5802 m = fwmtype_to_hwmtype(dparams->memtype); 5803 rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf); 5804 if (rc != 0) 5805 goto done; 5806 5807 for (i = 0; i < nentries; i++) { 5808 e = &buf[i]; 5809 5810 if (e->timestamp == 0) 5811 break; /* end */ 5812 5813 e->timestamp = be64toh(e->timestamp); 5814 e->seqno = be32toh(e->seqno); 5815 for (j = 0; j < 8; j++) 5816 e->params[j] = be32toh(e->params[j]); 5817 5818 if (e->timestamp < ftstamp) { 5819 ftstamp = e->timestamp; 5820 first = i; 5821 } 5822 } 5823 5824 if (buf[first].timestamp == 0) 5825 goto done; /* nothing in the log */ 5826 5827 rc = sysctl_wire_old_buffer(req, 0); 5828 if (rc != 0) 5829 goto done; 5830 5831 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5832 if (sb == NULL) { 5833 rc = ENOMEM; 5834 goto done; 5835 } 5836 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 5837 "Seq#", "Tstamp", "Level", "Facility", "Message"); 5838 5839 i = first; 5840 do { 5841 e = &buf[i]; 5842 if (e->timestamp == 0) 5843 break; /* end */ 5844 5845 sbuf_printf(sb, "%10d %15ju %8s %8s ", 5846 e->seqno, e->timestamp, 5847 (e->level < nitems(devlog_level_strings) ? 5848 devlog_level_strings[e->level] : "UNKNOWN"), 5849 (e->facility < nitems(devlog_facility_strings) ? 5850 devlog_facility_strings[e->facility] : "UNKNOWN")); 5851 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 5852 e->params[2], e->params[3], e->params[4], 5853 e->params[5], e->params[6], e->params[7]); 5854 5855 if (++i == nentries) 5856 i = 0; 5857 } while (i != first); 5858 5859 rc = sbuf_finish(sb); 5860 sbuf_delete(sb); 5861 done: 5862 free(buf, M_CXGBE); 5863 return (rc); 5864 } 5865 5866 static int 5867 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 5868 { 5869 struct adapter *sc = arg1; 5870 struct sbuf *sb; 5871 int rc; 5872 struct tp_fcoe_stats stats[4]; 5873 5874 rc = sysctl_wire_old_buffer(req, 0); 5875 if (rc != 0) 5876 return (rc); 5877 5878 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5879 if (sb == NULL) 5880 return (ENOMEM); 5881 5882 t4_get_fcoe_stats(sc, 0, &stats[0]); 5883 t4_get_fcoe_stats(sc, 1, &stats[1]); 5884 t4_get_fcoe_stats(sc, 2, &stats[2]); 5885 t4_get_fcoe_stats(sc, 3, &stats[3]); 5886 5887 sbuf_printf(sb, " channel 0 channel 1 " 5888 "channel 2 channel 3\n"); 5889 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 5890 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 5891 stats[3].octetsDDP); 5892 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 5893 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 5894 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 5895 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 5896 stats[3].framesDrop); 5897 5898 rc = sbuf_finish(sb); 5899 sbuf_delete(sb); 5900 5901 return (rc); 5902 } 5903 5904 static int 5905 sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 5906 { 5907 struct adapter *sc = arg1; 5908 struct sbuf *sb; 5909 int rc, i; 5910 unsigned int map, kbps, ipg, mode; 5911 unsigned int pace_tab[NTX_SCHED]; 5912 5913 rc = sysctl_wire_old_buffer(req, 0); 5914 if (rc != 0) 5915 return (rc); 5916 5917 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5918 if (sb == NULL) 5919 return (ENOMEM); 5920 5921 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 5922 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 5923 t4_read_pace_tbl(sc, pace_tab); 5924 5925 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 5926 "Class IPG (0.1 ns) Flow IPG (us)"); 5927 5928 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 5929 t4_get_tx_sched(sc, i, &kbps, &ipg); 5930 sbuf_printf(sb, "\n %u %-5s %u ", i, 5931 (mode & (1 << i)) ? "flow" : "class", map & 3); 5932 if (kbps) 5933 sbuf_printf(sb, "%9u ", kbps); 5934 else 5935 sbuf_printf(sb, " disabled "); 5936 5937 if (ipg) 5938 sbuf_printf(sb, "%13u ", ipg); 5939 else 5940 sbuf_printf(sb, " disabled "); 5941 5942 if (pace_tab[i]) 5943 sbuf_printf(sb, "%10u", pace_tab[i]); 5944 else 5945 sbuf_printf(sb, " disabled"); 5946 } 5947 5948 rc = sbuf_finish(sb); 5949 sbuf_delete(sb); 5950 5951 return (rc); 5952 } 5953 5954 static int 5955 sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 5956 { 5957 struct adapter *sc = arg1; 5958 struct sbuf *sb; 5959 int rc, i, j; 5960 uint64_t *p0, *p1; 5961 struct lb_port_stats s[2]; 5962 static const char *stat_name[] = { 5963 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 5964 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 5965 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 5966 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 5967 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 5968 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 5969 "BG2FramesTrunc:", "BG3FramesTrunc:" 5970 }; 5971 5972 rc = sysctl_wire_old_buffer(req, 0); 5973 if (rc != 0) 5974 return (rc); 5975 5976 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5977 if (sb == NULL) 5978 return (ENOMEM); 5979 5980 memset(s, 0, sizeof(s)); 5981 5982 for (i = 0; i < 4; i += 2) { 5983 t4_get_lb_stats(sc, i, &s[0]); 5984 t4_get_lb_stats(sc, i + 1, &s[1]); 5985 5986 p0 = &s[0].octets; 5987 p1 = &s[1].octets; 5988 sbuf_printf(sb, "%s Loopback %u" 5989 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 5990 5991 for (j = 0; j < nitems(stat_name); j++) 5992 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 5993 *p0++, *p1++); 5994 } 5995 5996 rc = sbuf_finish(sb); 5997 sbuf_delete(sb); 5998 5999 return (rc); 6000 } 6001 6002 static int 6003 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) 6004 { 6005 int rc = 0; 6006 struct port_info *pi = arg1; 6007 struct sbuf *sb; 6008 static const char *linkdnreasons[] = { 6009 "non-specific", "remote fault", "autoneg failed", "reserved3", 6010 "PHY overheated", "unknown", "rx los", "reserved7" 6011 }; 6012 6013 rc = sysctl_wire_old_buffer(req, 0); 6014 if (rc != 0) 6015 return(rc); 6016 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); 6017 if (sb == NULL) 6018 return (ENOMEM); 6019 6020 if (pi->linkdnrc < 0) 6021 sbuf_printf(sb, "n/a"); 6022 else if (pi->linkdnrc < nitems(linkdnreasons)) 6023 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]); 6024 else 6025 sbuf_printf(sb, "%d", pi->linkdnrc); 6026 6027 rc = sbuf_finish(sb); 6028 sbuf_delete(sb); 6029 6030 return (rc); 6031 } 6032 6033 struct mem_desc { 6034 unsigned int base; 6035 unsigned int limit; 6036 unsigned int idx; 6037 }; 6038 6039 static int 6040 mem_desc_cmp(const void *a, const void *b) 6041 { 6042 return ((const struct mem_desc *)a)->base - 6043 ((const struct mem_desc *)b)->base; 6044 } 6045 6046 static void 6047 mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 6048 unsigned int to) 6049 { 6050 unsigned int size; 6051 6052 size = to - from + 1; 6053 if (size == 0) 6054 return; 6055 6056 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 6057 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 6058 } 6059 6060 static int 6061 sysctl_meminfo(SYSCTL_HANDLER_ARGS) 6062 { 6063 struct adapter *sc = arg1; 6064 struct sbuf *sb; 6065 int rc, i, n; 6066 uint32_t lo, hi, used, alloc; 6067 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 6068 static const char *region[] = { 6069 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 6070 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 6071 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 6072 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 6073 "RQUDP region:", "PBL region:", "TXPBL region:", 6074 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 6075 "On-chip queues:" 6076 }; 6077 struct mem_desc avail[4]; 6078 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 6079 struct mem_desc *md = mem; 6080 6081 rc = sysctl_wire_old_buffer(req, 0); 6082 if (rc != 0) 6083 return (rc); 6084 6085 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6086 if (sb == NULL) 6087 return (ENOMEM); 6088 6089 for (i = 0; i < nitems(mem); i++) { 6090 mem[i].limit = 0; 6091 mem[i].idx = i; 6092 } 6093 6094 /* Find and sort the populated memory ranges */ 6095 i = 0; 6096 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 6097 if (lo & F_EDRAM0_ENABLE) { 6098 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 6099 avail[i].base = G_EDRAM0_BASE(hi) << 20; 6100 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 6101 avail[i].idx = 0; 6102 i++; 6103 } 6104 if (lo & F_EDRAM1_ENABLE) { 6105 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 6106 avail[i].base = G_EDRAM1_BASE(hi) << 20; 6107 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 6108 avail[i].idx = 1; 6109 i++; 6110 } 6111 if (lo & F_EXT_MEM_ENABLE) { 6112 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 6113 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 6114 avail[i].limit = avail[i].base + 6115 (G_EXT_MEM_SIZE(hi) << 20); 6116 avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */ 6117 i++; 6118 } 6119 if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) { 6120 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 6121 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 6122 avail[i].limit = avail[i].base + 6123 (G_EXT_MEM1_SIZE(hi) << 20); 6124 avail[i].idx = 4; 6125 i++; 6126 } 6127 if (!i) /* no memory available */ 6128 return 0; 6129 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 6130 6131 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 6132 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 6133 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 6134 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 6135 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 6136 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 6137 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 6138 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 6139 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 6140 6141 /* the next few have explicit upper bounds */ 6142 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 6143 md->limit = md->base - 1 + 6144 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 6145 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 6146 md++; 6147 6148 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 6149 md->limit = md->base - 1 + 6150 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 6151 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 6152 md++; 6153 6154 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6155 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 6156 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 6157 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 6158 } else { 6159 md->base = 0; 6160 md->idx = nitems(region); /* hide it */ 6161 } 6162 md++; 6163 6164 #define ulp_region(reg) \ 6165 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 6166 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 6167 6168 ulp_region(RX_ISCSI); 6169 ulp_region(RX_TDDP); 6170 ulp_region(TX_TPT); 6171 ulp_region(RX_STAG); 6172 ulp_region(RX_RQ); 6173 ulp_region(RX_RQUDP); 6174 ulp_region(RX_PBL); 6175 ulp_region(TX_PBL); 6176 #undef ulp_region 6177 6178 md->base = 0; 6179 md->idx = nitems(region); 6180 if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) { 6181 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); 6182 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc, 6183 A_SGE_DBVFIFO_SIZE))) << 2) - 1; 6184 } 6185 md++; 6186 6187 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 6188 md->limit = md->base + sc->tids.ntids - 1; 6189 md++; 6190 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 6191 md->limit = md->base + sc->tids.ntids - 1; 6192 md++; 6193 6194 md->base = sc->vres.ocq.start; 6195 if (sc->vres.ocq.size) 6196 md->limit = md->base + sc->vres.ocq.size - 1; 6197 else 6198 md->idx = nitems(region); /* hide it */ 6199 md++; 6200 6201 /* add any address-space holes, there can be up to 3 */ 6202 for (n = 0; n < i - 1; n++) 6203 if (avail[n].limit < avail[n + 1].base) 6204 (md++)->base = avail[n].limit; 6205 if (avail[n].limit) 6206 (md++)->base = avail[n].limit; 6207 6208 n = md - mem; 6209 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 6210 6211 for (lo = 0; lo < i; lo++) 6212 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 6213 avail[lo].limit - 1); 6214 6215 sbuf_printf(sb, "\n"); 6216 for (i = 0; i < n; i++) { 6217 if (mem[i].idx >= nitems(region)) 6218 continue; /* skip holes */ 6219 if (!mem[i].limit) 6220 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 6221 mem_region_show(sb, region[mem[i].idx], mem[i].base, 6222 mem[i].limit); 6223 } 6224 6225 sbuf_printf(sb, "\n"); 6226 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 6227 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 6228 mem_region_show(sb, "uP RAM:", lo, hi); 6229 6230 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 6231 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 6232 mem_region_show(sb, "uP Extmem2:", lo, hi); 6233 6234 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 6235 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 6236 G_PMRXMAXPAGE(lo), 6237 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 6238 (lo & F_PMRXNUMCHN) ? 2 : 1); 6239 6240 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 6241 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 6242 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 6243 G_PMTXMAXPAGE(lo), 6244 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 6245 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 6246 sbuf_printf(sb, "%u p-structs\n", 6247 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 6248 6249 for (i = 0; i < 4; i++) { 6250 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 6251 if (is_t4(sc)) { 6252 used = G_USED(lo); 6253 alloc = G_ALLOC(lo); 6254 } else { 6255 used = G_T5_USED(lo); 6256 alloc = G_T5_ALLOC(lo); 6257 } 6258 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 6259 i, used, alloc); 6260 } 6261 for (i = 0; i < 4; i++) { 6262 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 6263 if (is_t4(sc)) { 6264 used = G_USED(lo); 6265 alloc = G_ALLOC(lo); 6266 } else { 6267 used = G_T5_USED(lo); 6268 alloc = G_T5_ALLOC(lo); 6269 } 6270 sbuf_printf(sb, 6271 "\nLoopback %d using %u pages out of %u allocated", 6272 i, used, alloc); 6273 } 6274 6275 rc = sbuf_finish(sb); 6276 sbuf_delete(sb); 6277 6278 return (rc); 6279 } 6280 6281 static inline void 6282 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) 6283 { 6284 *mask = x | y; 6285 y = htobe64(y); 6286 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); 6287 } 6288 6289 static int 6290 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) 6291 { 6292 struct adapter *sc = arg1; 6293 struct sbuf *sb; 6294 int rc, i, n; 6295 6296 rc = sysctl_wire_old_buffer(req, 0); 6297 if (rc != 0) 6298 return (rc); 6299 6300 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6301 if (sb == NULL) 6302 return (ENOMEM); 6303 6304 sbuf_printf(sb, 6305 "Idx Ethernet address Mask Vld Ports PF" 6306 " VF Replication P0 P1 P2 P3 ML"); 6307 n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES : 6308 NUM_MPS_T5_CLS_SRAM_L_INSTANCES; 6309 for (i = 0; i < n; i++) { 6310 uint64_t tcamx, tcamy, mask; 6311 uint32_t cls_lo, cls_hi; 6312 uint8_t addr[ETHER_ADDR_LEN]; 6313 6314 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); 6315 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); 6316 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 6317 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 6318 6319 if (tcamx & tcamy) 6320 continue; 6321 6322 tcamxy2valmask(tcamx, tcamy, addr, &mask); 6323 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" 6324 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], 6325 addr[3], addr[4], addr[5], (uintmax_t)mask, 6326 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', 6327 G_PORTMAP(cls_hi), G_PF(cls_lo), 6328 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); 6329 6330 if (cls_lo & F_REPLICATE) { 6331 struct fw_ldst_cmd ldst_cmd; 6332 6333 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 6334 ldst_cmd.op_to_addrspace = 6335 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 6336 F_FW_CMD_REQUEST | F_FW_CMD_READ | 6337 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 6338 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 6339 ldst_cmd.u.mps.fid_ctl = 6340 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 6341 V_FW_LDST_CMD_CTL(i)); 6342 6343 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 6344 "t4mps"); 6345 if (rc) 6346 break; 6347 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 6348 sizeof(ldst_cmd), &ldst_cmd); 6349 end_synchronized_op(sc, 0); 6350 6351 if (rc != 0) { 6352 sbuf_printf(sb, 6353 " ------------ error %3u ------------", rc); 6354 rc = 0; 6355 } else { 6356 sbuf_printf(sb, " %08x %08x %08x %08x", 6357 be32toh(ldst_cmd.u.mps.rplc127_96), 6358 be32toh(ldst_cmd.u.mps.rplc95_64), 6359 be32toh(ldst_cmd.u.mps.rplc63_32), 6360 be32toh(ldst_cmd.u.mps.rplc31_0)); 6361 } 6362 } else 6363 sbuf_printf(sb, "%36s", ""); 6364 6365 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), 6366 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), 6367 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); 6368 } 6369 6370 if (rc) 6371 (void) sbuf_finish(sb); 6372 else 6373 rc = sbuf_finish(sb); 6374 sbuf_delete(sb); 6375 6376 return (rc); 6377 } 6378 6379 static int 6380 sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 6381 { 6382 struct adapter *sc = arg1; 6383 struct sbuf *sb; 6384 int rc; 6385 uint16_t mtus[NMTUS]; 6386 6387 rc = sysctl_wire_old_buffer(req, 0); 6388 if (rc != 0) 6389 return (rc); 6390 6391 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6392 if (sb == NULL) 6393 return (ENOMEM); 6394 6395 t4_read_mtu_tbl(sc, mtus, NULL); 6396 6397 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 6398 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 6399 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 6400 mtus[14], mtus[15]); 6401 6402 rc = sbuf_finish(sb); 6403 sbuf_delete(sb); 6404 6405 return (rc); 6406 } 6407 6408 static int 6409 sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 6410 { 6411 struct adapter *sc = arg1; 6412 struct sbuf *sb; 6413 int rc, i; 6414 uint32_t cnt[PM_NSTATS]; 6415 uint64_t cyc[PM_NSTATS]; 6416 static const char *rx_stats[] = { 6417 "Read:", "Write bypass:", "Write mem:", "Flush:" 6418 }; 6419 static const char *tx_stats[] = { 6420 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:" 6421 }; 6422 6423 rc = sysctl_wire_old_buffer(req, 0); 6424 if (rc != 0) 6425 return (rc); 6426 6427 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6428 if (sb == NULL) 6429 return (ENOMEM); 6430 6431 t4_pmtx_get_stats(sc, cnt, cyc); 6432 sbuf_printf(sb, " Tx pcmds Tx bytes"); 6433 for (i = 0; i < ARRAY_SIZE(tx_stats); i++) 6434 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i], 6435 cyc[i]); 6436 6437 t4_pmrx_get_stats(sc, cnt, cyc); 6438 sbuf_printf(sb, "\n Rx pcmds Rx bytes"); 6439 for (i = 0; i < ARRAY_SIZE(rx_stats); i++) 6440 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i], 6441 cyc[i]); 6442 6443 rc = sbuf_finish(sb); 6444 sbuf_delete(sb); 6445 6446 return (rc); 6447 } 6448 6449 static int 6450 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 6451 { 6452 struct adapter *sc = arg1; 6453 struct sbuf *sb; 6454 int rc; 6455 struct tp_rdma_stats stats; 6456 6457 rc = sysctl_wire_old_buffer(req, 0); 6458 if (rc != 0) 6459 return (rc); 6460 6461 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6462 if (sb == NULL) 6463 return (ENOMEM); 6464 6465 t4_tp_get_rdma_stats(sc, &stats); 6466 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 6467 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 6468 6469 rc = sbuf_finish(sb); 6470 sbuf_delete(sb); 6471 6472 return (rc); 6473 } 6474 6475 static int 6476 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 6477 { 6478 struct adapter *sc = arg1; 6479 struct sbuf *sb; 6480 int rc; 6481 struct tp_tcp_stats v4, v6; 6482 6483 rc = sysctl_wire_old_buffer(req, 0); 6484 if (rc != 0) 6485 return (rc); 6486 6487 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6488 if (sb == NULL) 6489 return (ENOMEM); 6490 6491 t4_tp_get_tcp_stats(sc, &v4, &v6); 6492 sbuf_printf(sb, 6493 " IP IPv6\n"); 6494 sbuf_printf(sb, "OutRsts: %20u %20u\n", 6495 v4.tcpOutRsts, v6.tcpOutRsts); 6496 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 6497 v4.tcpInSegs, v6.tcpInSegs); 6498 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 6499 v4.tcpOutSegs, v6.tcpOutSegs); 6500 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 6501 v4.tcpRetransSegs, v6.tcpRetransSegs); 6502 6503 rc = sbuf_finish(sb); 6504 sbuf_delete(sb); 6505 6506 return (rc); 6507 } 6508 6509 static int 6510 sysctl_tids(SYSCTL_HANDLER_ARGS) 6511 { 6512 struct adapter *sc = arg1; 6513 struct sbuf *sb; 6514 int rc; 6515 struct tid_info *t = &sc->tids; 6516 6517 rc = sysctl_wire_old_buffer(req, 0); 6518 if (rc != 0) 6519 return (rc); 6520 6521 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6522 if (sb == NULL) 6523 return (ENOMEM); 6524 6525 if (t->natids) { 6526 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 6527 t->atids_in_use); 6528 } 6529 6530 if (t->ntids) { 6531 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6532 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 6533 6534 if (b) { 6535 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 6536 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6537 t->ntids - 1); 6538 } else { 6539 sbuf_printf(sb, "TID range: %u-%u", 6540 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6541 t->ntids - 1); 6542 } 6543 } else 6544 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 6545 sbuf_printf(sb, ", in use: %u\n", 6546 atomic_load_acq_int(&t->tids_in_use)); 6547 } 6548 6549 if (t->nstids) { 6550 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 6551 t->stid_base + t->nstids - 1, t->stids_in_use); 6552 } 6553 6554 if (t->nftids) { 6555 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 6556 t->ftid_base + t->nftids - 1); 6557 } 6558 6559 if (t->netids) { 6560 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base, 6561 t->etid_base + t->netids - 1); 6562 } 6563 6564 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 6565 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 6566 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 6567 6568 rc = sbuf_finish(sb); 6569 sbuf_delete(sb); 6570 6571 return (rc); 6572 } 6573 6574 static int 6575 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 6576 { 6577 struct adapter *sc = arg1; 6578 struct sbuf *sb; 6579 int rc; 6580 struct tp_err_stats stats; 6581 6582 rc = sysctl_wire_old_buffer(req, 0); 6583 if (rc != 0) 6584 return (rc); 6585 6586 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6587 if (sb == NULL) 6588 return (ENOMEM); 6589 6590 t4_tp_get_err_stats(sc, &stats); 6591 6592 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6593 "channel 3\n"); 6594 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 6595 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 6596 stats.macInErrs[3]); 6597 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 6598 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 6599 stats.hdrInErrs[3]); 6600 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 6601 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 6602 stats.tcpInErrs[3]); 6603 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 6604 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 6605 stats.tcp6InErrs[3]); 6606 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 6607 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 6608 stats.tnlCongDrops[3]); 6609 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 6610 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 6611 stats.tnlTxDrops[3]); 6612 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 6613 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 6614 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 6615 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 6616 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 6617 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 6618 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 6619 stats.ofldNoNeigh, stats.ofldCongDefer); 6620 6621 rc = sbuf_finish(sb); 6622 sbuf_delete(sb); 6623 6624 return (rc); 6625 } 6626 6627 struct field_desc { 6628 const char *name; 6629 u_int start; 6630 u_int width; 6631 }; 6632 6633 static void 6634 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) 6635 { 6636 char buf[32]; 6637 int line_size = 0; 6638 6639 while (f->name) { 6640 uint64_t mask = (1ULL << f->width) - 1; 6641 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, 6642 ((uintmax_t)v >> f->start) & mask); 6643 6644 if (line_size + len >= 79) { 6645 line_size = 8; 6646 sbuf_printf(sb, "\n "); 6647 } 6648 sbuf_printf(sb, "%s ", buf); 6649 line_size += len + 1; 6650 f++; 6651 } 6652 sbuf_printf(sb, "\n"); 6653 } 6654 6655 static struct field_desc tp_la0[] = { 6656 { "RcfOpCodeOut", 60, 4 }, 6657 { "State", 56, 4 }, 6658 { "WcfState", 52, 4 }, 6659 { "RcfOpcSrcOut", 50, 2 }, 6660 { "CRxError", 49, 1 }, 6661 { "ERxError", 48, 1 }, 6662 { "SanityFailed", 47, 1 }, 6663 { "SpuriousMsg", 46, 1 }, 6664 { "FlushInputMsg", 45, 1 }, 6665 { "FlushInputCpl", 44, 1 }, 6666 { "RssUpBit", 43, 1 }, 6667 { "RssFilterHit", 42, 1 }, 6668 { "Tid", 32, 10 }, 6669 { "InitTcb", 31, 1 }, 6670 { "LineNumber", 24, 7 }, 6671 { "Emsg", 23, 1 }, 6672 { "EdataOut", 22, 1 }, 6673 { "Cmsg", 21, 1 }, 6674 { "CdataOut", 20, 1 }, 6675 { "EreadPdu", 19, 1 }, 6676 { "CreadPdu", 18, 1 }, 6677 { "TunnelPkt", 17, 1 }, 6678 { "RcfPeerFin", 16, 1 }, 6679 { "RcfReasonOut", 12, 4 }, 6680 { "TxCchannel", 10, 2 }, 6681 { "RcfTxChannel", 8, 2 }, 6682 { "RxEchannel", 6, 2 }, 6683 { "RcfRxChannel", 5, 1 }, 6684 { "RcfDataOutSrdy", 4, 1 }, 6685 { "RxDvld", 3, 1 }, 6686 { "RxOoDvld", 2, 1 }, 6687 { "RxCongestion", 1, 1 }, 6688 { "TxCongestion", 0, 1 }, 6689 { NULL } 6690 }; 6691 6692 static struct field_desc tp_la1[] = { 6693 { "CplCmdIn", 56, 8 }, 6694 { "CplCmdOut", 48, 8 }, 6695 { "ESynOut", 47, 1 }, 6696 { "EAckOut", 46, 1 }, 6697 { "EFinOut", 45, 1 }, 6698 { "ERstOut", 44, 1 }, 6699 { "SynIn", 43, 1 }, 6700 { "AckIn", 42, 1 }, 6701 { "FinIn", 41, 1 }, 6702 { "RstIn", 40, 1 }, 6703 { "DataIn", 39, 1 }, 6704 { "DataInVld", 38, 1 }, 6705 { "PadIn", 37, 1 }, 6706 { "RxBufEmpty", 36, 1 }, 6707 { "RxDdp", 35, 1 }, 6708 { "RxFbCongestion", 34, 1 }, 6709 { "TxFbCongestion", 33, 1 }, 6710 { "TxPktSumSrdy", 32, 1 }, 6711 { "RcfUlpType", 28, 4 }, 6712 { "Eread", 27, 1 }, 6713 { "Ebypass", 26, 1 }, 6714 { "Esave", 25, 1 }, 6715 { "Static0", 24, 1 }, 6716 { "Cread", 23, 1 }, 6717 { "Cbypass", 22, 1 }, 6718 { "Csave", 21, 1 }, 6719 { "CPktOut", 20, 1 }, 6720 { "RxPagePoolFull", 18, 2 }, 6721 { "RxLpbkPkt", 17, 1 }, 6722 { "TxLpbkPkt", 16, 1 }, 6723 { "RxVfValid", 15, 1 }, 6724 { "SynLearned", 14, 1 }, 6725 { "SetDelEntry", 13, 1 }, 6726 { "SetInvEntry", 12, 1 }, 6727 { "CpcmdDvld", 11, 1 }, 6728 { "CpcmdSave", 10, 1 }, 6729 { "RxPstructsFull", 8, 2 }, 6730 { "EpcmdDvld", 7, 1 }, 6731 { "EpcmdFlush", 6, 1 }, 6732 { "EpcmdTrimPrefix", 5, 1 }, 6733 { "EpcmdTrimPostfix", 4, 1 }, 6734 { "ERssIp4Pkt", 3, 1 }, 6735 { "ERssIp6Pkt", 2, 1 }, 6736 { "ERssTcpUdpPkt", 1, 1 }, 6737 { "ERssFceFipPkt", 0, 1 }, 6738 { NULL } 6739 }; 6740 6741 static struct field_desc tp_la2[] = { 6742 { "CplCmdIn", 56, 8 }, 6743 { "MpsVfVld", 55, 1 }, 6744 { "MpsPf", 52, 3 }, 6745 { "MpsVf", 44, 8 }, 6746 { "SynIn", 43, 1 }, 6747 { "AckIn", 42, 1 }, 6748 { "FinIn", 41, 1 }, 6749 { "RstIn", 40, 1 }, 6750 { "DataIn", 39, 1 }, 6751 { "DataInVld", 38, 1 }, 6752 { "PadIn", 37, 1 }, 6753 { "RxBufEmpty", 36, 1 }, 6754 { "RxDdp", 35, 1 }, 6755 { "RxFbCongestion", 34, 1 }, 6756 { "TxFbCongestion", 33, 1 }, 6757 { "TxPktSumSrdy", 32, 1 }, 6758 { "RcfUlpType", 28, 4 }, 6759 { "Eread", 27, 1 }, 6760 { "Ebypass", 26, 1 }, 6761 { "Esave", 25, 1 }, 6762 { "Static0", 24, 1 }, 6763 { "Cread", 23, 1 }, 6764 { "Cbypass", 22, 1 }, 6765 { "Csave", 21, 1 }, 6766 { "CPktOut", 20, 1 }, 6767 { "RxPagePoolFull", 18, 2 }, 6768 { "RxLpbkPkt", 17, 1 }, 6769 { "TxLpbkPkt", 16, 1 }, 6770 { "RxVfValid", 15, 1 }, 6771 { "SynLearned", 14, 1 }, 6772 { "SetDelEntry", 13, 1 }, 6773 { "SetInvEntry", 12, 1 }, 6774 { "CpcmdDvld", 11, 1 }, 6775 { "CpcmdSave", 10, 1 }, 6776 { "RxPstructsFull", 8, 2 }, 6777 { "EpcmdDvld", 7, 1 }, 6778 { "EpcmdFlush", 6, 1 }, 6779 { "EpcmdTrimPrefix", 5, 1 }, 6780 { "EpcmdTrimPostfix", 4, 1 }, 6781 { "ERssIp4Pkt", 3, 1 }, 6782 { "ERssIp6Pkt", 2, 1 }, 6783 { "ERssTcpUdpPkt", 1, 1 }, 6784 { "ERssFceFipPkt", 0, 1 }, 6785 { NULL } 6786 }; 6787 6788 static void 6789 tp_la_show(struct sbuf *sb, uint64_t *p, int idx) 6790 { 6791 6792 field_desc_show(sb, *p, tp_la0); 6793 } 6794 6795 static void 6796 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) 6797 { 6798 6799 if (idx) 6800 sbuf_printf(sb, "\n"); 6801 field_desc_show(sb, p[0], tp_la0); 6802 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6803 field_desc_show(sb, p[1], tp_la0); 6804 } 6805 6806 static void 6807 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) 6808 { 6809 6810 if (idx) 6811 sbuf_printf(sb, "\n"); 6812 field_desc_show(sb, p[0], tp_la0); 6813 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6814 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); 6815 } 6816 6817 static int 6818 sysctl_tp_la(SYSCTL_HANDLER_ARGS) 6819 { 6820 struct adapter *sc = arg1; 6821 struct sbuf *sb; 6822 uint64_t *buf, *p; 6823 int rc; 6824 u_int i, inc; 6825 void (*show_func)(struct sbuf *, uint64_t *, int); 6826 6827 rc = sysctl_wire_old_buffer(req, 0); 6828 if (rc != 0) 6829 return (rc); 6830 6831 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6832 if (sb == NULL) 6833 return (ENOMEM); 6834 6835 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); 6836 6837 t4_tp_read_la(sc, buf, NULL); 6838 p = buf; 6839 6840 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { 6841 case 2: 6842 inc = 2; 6843 show_func = tp_la_show2; 6844 break; 6845 case 3: 6846 inc = 2; 6847 show_func = tp_la_show3; 6848 break; 6849 default: 6850 inc = 1; 6851 show_func = tp_la_show; 6852 } 6853 6854 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) 6855 (*show_func)(sb, p, i); 6856 6857 rc = sbuf_finish(sb); 6858 sbuf_delete(sb); 6859 free(buf, M_CXGBE); 6860 return (rc); 6861 } 6862 6863 static int 6864 sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 6865 { 6866 struct adapter *sc = arg1; 6867 struct sbuf *sb; 6868 int rc; 6869 u64 nrate[NCHAN], orate[NCHAN]; 6870 6871 rc = sysctl_wire_old_buffer(req, 0); 6872 if (rc != 0) 6873 return (rc); 6874 6875 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6876 if (sb == NULL) 6877 return (ENOMEM); 6878 6879 t4_get_chan_txrate(sc, nrate, orate); 6880 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6881 "channel 3\n"); 6882 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 6883 nrate[0], nrate[1], nrate[2], nrate[3]); 6884 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 6885 orate[0], orate[1], orate[2], orate[3]); 6886 6887 rc = sbuf_finish(sb); 6888 sbuf_delete(sb); 6889 6890 return (rc); 6891 } 6892 6893 static int 6894 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) 6895 { 6896 struct adapter *sc = arg1; 6897 struct sbuf *sb; 6898 uint32_t *buf, *p; 6899 int rc, i; 6900 6901 rc = sysctl_wire_old_buffer(req, 0); 6902 if (rc != 0) 6903 return (rc); 6904 6905 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6906 if (sb == NULL) 6907 return (ENOMEM); 6908 6909 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, 6910 M_ZERO | M_WAITOK); 6911 6912 t4_ulprx_read_la(sc, buf); 6913 p = buf; 6914 6915 sbuf_printf(sb, " Pcmd Type Message" 6916 " Data"); 6917 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { 6918 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", 6919 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); 6920 } 6921 6922 rc = sbuf_finish(sb); 6923 sbuf_delete(sb); 6924 free(buf, M_CXGBE); 6925 return (rc); 6926 } 6927 6928 static int 6929 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 6930 { 6931 struct adapter *sc = arg1; 6932 struct sbuf *sb; 6933 int rc, v; 6934 6935 rc = sysctl_wire_old_buffer(req, 0); 6936 if (rc != 0) 6937 return (rc); 6938 6939 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6940 if (sb == NULL) 6941 return (ENOMEM); 6942 6943 v = t4_read_reg(sc, A_SGE_STAT_CFG); 6944 if (G_STATSOURCE_T5(v) == 7) { 6945 if (G_STATMODE(v) == 0) { 6946 sbuf_printf(sb, "total %d, incomplete %d", 6947 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6948 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6949 } else if (G_STATMODE(v) == 1) { 6950 sbuf_printf(sb, "total %d, data overflow %d", 6951 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6952 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6953 } 6954 } 6955 rc = sbuf_finish(sb); 6956 sbuf_delete(sb); 6957 6958 return (rc); 6959 } 6960 #endif 6961 6962 static uint32_t 6963 fconf_to_mode(uint32_t fconf) 6964 { 6965 uint32_t mode; 6966 6967 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 6968 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 6969 6970 if (fconf & F_FRAGMENTATION) 6971 mode |= T4_FILTER_IP_FRAGMENT; 6972 6973 if (fconf & F_MPSHITTYPE) 6974 mode |= T4_FILTER_MPS_HIT_TYPE; 6975 6976 if (fconf & F_MACMATCH) 6977 mode |= T4_FILTER_MAC_IDX; 6978 6979 if (fconf & F_ETHERTYPE) 6980 mode |= T4_FILTER_ETH_TYPE; 6981 6982 if (fconf & F_PROTOCOL) 6983 mode |= T4_FILTER_IP_PROTO; 6984 6985 if (fconf & F_TOS) 6986 mode |= T4_FILTER_IP_TOS; 6987 6988 if (fconf & F_VLAN) 6989 mode |= T4_FILTER_VLAN; 6990 6991 if (fconf & F_VNIC_ID) 6992 mode |= T4_FILTER_VNIC; 6993 6994 if (fconf & F_PORT) 6995 mode |= T4_FILTER_PORT; 6996 6997 if (fconf & F_FCOE) 6998 mode |= T4_FILTER_FCoE; 6999 7000 return (mode); 7001 } 7002 7003 static uint32_t 7004 mode_to_fconf(uint32_t mode) 7005 { 7006 uint32_t fconf = 0; 7007 7008 if (mode & T4_FILTER_IP_FRAGMENT) 7009 fconf |= F_FRAGMENTATION; 7010 7011 if (mode & T4_FILTER_MPS_HIT_TYPE) 7012 fconf |= F_MPSHITTYPE; 7013 7014 if (mode & T4_FILTER_MAC_IDX) 7015 fconf |= F_MACMATCH; 7016 7017 if (mode & T4_FILTER_ETH_TYPE) 7018 fconf |= F_ETHERTYPE; 7019 7020 if (mode & T4_FILTER_IP_PROTO) 7021 fconf |= F_PROTOCOL; 7022 7023 if (mode & T4_FILTER_IP_TOS) 7024 fconf |= F_TOS; 7025 7026 if (mode & T4_FILTER_VLAN) 7027 fconf |= F_VLAN; 7028 7029 if (mode & T4_FILTER_VNIC) 7030 fconf |= F_VNIC_ID; 7031 7032 if (mode & T4_FILTER_PORT) 7033 fconf |= F_PORT; 7034 7035 if (mode & T4_FILTER_FCoE) 7036 fconf |= F_FCOE; 7037 7038 return (fconf); 7039 } 7040 7041 static uint32_t 7042 fspec_to_fconf(struct t4_filter_specification *fs) 7043 { 7044 uint32_t fconf = 0; 7045 7046 if (fs->val.frag || fs->mask.frag) 7047 fconf |= F_FRAGMENTATION; 7048 7049 if (fs->val.matchtype || fs->mask.matchtype) 7050 fconf |= F_MPSHITTYPE; 7051 7052 if (fs->val.macidx || fs->mask.macidx) 7053 fconf |= F_MACMATCH; 7054 7055 if (fs->val.ethtype || fs->mask.ethtype) 7056 fconf |= F_ETHERTYPE; 7057 7058 if (fs->val.proto || fs->mask.proto) 7059 fconf |= F_PROTOCOL; 7060 7061 if (fs->val.tos || fs->mask.tos) 7062 fconf |= F_TOS; 7063 7064 if (fs->val.vlan_vld || fs->mask.vlan_vld) 7065 fconf |= F_VLAN; 7066 7067 if (fs->val.vnic_vld || fs->mask.vnic_vld) 7068 fconf |= F_VNIC_ID; 7069 7070 if (fs->val.iport || fs->mask.iport) 7071 fconf |= F_PORT; 7072 7073 if (fs->val.fcoe || fs->mask.fcoe) 7074 fconf |= F_FCOE; 7075 7076 return (fconf); 7077 } 7078 7079 static int 7080 get_filter_mode(struct adapter *sc, uint32_t *mode) 7081 { 7082 int rc; 7083 uint32_t fconf; 7084 7085 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7086 "t4getfm"); 7087 if (rc) 7088 return (rc); 7089 7090 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 7091 A_TP_VLAN_PRI_MAP); 7092 7093 if (sc->params.tp.vlan_pri_map != fconf) { 7094 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 7095 device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map, 7096 fconf); 7097 } 7098 7099 *mode = fconf_to_mode(fconf); 7100 7101 end_synchronized_op(sc, LOCK_HELD); 7102 return (0); 7103 } 7104 7105 static int 7106 set_filter_mode(struct adapter *sc, uint32_t mode) 7107 { 7108 uint32_t fconf; 7109 int rc; 7110 7111 fconf = mode_to_fconf(mode); 7112 7113 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7114 "t4setfm"); 7115 if (rc) 7116 return (rc); 7117 7118 if (sc->tids.ftids_in_use > 0) { 7119 rc = EBUSY; 7120 goto done; 7121 } 7122 7123 #ifdef TCP_OFFLOAD 7124 if (uld_active(sc, ULD_TOM)) { 7125 rc = EBUSY; 7126 goto done; 7127 } 7128 #endif 7129 7130 rc = -t4_set_filter_mode(sc, fconf); 7131 done: 7132 end_synchronized_op(sc, LOCK_HELD); 7133 return (rc); 7134 } 7135 7136 static inline uint64_t 7137 get_filter_hits(struct adapter *sc, uint32_t fid) 7138 { 7139 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 7140 uint64_t hits; 7141 7142 memwin_info(sc, 0, &mw_base, NULL); 7143 off = position_memwin(sc, 0, 7144 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 7145 if (is_t4(sc)) { 7146 hits = t4_read_reg64(sc, mw_base + off + 16); 7147 hits = be64toh(hits); 7148 } else { 7149 hits = t4_read_reg(sc, mw_base + off + 24); 7150 hits = be32toh(hits); 7151 } 7152 7153 return (hits); 7154 } 7155 7156 static int 7157 get_filter(struct adapter *sc, struct t4_filter *t) 7158 { 7159 int i, rc, nfilters = sc->tids.nftids; 7160 struct filter_entry *f; 7161 7162 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7163 "t4getf"); 7164 if (rc) 7165 return (rc); 7166 7167 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 7168 t->idx >= nfilters) { 7169 t->idx = 0xffffffff; 7170 goto done; 7171 } 7172 7173 f = &sc->tids.ftid_tab[t->idx]; 7174 for (i = t->idx; i < nfilters; i++, f++) { 7175 if (f->valid) { 7176 t->idx = i; 7177 t->l2tidx = f->l2t ? f->l2t->idx : 0; 7178 t->smtidx = f->smtidx; 7179 if (f->fs.hitcnts) 7180 t->hits = get_filter_hits(sc, t->idx); 7181 else 7182 t->hits = UINT64_MAX; 7183 t->fs = f->fs; 7184 7185 goto done; 7186 } 7187 } 7188 7189 t->idx = 0xffffffff; 7190 done: 7191 end_synchronized_op(sc, LOCK_HELD); 7192 return (0); 7193 } 7194 7195 static int 7196 set_filter(struct adapter *sc, struct t4_filter *t) 7197 { 7198 unsigned int nfilters, nports; 7199 struct filter_entry *f; 7200 int i, rc; 7201 7202 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 7203 if (rc) 7204 return (rc); 7205 7206 nfilters = sc->tids.nftids; 7207 nports = sc->params.nports; 7208 7209 if (nfilters == 0) { 7210 rc = ENOTSUP; 7211 goto done; 7212 } 7213 7214 if (!(sc->flags & FULL_INIT_DONE)) { 7215 rc = EAGAIN; 7216 goto done; 7217 } 7218 7219 if (t->idx >= nfilters) { 7220 rc = EINVAL; 7221 goto done; 7222 } 7223 7224 /* Validate against the global filter mode */ 7225 if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) != 7226 sc->params.tp.vlan_pri_map) { 7227 rc = E2BIG; 7228 goto done; 7229 } 7230 7231 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 7232 rc = EINVAL; 7233 goto done; 7234 } 7235 7236 if (t->fs.val.iport >= nports) { 7237 rc = EINVAL; 7238 goto done; 7239 } 7240 7241 /* Can't specify an iq if not steering to it */ 7242 if (!t->fs.dirsteer && t->fs.iq) { 7243 rc = EINVAL; 7244 goto done; 7245 } 7246 7247 /* IPv6 filter idx must be 4 aligned */ 7248 if (t->fs.type == 1 && 7249 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 7250 rc = EINVAL; 7251 goto done; 7252 } 7253 7254 if (sc->tids.ftid_tab == NULL) { 7255 KASSERT(sc->tids.ftids_in_use == 0, 7256 ("%s: no memory allocated but filters_in_use > 0", 7257 __func__)); 7258 7259 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 7260 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 7261 if (sc->tids.ftid_tab == NULL) { 7262 rc = ENOMEM; 7263 goto done; 7264 } 7265 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 7266 } 7267 7268 for (i = 0; i < 4; i++) { 7269 f = &sc->tids.ftid_tab[t->idx + i]; 7270 7271 if (f->pending || f->valid) { 7272 rc = EBUSY; 7273 goto done; 7274 } 7275 if (f->locked) { 7276 rc = EPERM; 7277 goto done; 7278 } 7279 7280 if (t->fs.type == 0) 7281 break; 7282 } 7283 7284 f = &sc->tids.ftid_tab[t->idx]; 7285 f->fs = t->fs; 7286 7287 rc = set_filter_wr(sc, t->idx); 7288 done: 7289 end_synchronized_op(sc, 0); 7290 7291 if (rc == 0) { 7292 mtx_lock(&sc->tids.ftid_lock); 7293 for (;;) { 7294 if (f->pending == 0) { 7295 rc = f->valid ? 0 : EIO; 7296 break; 7297 } 7298 7299 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7300 PCATCH, "t4setfw", 0)) { 7301 rc = EINPROGRESS; 7302 break; 7303 } 7304 } 7305 mtx_unlock(&sc->tids.ftid_lock); 7306 } 7307 return (rc); 7308 } 7309 7310 static int 7311 del_filter(struct adapter *sc, struct t4_filter *t) 7312 { 7313 unsigned int nfilters; 7314 struct filter_entry *f; 7315 int rc; 7316 7317 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 7318 if (rc) 7319 return (rc); 7320 7321 nfilters = sc->tids.nftids; 7322 7323 if (nfilters == 0) { 7324 rc = ENOTSUP; 7325 goto done; 7326 } 7327 7328 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 7329 t->idx >= nfilters) { 7330 rc = EINVAL; 7331 goto done; 7332 } 7333 7334 if (!(sc->flags & FULL_INIT_DONE)) { 7335 rc = EAGAIN; 7336 goto done; 7337 } 7338 7339 f = &sc->tids.ftid_tab[t->idx]; 7340 7341 if (f->pending) { 7342 rc = EBUSY; 7343 goto done; 7344 } 7345 if (f->locked) { 7346 rc = EPERM; 7347 goto done; 7348 } 7349 7350 if (f->valid) { 7351 t->fs = f->fs; /* extra info for the caller */ 7352 rc = del_filter_wr(sc, t->idx); 7353 } 7354 7355 done: 7356 end_synchronized_op(sc, 0); 7357 7358 if (rc == 0) { 7359 mtx_lock(&sc->tids.ftid_lock); 7360 for (;;) { 7361 if (f->pending == 0) { 7362 rc = f->valid ? EIO : 0; 7363 break; 7364 } 7365 7366 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7367 PCATCH, "t4delfw", 0)) { 7368 rc = EINPROGRESS; 7369 break; 7370 } 7371 } 7372 mtx_unlock(&sc->tids.ftid_lock); 7373 } 7374 7375 return (rc); 7376 } 7377 7378 static void 7379 clear_filter(struct filter_entry *f) 7380 { 7381 if (f->l2t) 7382 t4_l2t_release(f->l2t); 7383 7384 bzero(f, sizeof (*f)); 7385 } 7386 7387 static int 7388 set_filter_wr(struct adapter *sc, int fidx) 7389 { 7390 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7391 struct fw_filter_wr *fwr; 7392 unsigned int ftid; 7393 struct wrq_cookie cookie; 7394 7395 ASSERT_SYNCHRONIZED_OP(sc); 7396 7397 if (f->fs.newdmac || f->fs.newvlan) { 7398 /* This filter needs an L2T entry; allocate one. */ 7399 f->l2t = t4_l2t_alloc_switching(sc->l2t); 7400 if (f->l2t == NULL) 7401 return (EAGAIN); 7402 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 7403 f->fs.dmac)) { 7404 t4_l2t_release(f->l2t); 7405 f->l2t = NULL; 7406 return (ENOMEM); 7407 } 7408 } 7409 7410 ftid = sc->tids.ftid_base + fidx; 7411 7412 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 7413 if (fwr == NULL) 7414 return (ENOMEM); 7415 bzero(fwr, sizeof(*fwr)); 7416 7417 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 7418 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 7419 fwr->tid_to_iq = 7420 htobe32(V_FW_FILTER_WR_TID(ftid) | 7421 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 7422 V_FW_FILTER_WR_NOREPLY(0) | 7423 V_FW_FILTER_WR_IQ(f->fs.iq)); 7424 fwr->del_filter_to_l2tix = 7425 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 7426 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 7427 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 7428 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 7429 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 7430 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 7431 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 7432 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 7433 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 7434 f->fs.newvlan == VLAN_REWRITE) | 7435 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 7436 f->fs.newvlan == VLAN_REWRITE) | 7437 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 7438 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 7439 V_FW_FILTER_WR_PRIO(f->fs.prio) | 7440 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 7441 fwr->ethtype = htobe16(f->fs.val.ethtype); 7442 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 7443 fwr->frag_to_ovlan_vldm = 7444 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 7445 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 7446 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 7447 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 7448 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 7449 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 7450 fwr->smac_sel = 0; 7451 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 7452 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 7453 fwr->maci_to_matchtypem = 7454 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 7455 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 7456 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 7457 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 7458 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 7459 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 7460 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 7461 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 7462 fwr->ptcl = f->fs.val.proto; 7463 fwr->ptclm = f->fs.mask.proto; 7464 fwr->ttyp = f->fs.val.tos; 7465 fwr->ttypm = f->fs.mask.tos; 7466 fwr->ivlan = htobe16(f->fs.val.vlan); 7467 fwr->ivlanm = htobe16(f->fs.mask.vlan); 7468 fwr->ovlan = htobe16(f->fs.val.vnic); 7469 fwr->ovlanm = htobe16(f->fs.mask.vnic); 7470 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 7471 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 7472 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 7473 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 7474 fwr->lp = htobe16(f->fs.val.dport); 7475 fwr->lpm = htobe16(f->fs.mask.dport); 7476 fwr->fp = htobe16(f->fs.val.sport); 7477 fwr->fpm = htobe16(f->fs.mask.sport); 7478 if (f->fs.newsmac) 7479 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 7480 7481 f->pending = 1; 7482 sc->tids.ftids_in_use++; 7483 7484 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 7485 return (0); 7486 } 7487 7488 static int 7489 del_filter_wr(struct adapter *sc, int fidx) 7490 { 7491 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7492 struct fw_filter_wr *fwr; 7493 unsigned int ftid; 7494 struct wrq_cookie cookie; 7495 7496 ftid = sc->tids.ftid_base + fidx; 7497 7498 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 7499 if (fwr == NULL) 7500 return (ENOMEM); 7501 bzero(fwr, sizeof (*fwr)); 7502 7503 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 7504 7505 f->pending = 1; 7506 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 7507 return (0); 7508 } 7509 7510 int 7511 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 7512 { 7513 struct adapter *sc = iq->adapter; 7514 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 7515 unsigned int idx = GET_TID(rpl); 7516 unsigned int rc; 7517 struct filter_entry *f; 7518 7519 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 7520 rss->opcode)); 7521 7522 if (is_ftid(sc, idx)) { 7523 7524 idx -= sc->tids.ftid_base; 7525 f = &sc->tids.ftid_tab[idx]; 7526 rc = G_COOKIE(rpl->cookie); 7527 7528 mtx_lock(&sc->tids.ftid_lock); 7529 if (rc == FW_FILTER_WR_FLT_ADDED) { 7530 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 7531 __func__, idx)); 7532 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 7533 f->pending = 0; /* asynchronous setup completed */ 7534 f->valid = 1; 7535 } else { 7536 if (rc != FW_FILTER_WR_FLT_DELETED) { 7537 /* Add or delete failed, display an error */ 7538 log(LOG_ERR, 7539 "filter %u setup failed with error %u\n", 7540 idx, rc); 7541 } 7542 7543 clear_filter(f); 7544 sc->tids.ftids_in_use--; 7545 } 7546 wakeup(&sc->tids.ftid_tab); 7547 mtx_unlock(&sc->tids.ftid_lock); 7548 } 7549 7550 return (0); 7551 } 7552 7553 static int 7554 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 7555 { 7556 int rc; 7557 7558 if (cntxt->cid > M_CTXTQID) 7559 return (EINVAL); 7560 7561 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 7562 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 7563 return (EINVAL); 7564 7565 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 7566 if (rc) 7567 return (rc); 7568 7569 if (sc->flags & FW_OK) { 7570 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 7571 &cntxt->data[0]); 7572 if (rc == 0) 7573 goto done; 7574 } 7575 7576 /* 7577 * Read via firmware failed or wasn't even attempted. Read directly via 7578 * the backdoor. 7579 */ 7580 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 7581 done: 7582 end_synchronized_op(sc, 0); 7583 return (rc); 7584 } 7585 7586 static int 7587 load_fw(struct adapter *sc, struct t4_data *fw) 7588 { 7589 int rc; 7590 uint8_t *fw_data; 7591 7592 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 7593 if (rc) 7594 return (rc); 7595 7596 if (sc->flags & FULL_INIT_DONE) { 7597 rc = EBUSY; 7598 goto done; 7599 } 7600 7601 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 7602 if (fw_data == NULL) { 7603 rc = ENOMEM; 7604 goto done; 7605 } 7606 7607 rc = copyin(fw->data, fw_data, fw->len); 7608 if (rc == 0) 7609 rc = -t4_load_fw(sc, fw_data, fw->len); 7610 7611 free(fw_data, M_CXGBE); 7612 done: 7613 end_synchronized_op(sc, 0); 7614 return (rc); 7615 } 7616 7617 static int 7618 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 7619 { 7620 uint32_t addr, off, remaining, i, n; 7621 uint32_t *buf, *b; 7622 uint32_t mw_base, mw_aperture; 7623 int rc; 7624 uint8_t *dst; 7625 7626 rc = validate_mem_range(sc, mr->addr, mr->len); 7627 if (rc != 0) 7628 return (rc); 7629 7630 memwin_info(sc, win, &mw_base, &mw_aperture); 7631 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 7632 addr = mr->addr; 7633 remaining = mr->len; 7634 dst = (void *)mr->data; 7635 7636 while (remaining) { 7637 off = position_memwin(sc, win, addr); 7638 7639 /* number of bytes that we'll copy in the inner loop */ 7640 n = min(remaining, mw_aperture - off); 7641 for (i = 0; i < n; i += 4) 7642 *b++ = t4_read_reg(sc, mw_base + off + i); 7643 7644 rc = copyout(buf, dst, n); 7645 if (rc != 0) 7646 break; 7647 7648 b = buf; 7649 dst += n; 7650 remaining -= n; 7651 addr += n; 7652 } 7653 7654 free(buf, M_CXGBE); 7655 return (rc); 7656 } 7657 7658 static int 7659 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 7660 { 7661 int rc; 7662 7663 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 7664 return (EINVAL); 7665 7666 if (i2cd->len > sizeof(i2cd->data)) 7667 return (EFBIG); 7668 7669 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 7670 if (rc) 7671 return (rc); 7672 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 7673 i2cd->offset, i2cd->len, &i2cd->data[0]); 7674 end_synchronized_op(sc, 0); 7675 7676 return (rc); 7677 } 7678 7679 static int 7680 in_range(int val, int lo, int hi) 7681 { 7682 7683 return (val < 0 || (val <= hi && val >= lo)); 7684 } 7685 7686 static int 7687 set_sched_class(struct adapter *sc, struct t4_sched_params *p) 7688 { 7689 int fw_subcmd, fw_type, rc; 7690 7691 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc"); 7692 if (rc) 7693 return (rc); 7694 7695 if (!(sc->flags & FULL_INIT_DONE)) { 7696 rc = EAGAIN; 7697 goto done; 7698 } 7699 7700 /* 7701 * Translate the cxgbetool parameters into T4 firmware parameters. (The 7702 * sub-command and type are in common locations.) 7703 */ 7704 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG) 7705 fw_subcmd = FW_SCHED_SC_CONFIG; 7706 else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS) 7707 fw_subcmd = FW_SCHED_SC_PARAMS; 7708 else { 7709 rc = EINVAL; 7710 goto done; 7711 } 7712 if (p->type == SCHED_CLASS_TYPE_PACKET) 7713 fw_type = FW_SCHED_TYPE_PKTSCHED; 7714 else { 7715 rc = EINVAL; 7716 goto done; 7717 } 7718 7719 if (fw_subcmd == FW_SCHED_SC_CONFIG) { 7720 /* Vet our parameters ..*/ 7721 if (p->u.config.minmax < 0) { 7722 rc = EINVAL; 7723 goto done; 7724 } 7725 7726 /* And pass the request to the firmware ...*/ 7727 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1); 7728 goto done; 7729 } 7730 7731 if (fw_subcmd == FW_SCHED_SC_PARAMS) { 7732 int fw_level; 7733 int fw_mode; 7734 int fw_rateunit; 7735 int fw_ratemode; 7736 7737 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL) 7738 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL; 7739 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) 7740 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR; 7741 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) 7742 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL; 7743 else { 7744 rc = EINVAL; 7745 goto done; 7746 } 7747 7748 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS) 7749 fw_mode = FW_SCHED_PARAMS_MODE_CLASS; 7750 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW) 7751 fw_mode = FW_SCHED_PARAMS_MODE_FLOW; 7752 else { 7753 rc = EINVAL; 7754 goto done; 7755 } 7756 7757 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS) 7758 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 7759 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS) 7760 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE; 7761 else { 7762 rc = EINVAL; 7763 goto done; 7764 } 7765 7766 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL) 7767 fw_ratemode = FW_SCHED_PARAMS_RATE_REL; 7768 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS) 7769 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS; 7770 else { 7771 rc = EINVAL; 7772 goto done; 7773 } 7774 7775 /* Vet our parameters ... */ 7776 if (!in_range(p->u.params.channel, 0, 3) || 7777 !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) || 7778 !in_range(p->u.params.minrate, 0, 10000000) || 7779 !in_range(p->u.params.maxrate, 0, 10000000) || 7780 !in_range(p->u.params.weight, 0, 100)) { 7781 rc = ERANGE; 7782 goto done; 7783 } 7784 7785 /* 7786 * Translate any unset parameters into the firmware's 7787 * nomenclature and/or fail the call if the parameters 7788 * are required ... 7789 */ 7790 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 || 7791 p->u.params.channel < 0 || p->u.params.cl < 0) { 7792 rc = EINVAL; 7793 goto done; 7794 } 7795 if (p->u.params.minrate < 0) 7796 p->u.params.minrate = 0; 7797 if (p->u.params.maxrate < 0) { 7798 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7799 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7800 rc = EINVAL; 7801 goto done; 7802 } else 7803 p->u.params.maxrate = 0; 7804 } 7805 if (p->u.params.weight < 0) { 7806 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) { 7807 rc = EINVAL; 7808 goto done; 7809 } else 7810 p->u.params.weight = 0; 7811 } 7812 if (p->u.params.pktsize < 0) { 7813 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7814 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7815 rc = EINVAL; 7816 goto done; 7817 } else 7818 p->u.params.pktsize = 0; 7819 } 7820 7821 /* See what the firmware thinks of the request ... */ 7822 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode, 7823 fw_rateunit, fw_ratemode, p->u.params.channel, 7824 p->u.params.cl, p->u.params.minrate, p->u.params.maxrate, 7825 p->u.params.weight, p->u.params.pktsize, 1); 7826 goto done; 7827 } 7828 7829 rc = EINVAL; 7830 done: 7831 end_synchronized_op(sc, 0); 7832 return (rc); 7833 } 7834 7835 static int 7836 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p) 7837 { 7838 struct port_info *pi = NULL; 7839 struct sge_txq *txq; 7840 uint32_t fw_mnem, fw_queue, fw_class; 7841 int i, rc; 7842 7843 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq"); 7844 if (rc) 7845 return (rc); 7846 7847 if (!(sc->flags & FULL_INIT_DONE)) { 7848 rc = EAGAIN; 7849 goto done; 7850 } 7851 7852 if (p->port >= sc->params.nports) { 7853 rc = EINVAL; 7854 goto done; 7855 } 7856 7857 pi = sc->port[p->port]; 7858 if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) { 7859 rc = EINVAL; 7860 goto done; 7861 } 7862 7863 /* 7864 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX 7865 * Scheduling Class in this case). 7866 */ 7867 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 7868 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH)); 7869 fw_class = p->cl < 0 ? 0xffffffff : p->cl; 7870 7871 /* 7872 * If op.queue is non-negative, then we're only changing the scheduling 7873 * on a single specified TX queue. 7874 */ 7875 if (p->queue >= 0) { 7876 txq = &sc->sge.txq[pi->first_txq + p->queue]; 7877 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7878 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7879 &fw_class); 7880 goto done; 7881 } 7882 7883 /* 7884 * Change the scheduling on all the TX queues for the 7885 * interface. 7886 */ 7887 for_each_txq(pi, i, txq) { 7888 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7889 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7890 &fw_class); 7891 if (rc) 7892 goto done; 7893 } 7894 7895 rc = 0; 7896 done: 7897 end_synchronized_op(sc, 0); 7898 return (rc); 7899 } 7900 7901 int 7902 t4_os_find_pci_capability(struct adapter *sc, int cap) 7903 { 7904 int i; 7905 7906 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 7907 } 7908 7909 int 7910 t4_os_pci_save_state(struct adapter *sc) 7911 { 7912 device_t dev; 7913 struct pci_devinfo *dinfo; 7914 7915 dev = sc->dev; 7916 dinfo = device_get_ivars(dev); 7917 7918 pci_cfg_save(dev, dinfo, 0); 7919 return (0); 7920 } 7921 7922 int 7923 t4_os_pci_restore_state(struct adapter *sc) 7924 { 7925 device_t dev; 7926 struct pci_devinfo *dinfo; 7927 7928 dev = sc->dev; 7929 dinfo = device_get_ivars(dev); 7930 7931 pci_cfg_restore(dev, dinfo); 7932 return (0); 7933 } 7934 7935 void 7936 t4_os_portmod_changed(const struct adapter *sc, int idx) 7937 { 7938 struct port_info *pi = sc->port[idx]; 7939 static const char *mod_str[] = { 7940 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 7941 }; 7942 7943 build_medialist(pi, &pi->media); 7944 #ifdef DEV_NETMAP 7945 build_medialist(pi, &pi->nm_media); 7946 #endif 7947 7948 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 7949 if_printf(pi->ifp, "transceiver unplugged.\n"); 7950 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 7951 if_printf(pi->ifp, "unknown transceiver inserted.\n"); 7952 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 7953 if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 7954 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 7955 if_printf(pi->ifp, "%s transceiver inserted.\n", 7956 mod_str[pi->mod_type]); 7957 } else { 7958 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 7959 pi->mod_type); 7960 } 7961 } 7962 7963 void 7964 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason) 7965 { 7966 struct port_info *pi = sc->port[idx]; 7967 struct ifnet *ifp = pi->ifp; 7968 7969 if (link_stat) { 7970 pi->linkdnrc = -1; 7971 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 7972 if_link_state_change(ifp, LINK_STATE_UP); 7973 } else { 7974 if (reason >= 0) 7975 pi->linkdnrc = reason; 7976 if_link_state_change(ifp, LINK_STATE_DOWN); 7977 } 7978 } 7979 7980 void 7981 t4_iterate(void (*func)(struct adapter *, void *), void *arg) 7982 { 7983 struct adapter *sc; 7984 7985 sx_slock(&t4_list_lock); 7986 SLIST_FOREACH(sc, &t4_list, link) { 7987 /* 7988 * func should not make any assumptions about what state sc is 7989 * in - the only guarantee is that sc->sc_lock is a valid lock. 7990 */ 7991 func(sc, arg); 7992 } 7993 sx_sunlock(&t4_list_lock); 7994 } 7995 7996 static int 7997 t4_open(struct cdev *dev, int flags, int type, struct thread *td) 7998 { 7999 return (0); 8000 } 8001 8002 static int 8003 t4_close(struct cdev *dev, int flags, int type, struct thread *td) 8004 { 8005 return (0); 8006 } 8007 8008 static int 8009 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 8010 struct thread *td) 8011 { 8012 int rc; 8013 struct adapter *sc = dev->si_drv1; 8014 8015 rc = priv_check(td, PRIV_DRIVER); 8016 if (rc != 0) 8017 return (rc); 8018 8019 switch (cmd) { 8020 case CHELSIO_T4_GETREG: { 8021 struct t4_reg *edata = (struct t4_reg *)data; 8022 8023 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8024 return (EFAULT); 8025 8026 if (edata->size == 4) 8027 edata->val = t4_read_reg(sc, edata->addr); 8028 else if (edata->size == 8) 8029 edata->val = t4_read_reg64(sc, edata->addr); 8030 else 8031 return (EINVAL); 8032 8033 break; 8034 } 8035 case CHELSIO_T4_SETREG: { 8036 struct t4_reg *edata = (struct t4_reg *)data; 8037 8038 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8039 return (EFAULT); 8040 8041 if (edata->size == 4) { 8042 if (edata->val & 0xffffffff00000000) 8043 return (EINVAL); 8044 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 8045 } else if (edata->size == 8) 8046 t4_write_reg64(sc, edata->addr, edata->val); 8047 else 8048 return (EINVAL); 8049 break; 8050 } 8051 case CHELSIO_T4_REGDUMP: { 8052 struct t4_regdump *regs = (struct t4_regdump *)data; 8053 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 8054 uint8_t *buf; 8055 8056 if (regs->len < reglen) { 8057 regs->len = reglen; /* hint to the caller */ 8058 return (ENOBUFS); 8059 } 8060 8061 regs->len = reglen; 8062 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 8063 t4_get_regs(sc, regs, buf); 8064 rc = copyout(buf, regs->data, reglen); 8065 free(buf, M_CXGBE); 8066 break; 8067 } 8068 case CHELSIO_T4_GET_FILTER_MODE: 8069 rc = get_filter_mode(sc, (uint32_t *)data); 8070 break; 8071 case CHELSIO_T4_SET_FILTER_MODE: 8072 rc = set_filter_mode(sc, *(uint32_t *)data); 8073 break; 8074 case CHELSIO_T4_GET_FILTER: 8075 rc = get_filter(sc, (struct t4_filter *)data); 8076 break; 8077 case CHELSIO_T4_SET_FILTER: 8078 rc = set_filter(sc, (struct t4_filter *)data); 8079 break; 8080 case CHELSIO_T4_DEL_FILTER: 8081 rc = del_filter(sc, (struct t4_filter *)data); 8082 break; 8083 case CHELSIO_T4_GET_SGE_CONTEXT: 8084 rc = get_sge_context(sc, (struct t4_sge_context *)data); 8085 break; 8086 case CHELSIO_T4_LOAD_FW: 8087 rc = load_fw(sc, (struct t4_data *)data); 8088 break; 8089 case CHELSIO_T4_GET_MEM: 8090 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 8091 break; 8092 case CHELSIO_T4_GET_I2C: 8093 rc = read_i2c(sc, (struct t4_i2c_data *)data); 8094 break; 8095 case CHELSIO_T4_CLEAR_STATS: { 8096 int i; 8097 u_int port_id = *(uint32_t *)data; 8098 struct port_info *pi; 8099 8100 if (port_id >= sc->params.nports) 8101 return (EINVAL); 8102 pi = sc->port[port_id]; 8103 8104 /* MAC stats */ 8105 t4_clr_port_stats(sc, pi->tx_chan); 8106 pi->tx_parse_error = 0; 8107 8108 if (pi->flags & PORT_INIT_DONE) { 8109 struct sge_rxq *rxq; 8110 struct sge_txq *txq; 8111 struct sge_wrq *wrq; 8112 8113 for_each_rxq(pi, i, rxq) { 8114 #if defined(INET) || defined(INET6) 8115 rxq->lro.lro_queued = 0; 8116 rxq->lro.lro_flushed = 0; 8117 #endif 8118 rxq->rxcsum = 0; 8119 rxq->vlan_extraction = 0; 8120 } 8121 8122 for_each_txq(pi, i, txq) { 8123 txq->txcsum = 0; 8124 txq->tso_wrs = 0; 8125 txq->vlan_insertion = 0; 8126 txq->imm_wrs = 0; 8127 txq->sgl_wrs = 0; 8128 txq->txpkt_wrs = 0; 8129 txq->txpkts0_wrs = 0; 8130 txq->txpkts1_wrs = 0; 8131 txq->txpkts0_pkts = 0; 8132 txq->txpkts1_pkts = 0; 8133 mp_ring_reset_stats(txq->r); 8134 } 8135 8136 #ifdef TCP_OFFLOAD 8137 /* nothing to clear for each ofld_rxq */ 8138 8139 for_each_ofld_txq(pi, i, wrq) { 8140 wrq->tx_wrs_direct = 0; 8141 wrq->tx_wrs_copied = 0; 8142 } 8143 #endif 8144 wrq = &sc->sge.ctrlq[pi->port_id]; 8145 wrq->tx_wrs_direct = 0; 8146 wrq->tx_wrs_copied = 0; 8147 } 8148 break; 8149 } 8150 case CHELSIO_T4_SCHED_CLASS: 8151 rc = set_sched_class(sc, (struct t4_sched_params *)data); 8152 break; 8153 case CHELSIO_T4_SCHED_QUEUE: 8154 rc = set_sched_queue(sc, (struct t4_sched_queue *)data); 8155 break; 8156 case CHELSIO_T4_GET_TRACER: 8157 rc = t4_get_tracer(sc, (struct t4_tracer *)data); 8158 break; 8159 case CHELSIO_T4_SET_TRACER: 8160 rc = t4_set_tracer(sc, (struct t4_tracer *)data); 8161 break; 8162 default: 8163 rc = EINVAL; 8164 } 8165 8166 return (rc); 8167 } 8168 8169 #ifdef TCP_OFFLOAD 8170 void 8171 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask, 8172 const unsigned int *pgsz_order) 8173 { 8174 struct port_info *pi = ifp->if_softc; 8175 struct adapter *sc = pi->adapter; 8176 8177 t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask); 8178 t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) | 8179 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) | 8180 V_HPZ3(pgsz_order[3])); 8181 } 8182 8183 static int 8184 toe_capability(struct port_info *pi, int enable) 8185 { 8186 int rc; 8187 struct adapter *sc = pi->adapter; 8188 8189 ASSERT_SYNCHRONIZED_OP(sc); 8190 8191 if (!is_offload(sc)) 8192 return (ENODEV); 8193 8194 if (enable) { 8195 /* 8196 * We need the port's queues around so that we're able to send 8197 * and receive CPLs to/from the TOE even if the ifnet for this 8198 * port has never been UP'd administratively. 8199 */ 8200 if (!(pi->flags & PORT_INIT_DONE)) { 8201 rc = cxgbe_init_synchronized(pi); 8202 if (rc) 8203 return (rc); 8204 } 8205 8206 if (isset(&sc->offload_map, pi->port_id)) 8207 return (0); 8208 8209 if (!uld_active(sc, ULD_TOM)) { 8210 rc = t4_activate_uld(sc, ULD_TOM); 8211 if (rc == EAGAIN) { 8212 log(LOG_WARNING, 8213 "You must kldload t4_tom.ko before trying " 8214 "to enable TOE on a cxgbe interface.\n"); 8215 } 8216 if (rc != 0) 8217 return (rc); 8218 KASSERT(sc->tom_softc != NULL, 8219 ("%s: TOM activated but softc NULL", __func__)); 8220 KASSERT(uld_active(sc, ULD_TOM), 8221 ("%s: TOM activated but flag not set", __func__)); 8222 } 8223 8224 /* Activate iWARP and iSCSI too, if the modules are loaded. */ 8225 if (!uld_active(sc, ULD_IWARP)) 8226 (void) t4_activate_uld(sc, ULD_IWARP); 8227 if (!uld_active(sc, ULD_ISCSI)) 8228 (void) t4_activate_uld(sc, ULD_ISCSI); 8229 8230 setbit(&sc->offload_map, pi->port_id); 8231 } else { 8232 if (!isset(&sc->offload_map, pi->port_id)) 8233 return (0); 8234 8235 KASSERT(uld_active(sc, ULD_TOM), 8236 ("%s: TOM never initialized?", __func__)); 8237 clrbit(&sc->offload_map, pi->port_id); 8238 } 8239 8240 return (0); 8241 } 8242 8243 /* 8244 * Add an upper layer driver to the global list. 8245 */ 8246 int 8247 t4_register_uld(struct uld_info *ui) 8248 { 8249 int rc = 0; 8250 struct uld_info *u; 8251 8252 sx_xlock(&t4_uld_list_lock); 8253 SLIST_FOREACH(u, &t4_uld_list, link) { 8254 if (u->uld_id == ui->uld_id) { 8255 rc = EEXIST; 8256 goto done; 8257 } 8258 } 8259 8260 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 8261 ui->refcount = 0; 8262 done: 8263 sx_xunlock(&t4_uld_list_lock); 8264 return (rc); 8265 } 8266 8267 int 8268 t4_unregister_uld(struct uld_info *ui) 8269 { 8270 int rc = EINVAL; 8271 struct uld_info *u; 8272 8273 sx_xlock(&t4_uld_list_lock); 8274 8275 SLIST_FOREACH(u, &t4_uld_list, link) { 8276 if (u == ui) { 8277 if (ui->refcount > 0) { 8278 rc = EBUSY; 8279 goto done; 8280 } 8281 8282 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 8283 rc = 0; 8284 goto done; 8285 } 8286 } 8287 done: 8288 sx_xunlock(&t4_uld_list_lock); 8289 return (rc); 8290 } 8291 8292 int 8293 t4_activate_uld(struct adapter *sc, int id) 8294 { 8295 int rc; 8296 struct uld_info *ui; 8297 8298 ASSERT_SYNCHRONIZED_OP(sc); 8299 8300 if (id < 0 || id > ULD_MAX) 8301 return (EINVAL); 8302 rc = EAGAIN; /* kldoad the module with this ULD and try again. */ 8303 8304 sx_slock(&t4_uld_list_lock); 8305 8306 SLIST_FOREACH(ui, &t4_uld_list, link) { 8307 if (ui->uld_id == id) { 8308 if (!(sc->flags & FULL_INIT_DONE)) { 8309 rc = adapter_full_init(sc); 8310 if (rc != 0) 8311 break; 8312 } 8313 8314 rc = ui->activate(sc); 8315 if (rc == 0) { 8316 setbit(&sc->active_ulds, id); 8317 ui->refcount++; 8318 } 8319 break; 8320 } 8321 } 8322 8323 sx_sunlock(&t4_uld_list_lock); 8324 8325 return (rc); 8326 } 8327 8328 int 8329 t4_deactivate_uld(struct adapter *sc, int id) 8330 { 8331 int rc; 8332 struct uld_info *ui; 8333 8334 ASSERT_SYNCHRONIZED_OP(sc); 8335 8336 if (id < 0 || id > ULD_MAX) 8337 return (EINVAL); 8338 rc = ENXIO; 8339 8340 sx_slock(&t4_uld_list_lock); 8341 8342 SLIST_FOREACH(ui, &t4_uld_list, link) { 8343 if (ui->uld_id == id) { 8344 rc = ui->deactivate(sc); 8345 if (rc == 0) { 8346 clrbit(&sc->active_ulds, id); 8347 ui->refcount--; 8348 } 8349 break; 8350 } 8351 } 8352 8353 sx_sunlock(&t4_uld_list_lock); 8354 8355 return (rc); 8356 } 8357 8358 int 8359 uld_active(struct adapter *sc, int uld_id) 8360 { 8361 8362 MPASS(uld_id >= 0 && uld_id <= ULD_MAX); 8363 8364 return (isset(&sc->active_ulds, uld_id)); 8365 } 8366 #endif 8367 8368 /* 8369 * Come up with reasonable defaults for some of the tunables, provided they're 8370 * not set by the user (in which case we'll use the values as is). 8371 */ 8372 static void 8373 tweak_tunables(void) 8374 { 8375 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 8376 8377 if (t4_ntxq10g < 1) 8378 t4_ntxq10g = min(nc, NTXQ_10G); 8379 8380 if (t4_ntxq1g < 1) 8381 t4_ntxq1g = min(nc, NTXQ_1G); 8382 8383 if (t4_nrxq10g < 1) 8384 t4_nrxq10g = min(nc, NRXQ_10G); 8385 8386 if (t4_nrxq1g < 1) 8387 t4_nrxq1g = min(nc, NRXQ_1G); 8388 8389 #ifdef TCP_OFFLOAD 8390 if (t4_nofldtxq10g < 1) 8391 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 8392 8393 if (t4_nofldtxq1g < 1) 8394 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 8395 8396 if (t4_nofldrxq10g < 1) 8397 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 8398 8399 if (t4_nofldrxq1g < 1) 8400 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 8401 8402 if (t4_toecaps_allowed == -1) 8403 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 8404 #else 8405 if (t4_toecaps_allowed == -1) 8406 t4_toecaps_allowed = 0; 8407 #endif 8408 8409 #ifdef DEV_NETMAP 8410 if (t4_nnmtxq10g < 1) 8411 t4_nnmtxq10g = min(nc, NNMTXQ_10G); 8412 8413 if (t4_nnmtxq1g < 1) 8414 t4_nnmtxq1g = min(nc, NNMTXQ_1G); 8415 8416 if (t4_nnmrxq10g < 1) 8417 t4_nnmrxq10g = min(nc, NNMRXQ_10G); 8418 8419 if (t4_nnmrxq1g < 1) 8420 t4_nnmrxq1g = min(nc, NNMRXQ_1G); 8421 #endif 8422 8423 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 8424 t4_tmr_idx_10g = TMR_IDX_10G; 8425 8426 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 8427 t4_pktc_idx_10g = PKTC_IDX_10G; 8428 8429 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 8430 t4_tmr_idx_1g = TMR_IDX_1G; 8431 8432 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 8433 t4_pktc_idx_1g = PKTC_IDX_1G; 8434 8435 if (t4_qsize_txq < 128) 8436 t4_qsize_txq = 128; 8437 8438 if (t4_qsize_rxq < 128) 8439 t4_qsize_rxq = 128; 8440 while (t4_qsize_rxq & 7) 8441 t4_qsize_rxq++; 8442 8443 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 8444 } 8445 8446 static struct sx mlu; /* mod load unload */ 8447 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload"); 8448 8449 static int 8450 mod_event(module_t mod, int cmd, void *arg) 8451 { 8452 int rc = 0; 8453 static int loaded = 0; 8454 8455 switch (cmd) { 8456 case MOD_LOAD: 8457 sx_xlock(&mlu); 8458 if (loaded++ == 0) { 8459 t4_sge_modload(); 8460 sx_init(&t4_list_lock, "T4/T5 adapters"); 8461 SLIST_INIT(&t4_list); 8462 #ifdef TCP_OFFLOAD 8463 sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); 8464 SLIST_INIT(&t4_uld_list); 8465 #endif 8466 t4_tracer_modload(); 8467 tweak_tunables(); 8468 } 8469 sx_xunlock(&mlu); 8470 break; 8471 8472 case MOD_UNLOAD: 8473 sx_xlock(&mlu); 8474 if (--loaded == 0) { 8475 int tries; 8476 8477 sx_slock(&t4_list_lock); 8478 if (!SLIST_EMPTY(&t4_list)) { 8479 rc = EBUSY; 8480 sx_sunlock(&t4_list_lock); 8481 goto done_unload; 8482 } 8483 #ifdef TCP_OFFLOAD 8484 sx_slock(&t4_uld_list_lock); 8485 if (!SLIST_EMPTY(&t4_uld_list)) { 8486 rc = EBUSY; 8487 sx_sunlock(&t4_uld_list_lock); 8488 sx_sunlock(&t4_list_lock); 8489 goto done_unload; 8490 } 8491 #endif 8492 tries = 0; 8493 while (tries++ < 5 && t4_sge_extfree_refs() != 0) { 8494 uprintf("%ju clusters with custom free routine " 8495 "still is use.\n", t4_sge_extfree_refs()); 8496 pause("t4unload", 2 * hz); 8497 } 8498 #ifdef TCP_OFFLOAD 8499 sx_sunlock(&t4_uld_list_lock); 8500 #endif 8501 sx_sunlock(&t4_list_lock); 8502 8503 if (t4_sge_extfree_refs() == 0) { 8504 t4_tracer_modunload(); 8505 #ifdef TCP_OFFLOAD 8506 sx_destroy(&t4_uld_list_lock); 8507 #endif 8508 sx_destroy(&t4_list_lock); 8509 t4_sge_modunload(); 8510 loaded = 0; 8511 } else { 8512 rc = EBUSY; 8513 loaded++; /* undo earlier decrement */ 8514 } 8515 } 8516 done_unload: 8517 sx_xunlock(&mlu); 8518 break; 8519 } 8520 8521 return (rc); 8522 } 8523 8524 static devclass_t t4_devclass, t5_devclass; 8525 static devclass_t cxgbe_devclass, cxl_devclass; 8526 8527 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 8528 MODULE_VERSION(t4nex, 1); 8529 MODULE_DEPEND(t4nex, firmware, 1, 1, 1); 8530 8531 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 8532 MODULE_VERSION(t5nex, 1); 8533 MODULE_DEPEND(t5nex, firmware, 1, 1, 1); 8534 8535 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 8536 MODULE_VERSION(cxgbe, 1); 8537 8538 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 8539 MODULE_VERSION(cxl, 1); 8540