1 /*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_inet.h" 32 #include "opt_inet6.h" 33 34 #include <sys/param.h> 35 #include <sys/conf.h> 36 #include <sys/priv.h> 37 #include <sys/kernel.h> 38 #include <sys/bus.h> 39 #include <sys/module.h> 40 #include <sys/malloc.h> 41 #include <sys/queue.h> 42 #include <sys/taskqueue.h> 43 #include <sys/pciio.h> 44 #include <dev/pci/pcireg.h> 45 #include <dev/pci/pcivar.h> 46 #include <dev/pci/pci_private.h> 47 #include <sys/firmware.h> 48 #include <sys/sbuf.h> 49 #include <sys/smp.h> 50 #include <sys/socket.h> 51 #include <sys/sockio.h> 52 #include <sys/sysctl.h> 53 #include <net/ethernet.h> 54 #include <net/if.h> 55 #include <net/if_types.h> 56 #include <net/if_dl.h> 57 #include <net/if_vlan_var.h> 58 #if defined(__i386__) || defined(__amd64__) 59 #include <vm/vm.h> 60 #include <vm/pmap.h> 61 #endif 62 63 #include "common/common.h" 64 #include "common/t4_msg.h" 65 #include "common/t4_regs.h" 66 #include "common/t4_regs_values.h" 67 #include "t4_ioctl.h" 68 #include "t4_l2t.h" 69 70 /* T4 bus driver interface */ 71 static int t4_probe(device_t); 72 static int t4_attach(device_t); 73 static int t4_detach(device_t); 74 static device_method_t t4_methods[] = { 75 DEVMETHOD(device_probe, t4_probe), 76 DEVMETHOD(device_attach, t4_attach), 77 DEVMETHOD(device_detach, t4_detach), 78 79 DEVMETHOD_END 80 }; 81 static driver_t t4_driver = { 82 "t4nex", 83 t4_methods, 84 sizeof(struct adapter) 85 }; 86 87 88 /* T4 port (cxgbe) interface */ 89 static int cxgbe_probe(device_t); 90 static int cxgbe_attach(device_t); 91 static int cxgbe_detach(device_t); 92 static device_method_t cxgbe_methods[] = { 93 DEVMETHOD(device_probe, cxgbe_probe), 94 DEVMETHOD(device_attach, cxgbe_attach), 95 DEVMETHOD(device_detach, cxgbe_detach), 96 { 0, 0 } 97 }; 98 static driver_t cxgbe_driver = { 99 "cxgbe", 100 cxgbe_methods, 101 sizeof(struct port_info) 102 }; 103 104 static d_ioctl_t t4_ioctl; 105 static d_open_t t4_open; 106 static d_close_t t4_close; 107 108 static struct cdevsw t4_cdevsw = { 109 .d_version = D_VERSION, 110 .d_flags = 0, 111 .d_open = t4_open, 112 .d_close = t4_close, 113 .d_ioctl = t4_ioctl, 114 .d_name = "t4nex", 115 }; 116 117 /* T5 bus driver interface */ 118 static int t5_probe(device_t); 119 static device_method_t t5_methods[] = { 120 DEVMETHOD(device_probe, t5_probe), 121 DEVMETHOD(device_attach, t4_attach), 122 DEVMETHOD(device_detach, t4_detach), 123 124 DEVMETHOD_END 125 }; 126 static driver_t t5_driver = { 127 "t5nex", 128 t5_methods, 129 sizeof(struct adapter) 130 }; 131 132 133 /* T5 port (cxl) interface */ 134 static driver_t cxl_driver = { 135 "cxl", 136 cxgbe_methods, 137 sizeof(struct port_info) 138 }; 139 140 static struct cdevsw t5_cdevsw = { 141 .d_version = D_VERSION, 142 .d_flags = 0, 143 .d_open = t4_open, 144 .d_close = t4_close, 145 .d_ioctl = t4_ioctl, 146 .d_name = "t5nex", 147 }; 148 149 /* ifnet + media interface */ 150 static void cxgbe_init(void *); 151 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 152 static int cxgbe_transmit(struct ifnet *, struct mbuf *); 153 static void cxgbe_qflush(struct ifnet *); 154 static uint64_t cxgbe_get_counter(struct ifnet *, ift_counter); 155 static int cxgbe_media_change(struct ifnet *); 156 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 157 158 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 159 160 /* 161 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 162 * then ADAPTER_LOCK, then t4_uld_list_lock. 163 */ 164 static struct sx t4_list_lock; 165 SLIST_HEAD(, adapter) t4_list; 166 #ifdef TCP_OFFLOAD 167 static struct sx t4_uld_list_lock; 168 SLIST_HEAD(, uld_info) t4_uld_list; 169 #endif 170 171 /* 172 * Tunables. See tweak_tunables() too. 173 * 174 * Each tunable is set to a default value here if it's known at compile-time. 175 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 176 * provide a reasonable default when the driver is loaded. 177 * 178 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 179 * T5 are under hw.cxl. 180 */ 181 182 /* 183 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 184 */ 185 #define NTXQ_10G 16 186 static int t4_ntxq10g = -1; 187 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 188 189 #define NRXQ_10G 8 190 static int t4_nrxq10g = -1; 191 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 192 193 #define NTXQ_1G 4 194 static int t4_ntxq1g = -1; 195 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 196 197 #define NRXQ_1G 2 198 static int t4_nrxq1g = -1; 199 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 200 201 static int t4_rsrv_noflowq = 0; 202 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq); 203 204 #ifdef TCP_OFFLOAD 205 #define NOFLDTXQ_10G 8 206 static int t4_nofldtxq10g = -1; 207 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 208 209 #define NOFLDRXQ_10G 2 210 static int t4_nofldrxq10g = -1; 211 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 212 213 #define NOFLDTXQ_1G 2 214 static int t4_nofldtxq1g = -1; 215 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 216 217 #define NOFLDRXQ_1G 1 218 static int t4_nofldrxq1g = -1; 219 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 220 #endif 221 222 #ifdef DEV_NETMAP 223 #define NNMTXQ_10G 2 224 static int t4_nnmtxq10g = -1; 225 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g); 226 227 #define NNMRXQ_10G 2 228 static int t4_nnmrxq10g = -1; 229 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g); 230 231 #define NNMTXQ_1G 1 232 static int t4_nnmtxq1g = -1; 233 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g); 234 235 #define NNMRXQ_1G 1 236 static int t4_nnmrxq1g = -1; 237 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g); 238 #endif 239 240 /* 241 * Holdoff parameters for 10G and 1G ports. 242 */ 243 #define TMR_IDX_10G 1 244 static int t4_tmr_idx_10g = TMR_IDX_10G; 245 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 246 247 #define PKTC_IDX_10G (-1) 248 static int t4_pktc_idx_10g = PKTC_IDX_10G; 249 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 250 251 #define TMR_IDX_1G 1 252 static int t4_tmr_idx_1g = TMR_IDX_1G; 253 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 254 255 #define PKTC_IDX_1G (-1) 256 static int t4_pktc_idx_1g = PKTC_IDX_1G; 257 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 258 259 /* 260 * Size (# of entries) of each tx and rx queue. 261 */ 262 static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 263 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 264 265 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 266 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 267 268 /* 269 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 270 */ 271 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 272 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 273 274 /* 275 * Configuration file. 276 */ 277 #define DEFAULT_CF "default" 278 #define FLASH_CF "flash" 279 #define UWIRE_CF "uwire" 280 #define FPGA_CF "fpga" 281 static char t4_cfg_file[32] = DEFAULT_CF; 282 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 283 284 /* 285 * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively). 286 * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them. 287 * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water 288 * mark or when signalled to do so, 0 to never emit PAUSE. 289 */ 290 static int t4_pause_settings = PAUSE_TX | PAUSE_RX; 291 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings); 292 293 /* 294 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 295 * encouraged respectively). 296 */ 297 static unsigned int t4_fw_install = 1; 298 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 299 300 /* 301 * ASIC features that will be used. Disable the ones you don't want so that the 302 * chip resources aren't wasted on features that will not be used. 303 */ 304 static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 305 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 306 307 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 308 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 309 310 static int t4_toecaps_allowed = -1; 311 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 312 313 static int t4_rdmacaps_allowed = 0; 314 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 315 316 static int t4_iscsicaps_allowed = 0; 317 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 318 319 static int t4_fcoecaps_allowed = 0; 320 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 321 322 static int t5_write_combine = 0; 323 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 324 325 struct intrs_and_queues { 326 uint16_t intr_type; /* INTx, MSI, or MSI-X */ 327 uint16_t nirq; /* Total # of vectors */ 328 uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */ 329 uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */ 330 uint16_t ntxq10g; /* # of NIC txq's for each 10G port */ 331 uint16_t nrxq10g; /* # of NIC rxq's for each 10G port */ 332 uint16_t ntxq1g; /* # of NIC txq's for each 1G port */ 333 uint16_t nrxq1g; /* # of NIC rxq's for each 1G port */ 334 uint16_t rsrv_noflowq; /* Flag whether to reserve queue 0 */ 335 #ifdef TCP_OFFLOAD 336 uint16_t nofldtxq10g; /* # of TOE txq's for each 10G port */ 337 uint16_t nofldrxq10g; /* # of TOE rxq's for each 10G port */ 338 uint16_t nofldtxq1g; /* # of TOE txq's for each 1G port */ 339 uint16_t nofldrxq1g; /* # of TOE rxq's for each 1G port */ 340 #endif 341 #ifdef DEV_NETMAP 342 uint16_t nnmtxq10g; /* # of netmap txq's for each 10G port */ 343 uint16_t nnmrxq10g; /* # of netmap rxq's for each 10G port */ 344 uint16_t nnmtxq1g; /* # of netmap txq's for each 1G port */ 345 uint16_t nnmrxq1g; /* # of netmap rxq's for each 1G port */ 346 #endif 347 }; 348 349 struct filter_entry { 350 uint32_t valid:1; /* filter allocated and valid */ 351 uint32_t locked:1; /* filter is administratively locked */ 352 uint32_t pending:1; /* filter action is pending firmware reply */ 353 uint32_t smtidx:8; /* Source MAC Table index for smac */ 354 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 355 356 struct t4_filter_specification fs; 357 }; 358 359 static int map_bars_0_and_4(struct adapter *); 360 static int map_bar_2(struct adapter *); 361 static void setup_memwin(struct adapter *); 362 static int validate_mem_range(struct adapter *, uint32_t, int); 363 static int fwmtype_to_hwmtype(int); 364 static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 365 uint32_t *); 366 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 367 static uint32_t position_memwin(struct adapter *, int, uint32_t); 368 static int cfg_itype_and_nqueues(struct adapter *, int, int, 369 struct intrs_and_queues *); 370 static int prep_firmware(struct adapter *); 371 static int partition_resources(struct adapter *, const struct firmware *, 372 const char *); 373 static int get_params__pre_init(struct adapter *); 374 static int get_params__post_init(struct adapter *); 375 static int set_params__post_init(struct adapter *); 376 static void t4_set_desc(struct adapter *); 377 static void build_medialist(struct port_info *, struct ifmedia *); 378 static int cxgbe_init_synchronized(struct port_info *); 379 static int cxgbe_uninit_synchronized(struct port_info *); 380 static int setup_intr_handlers(struct adapter *); 381 static void quiesce_eq(struct adapter *, struct sge_eq *); 382 static void quiesce_iq(struct adapter *, struct sge_iq *); 383 static void quiesce_fl(struct adapter *, struct sge_fl *); 384 static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 385 driver_intr_t *, void *, char *); 386 static int t4_free_irq(struct adapter *, struct irq *); 387 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 388 unsigned int); 389 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 390 static void cxgbe_refresh_stats(struct adapter *, struct port_info *); 391 static void cxgbe_tick(void *); 392 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 393 static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 394 struct mbuf *); 395 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 396 static int fw_msg_not_handled(struct adapter *, const __be64 *); 397 static int t4_sysctls(struct adapter *); 398 static int cxgbe_sysctls(struct port_info *); 399 static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 400 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 401 static int sysctl_btphy(SYSCTL_HANDLER_ARGS); 402 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); 403 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 404 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 405 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 406 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 407 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS); 408 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 409 static int sysctl_temperature(SYSCTL_HANDLER_ARGS); 410 #ifdef SBUF_DRAIN 411 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 412 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 413 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 414 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); 415 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); 416 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 417 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 418 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 419 static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 420 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 421 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 422 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 423 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); 424 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 425 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); 426 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 427 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 428 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 429 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 430 static int sysctl_tids(SYSCTL_HANDLER_ARGS); 431 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 432 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 433 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 434 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 435 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 436 #endif 437 static inline void txq_start(struct ifnet *, struct sge_txq *); 438 static uint32_t fconf_to_mode(uint32_t); 439 static uint32_t mode_to_fconf(uint32_t); 440 static uint32_t fspec_to_fconf(struct t4_filter_specification *); 441 static int get_filter_mode(struct adapter *, uint32_t *); 442 static int set_filter_mode(struct adapter *, uint32_t); 443 static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 444 static int get_filter(struct adapter *, struct t4_filter *); 445 static int set_filter(struct adapter *, struct t4_filter *); 446 static int del_filter(struct adapter *, struct t4_filter *); 447 static void clear_filter(struct filter_entry *); 448 static int set_filter_wr(struct adapter *, int); 449 static int del_filter_wr(struct adapter *, int); 450 static int get_sge_context(struct adapter *, struct t4_sge_context *); 451 static int load_fw(struct adapter *, struct t4_data *); 452 static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 453 static int read_i2c(struct adapter *, struct t4_i2c_data *); 454 static int set_sched_class(struct adapter *, struct t4_sched_params *); 455 static int set_sched_queue(struct adapter *, struct t4_sched_queue *); 456 #ifdef TCP_OFFLOAD 457 static int toe_capability(struct port_info *, int); 458 #endif 459 static int mod_event(module_t, int, void *); 460 461 struct { 462 uint16_t device; 463 char *desc; 464 } t4_pciids[] = { 465 {0xa000, "Chelsio Terminator 4 FPGA"}, 466 {0x4400, "Chelsio T440-dbg"}, 467 {0x4401, "Chelsio T420-CR"}, 468 {0x4402, "Chelsio T422-CR"}, 469 {0x4403, "Chelsio T440-CR"}, 470 {0x4404, "Chelsio T420-BCH"}, 471 {0x4405, "Chelsio T440-BCH"}, 472 {0x4406, "Chelsio T440-CH"}, 473 {0x4407, "Chelsio T420-SO"}, 474 {0x4408, "Chelsio T420-CX"}, 475 {0x4409, "Chelsio T420-BT"}, 476 {0x440a, "Chelsio T404-BT"}, 477 {0x440e, "Chelsio T440-LP-CR"}, 478 }, t5_pciids[] = { 479 {0xb000, "Chelsio Terminator 5 FPGA"}, 480 {0x5400, "Chelsio T580-dbg"}, 481 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ 482 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ 483 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ 484 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ 485 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ 486 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ 487 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ 488 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ 489 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 490 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ 491 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ 492 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ 493 {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */ 494 #ifdef notyet 495 {0x5404, "Chelsio T520-BCH"}, 496 {0x5405, "Chelsio T540-BCH"}, 497 {0x5406, "Chelsio T540-CH"}, 498 {0x5408, "Chelsio T520-CX"}, 499 {0x540b, "Chelsio B520-SR"}, 500 {0x540c, "Chelsio B504-BT"}, 501 {0x540f, "Chelsio Amsterdam"}, 502 {0x5413, "Chelsio T580-CHR"}, 503 #endif 504 }; 505 506 #ifdef TCP_OFFLOAD 507 /* 508 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 509 * exactly the same for both rxq and ofld_rxq. 510 */ 511 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 512 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 513 #endif 514 515 /* No easy way to include t4_msg.h before adapter.h so we check this way */ 516 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 517 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 518 519 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); 520 521 static int 522 t4_probe(device_t dev) 523 { 524 int i; 525 uint16_t v = pci_get_vendor(dev); 526 uint16_t d = pci_get_device(dev); 527 uint8_t f = pci_get_function(dev); 528 529 if (v != PCI_VENDOR_ID_CHELSIO) 530 return (ENXIO); 531 532 /* Attach only to PF0 of the FPGA */ 533 if (d == 0xa000 && f != 0) 534 return (ENXIO); 535 536 for (i = 0; i < nitems(t4_pciids); i++) { 537 if (d == t4_pciids[i].device) { 538 device_set_desc(dev, t4_pciids[i].desc); 539 return (BUS_PROBE_DEFAULT); 540 } 541 } 542 543 return (ENXIO); 544 } 545 546 static int 547 t5_probe(device_t dev) 548 { 549 int i; 550 uint16_t v = pci_get_vendor(dev); 551 uint16_t d = pci_get_device(dev); 552 uint8_t f = pci_get_function(dev); 553 554 if (v != PCI_VENDOR_ID_CHELSIO) 555 return (ENXIO); 556 557 /* Attach only to PF0 of the FPGA */ 558 if (d == 0xb000 && f != 0) 559 return (ENXIO); 560 561 for (i = 0; i < nitems(t5_pciids); i++) { 562 if (d == t5_pciids[i].device) { 563 device_set_desc(dev, t5_pciids[i].desc); 564 return (BUS_PROBE_DEFAULT); 565 } 566 } 567 568 return (ENXIO); 569 } 570 571 static int 572 t4_attach(device_t dev) 573 { 574 struct adapter *sc; 575 int rc = 0, i, n10g, n1g, rqidx, tqidx; 576 struct intrs_and_queues iaq; 577 struct sge *s; 578 #ifdef TCP_OFFLOAD 579 int ofld_rqidx, ofld_tqidx; 580 #endif 581 #ifdef DEV_NETMAP 582 int nm_rqidx, nm_tqidx; 583 #endif 584 const char *pcie_ts; 585 586 sc = device_get_softc(dev); 587 sc->dev = dev; 588 589 pci_enable_busmaster(dev); 590 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 591 uint32_t v; 592 593 pci_set_max_read_req(dev, 4096); 594 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 595 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 596 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 597 598 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5); 599 } 600 601 sc->traceq = -1; 602 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); 603 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", 604 device_get_nameunit(dev)); 605 606 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 607 device_get_nameunit(dev)); 608 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 609 sx_xlock(&t4_list_lock); 610 SLIST_INSERT_HEAD(&t4_list, sc, link); 611 sx_xunlock(&t4_list_lock); 612 613 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 614 TAILQ_INIT(&sc->sfl); 615 callout_init(&sc->sfl_callout, CALLOUT_MPSAFE); 616 617 mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF); 618 619 rc = map_bars_0_and_4(sc); 620 if (rc != 0) 621 goto done; /* error message displayed already */ 622 623 /* 624 * This is the real PF# to which we're attaching. Works from within PCI 625 * passthrough environments too, where pci_get_function() could return a 626 * different PF# depending on the passthrough configuration. We need to 627 * use the real PF# in all our communication with the firmware. 628 */ 629 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 630 sc->mbox = sc->pf; 631 632 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 633 sc->an_handler = an_not_handled; 634 for (i = 0; i < nitems(sc->cpl_handler); i++) 635 sc->cpl_handler[i] = cpl_not_handled; 636 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 637 sc->fw_msg_handler[i] = fw_msg_not_handled; 638 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 639 t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt); 640 t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt); 641 t4_init_sge_cpl_handlers(sc); 642 643 /* Prepare the adapter for operation */ 644 rc = -t4_prep_adapter(sc); 645 if (rc != 0) { 646 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 647 goto done; 648 } 649 650 /* 651 * Do this really early, with the memory windows set up even before the 652 * character device. The userland tool's register i/o and mem read 653 * will work even in "recovery mode". 654 */ 655 setup_memwin(sc); 656 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 657 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 658 device_get_nameunit(dev)); 659 if (sc->cdev == NULL) 660 device_printf(dev, "failed to create nexus char device.\n"); 661 else 662 sc->cdev->si_drv1 = sc; 663 664 /* Go no further if recovery mode has been requested. */ 665 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 666 device_printf(dev, "recovery mode.\n"); 667 goto done; 668 } 669 670 /* Prepare the firmware for operation */ 671 rc = prep_firmware(sc); 672 if (rc != 0) 673 goto done; /* error message displayed already */ 674 675 rc = get_params__post_init(sc); 676 if (rc != 0) 677 goto done; /* error message displayed already */ 678 679 rc = set_params__post_init(sc); 680 if (rc != 0) 681 goto done; /* error message displayed already */ 682 683 rc = map_bar_2(sc); 684 if (rc != 0) 685 goto done; /* error message displayed already */ 686 687 rc = t4_create_dma_tag(sc); 688 if (rc != 0) 689 goto done; /* error message displayed already */ 690 691 /* 692 * First pass over all the ports - allocate VIs and initialize some 693 * basic parameters like mac address, port type, etc. We also figure 694 * out whether a port is 10G or 1G and use that information when 695 * calculating how many interrupts to attempt to allocate. 696 */ 697 n10g = n1g = 0; 698 for_each_port(sc, i) { 699 struct port_info *pi; 700 701 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 702 sc->port[i] = pi; 703 704 /* These must be set before t4_port_init */ 705 pi->adapter = sc; 706 pi->port_id = i; 707 708 /* Allocate the vi and initialize parameters like mac addr */ 709 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 710 if (rc != 0) { 711 device_printf(dev, "unable to initialize port %d: %d\n", 712 i, rc); 713 free(pi, M_CXGBE); 714 sc->port[i] = NULL; 715 goto done; 716 } 717 718 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX); 719 pi->link_cfg.requested_fc |= t4_pause_settings; 720 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX); 721 pi->link_cfg.fc |= t4_pause_settings; 722 723 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 724 if (rc != 0) { 725 device_printf(dev, "port %d l1cfg failed: %d\n", i, rc); 726 free(pi, M_CXGBE); 727 sc->port[i] = NULL; 728 goto done; 729 } 730 731 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 732 device_get_nameunit(dev), i); 733 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 734 sc->chan_map[pi->tx_chan] = i; 735 736 if (is_10G_port(pi) || is_40G_port(pi)) { 737 n10g++; 738 pi->tmr_idx = t4_tmr_idx_10g; 739 pi->pktc_idx = t4_pktc_idx_10g; 740 } else { 741 n1g++; 742 pi->tmr_idx = t4_tmr_idx_1g; 743 pi->pktc_idx = t4_pktc_idx_1g; 744 } 745 746 pi->xact_addr_filt = -1; 747 pi->linkdnrc = -1; 748 749 pi->qsize_rxq = t4_qsize_rxq; 750 pi->qsize_txq = t4_qsize_txq; 751 752 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 753 if (pi->dev == NULL) { 754 device_printf(dev, 755 "failed to add device for port %d.\n", i); 756 rc = ENXIO; 757 goto done; 758 } 759 device_set_softc(pi->dev, pi); 760 } 761 762 /* 763 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 764 */ 765 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 766 if (rc != 0) 767 goto done; /* error message displayed already */ 768 769 sc->intr_type = iaq.intr_type; 770 sc->intr_count = iaq.nirq; 771 772 s = &sc->sge; 773 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 774 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 775 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 776 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 777 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 778 #ifdef TCP_OFFLOAD 779 if (is_offload(sc)) { 780 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 781 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 782 s->neq += s->nofldtxq + s->nofldrxq; 783 s->niq += s->nofldrxq; 784 785 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 786 M_CXGBE, M_ZERO | M_WAITOK); 787 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 788 M_CXGBE, M_ZERO | M_WAITOK); 789 } 790 #endif 791 #ifdef DEV_NETMAP 792 s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g; 793 s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g; 794 s->neq += s->nnmtxq + s->nnmrxq; 795 s->niq += s->nnmrxq; 796 797 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq), 798 M_CXGBE, M_ZERO | M_WAITOK); 799 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq), 800 M_CXGBE, M_ZERO | M_WAITOK); 801 #endif 802 803 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 804 M_ZERO | M_WAITOK); 805 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 806 M_ZERO | M_WAITOK); 807 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 808 M_ZERO | M_WAITOK); 809 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 810 M_ZERO | M_WAITOK); 811 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 812 M_ZERO | M_WAITOK); 813 814 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 815 M_ZERO | M_WAITOK); 816 817 t4_init_l2t(sc, M_WAITOK); 818 819 /* 820 * Second pass over the ports. This time we know the number of rx and 821 * tx queues that each port should get. 822 */ 823 rqidx = tqidx = 0; 824 #ifdef TCP_OFFLOAD 825 ofld_rqidx = ofld_tqidx = 0; 826 #endif 827 #ifdef DEV_NETMAP 828 nm_rqidx = nm_tqidx = 0; 829 #endif 830 for_each_port(sc, i) { 831 struct port_info *pi = sc->port[i]; 832 833 if (pi == NULL) 834 continue; 835 836 pi->first_rxq = rqidx; 837 pi->first_txq = tqidx; 838 if (is_10G_port(pi) || is_40G_port(pi)) { 839 pi->flags |= iaq.intr_flags_10g; 840 pi->nrxq = iaq.nrxq10g; 841 pi->ntxq = iaq.ntxq10g; 842 } else { 843 pi->flags |= iaq.intr_flags_1g; 844 pi->nrxq = iaq.nrxq1g; 845 pi->ntxq = iaq.ntxq1g; 846 } 847 848 if (pi->ntxq > 1) 849 pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0; 850 else 851 pi->rsrv_noflowq = 0; 852 853 rqidx += pi->nrxq; 854 tqidx += pi->ntxq; 855 #ifdef TCP_OFFLOAD 856 if (is_offload(sc)) { 857 pi->first_ofld_rxq = ofld_rqidx; 858 pi->first_ofld_txq = ofld_tqidx; 859 if (is_10G_port(pi) || is_40G_port(pi)) { 860 pi->nofldrxq = iaq.nofldrxq10g; 861 pi->nofldtxq = iaq.nofldtxq10g; 862 } else { 863 pi->nofldrxq = iaq.nofldrxq1g; 864 pi->nofldtxq = iaq.nofldtxq1g; 865 } 866 ofld_rqidx += pi->nofldrxq; 867 ofld_tqidx += pi->nofldtxq; 868 } 869 #endif 870 #ifdef DEV_NETMAP 871 pi->first_nm_rxq = nm_rqidx; 872 pi->first_nm_txq = nm_tqidx; 873 if (is_10G_port(pi) || is_40G_port(pi)) { 874 pi->nnmrxq = iaq.nnmrxq10g; 875 pi->nnmtxq = iaq.nnmtxq10g; 876 } else { 877 pi->nnmrxq = iaq.nnmrxq1g; 878 pi->nnmtxq = iaq.nnmtxq1g; 879 } 880 nm_rqidx += pi->nnmrxq; 881 nm_tqidx += pi->nnmtxq; 882 #endif 883 } 884 885 rc = setup_intr_handlers(sc); 886 if (rc != 0) { 887 device_printf(dev, 888 "failed to setup interrupt handlers: %d\n", rc); 889 goto done; 890 } 891 892 rc = bus_generic_attach(dev); 893 if (rc != 0) { 894 device_printf(dev, 895 "failed to attach all child ports: %d\n", rc); 896 goto done; 897 } 898 899 switch (sc->params.pci.speed) { 900 case 0x1: 901 pcie_ts = "2.5"; 902 break; 903 case 0x2: 904 pcie_ts = "5.0"; 905 break; 906 case 0x3: 907 pcie_ts = "8.0"; 908 break; 909 default: 910 pcie_ts = "??"; 911 break; 912 } 913 device_printf(dev, 914 "PCIe x%d (%s GTS/s) (%d), %d ports, %d %s interrupt%s, %d eq, %d iq\n", 915 sc->params.pci.width, pcie_ts, sc->params.pci.speed, 916 sc->params.nports, sc->intr_count, 917 sc->intr_type == INTR_MSIX ? "MSI-X" : 918 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 919 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 920 921 t4_set_desc(sc); 922 923 done: 924 if (rc != 0 && sc->cdev) { 925 /* cdev was created and so cxgbetool works; recover that way. */ 926 device_printf(dev, 927 "error during attach, adapter is now in recovery mode.\n"); 928 rc = 0; 929 } 930 931 if (rc != 0) 932 t4_detach(dev); 933 else 934 t4_sysctls(sc); 935 936 return (rc); 937 } 938 939 /* 940 * Idempotent 941 */ 942 static int 943 t4_detach(device_t dev) 944 { 945 struct adapter *sc; 946 struct port_info *pi; 947 int i, rc; 948 949 sc = device_get_softc(dev); 950 951 if (sc->flags & FULL_INIT_DONE) 952 t4_intr_disable(sc); 953 954 if (sc->cdev) { 955 destroy_dev(sc->cdev); 956 sc->cdev = NULL; 957 } 958 959 rc = bus_generic_detach(dev); 960 if (rc) { 961 device_printf(dev, 962 "failed to detach child devices: %d\n", rc); 963 return (rc); 964 } 965 966 for (i = 0; i < sc->intr_count; i++) 967 t4_free_irq(sc, &sc->irq[i]); 968 969 for (i = 0; i < MAX_NPORTS; i++) { 970 pi = sc->port[i]; 971 if (pi) { 972 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid); 973 if (pi->dev) 974 device_delete_child(dev, pi->dev); 975 976 mtx_destroy(&pi->pi_lock); 977 free(pi, M_CXGBE); 978 } 979 } 980 981 if (sc->flags & FULL_INIT_DONE) 982 adapter_full_uninit(sc); 983 984 if (sc->flags & FW_OK) 985 t4_fw_bye(sc, sc->mbox); 986 987 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 988 pci_release_msi(dev); 989 990 if (sc->regs_res) 991 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 992 sc->regs_res); 993 994 if (sc->udbs_res) 995 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 996 sc->udbs_res); 997 998 if (sc->msix_res) 999 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 1000 sc->msix_res); 1001 1002 if (sc->l2t) 1003 t4_free_l2t(sc->l2t); 1004 1005 #ifdef TCP_OFFLOAD 1006 free(sc->sge.ofld_rxq, M_CXGBE); 1007 free(sc->sge.ofld_txq, M_CXGBE); 1008 #endif 1009 #ifdef DEV_NETMAP 1010 free(sc->sge.nm_rxq, M_CXGBE); 1011 free(sc->sge.nm_txq, M_CXGBE); 1012 #endif 1013 free(sc->irq, M_CXGBE); 1014 free(sc->sge.rxq, M_CXGBE); 1015 free(sc->sge.txq, M_CXGBE); 1016 free(sc->sge.ctrlq, M_CXGBE); 1017 free(sc->sge.iqmap, M_CXGBE); 1018 free(sc->sge.eqmap, M_CXGBE); 1019 free(sc->tids.ftid_tab, M_CXGBE); 1020 t4_destroy_dma_tag(sc); 1021 if (mtx_initialized(&sc->sc_lock)) { 1022 sx_xlock(&t4_list_lock); 1023 SLIST_REMOVE(&t4_list, sc, adapter, link); 1024 sx_xunlock(&t4_list_lock); 1025 mtx_destroy(&sc->sc_lock); 1026 } 1027 1028 if (mtx_initialized(&sc->tids.ftid_lock)) 1029 mtx_destroy(&sc->tids.ftid_lock); 1030 if (mtx_initialized(&sc->sfl_lock)) 1031 mtx_destroy(&sc->sfl_lock); 1032 if (mtx_initialized(&sc->ifp_lock)) 1033 mtx_destroy(&sc->ifp_lock); 1034 if (mtx_initialized(&sc->regwin_lock)) 1035 mtx_destroy(&sc->regwin_lock); 1036 1037 bzero(sc, sizeof(*sc)); 1038 1039 return (0); 1040 } 1041 1042 static int 1043 cxgbe_probe(device_t dev) 1044 { 1045 char buf[128]; 1046 struct port_info *pi = device_get_softc(dev); 1047 1048 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 1049 device_set_desc_copy(dev, buf); 1050 1051 return (BUS_PROBE_DEFAULT); 1052 } 1053 1054 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 1055 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 1056 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS) 1057 #define T4_CAP_ENABLE (T4_CAP) 1058 1059 static int 1060 cxgbe_attach(device_t dev) 1061 { 1062 struct port_info *pi = device_get_softc(dev); 1063 struct ifnet *ifp; 1064 char *s; 1065 int n, o; 1066 1067 /* Allocate an ifnet and set it up */ 1068 ifp = if_alloc(IFT_ETHER); 1069 if (ifp == NULL) { 1070 device_printf(dev, "Cannot allocate ifnet\n"); 1071 return (ENOMEM); 1072 } 1073 pi->ifp = ifp; 1074 ifp->if_softc = pi; 1075 1076 callout_init(&pi->tick, CALLOUT_MPSAFE); 1077 1078 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1079 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1080 1081 ifp->if_init = cxgbe_init; 1082 ifp->if_ioctl = cxgbe_ioctl; 1083 ifp->if_transmit = cxgbe_transmit; 1084 ifp->if_qflush = cxgbe_qflush; 1085 ifp->if_get_counter = cxgbe_get_counter; 1086 1087 ifp->if_capabilities = T4_CAP; 1088 #ifdef TCP_OFFLOAD 1089 if (is_offload(pi->adapter)) 1090 ifp->if_capabilities |= IFCAP_TOE; 1091 #endif 1092 ifp->if_capenable = T4_CAP_ENABLE; 1093 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1094 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1095 1096 ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1097 ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS; 1098 ifp->if_hw_tsomaxsegsize = 65536; 1099 1100 /* Initialize ifmedia for this port */ 1101 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 1102 cxgbe_media_status); 1103 build_medialist(pi, &pi->media); 1104 1105 pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 1106 EVENTHANDLER_PRI_ANY); 1107 1108 ether_ifattach(ifp, pi->hw_addr); 1109 1110 n = 128; 1111 s = malloc(n, M_CXGBE, M_WAITOK); 1112 o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq); 1113 MPASS(n > o); 1114 #ifdef TCP_OFFLOAD 1115 if (is_offload(pi->adapter)) { 1116 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)", 1117 pi->nofldtxq, pi->nofldrxq); 1118 MPASS(n > o); 1119 } 1120 #endif 1121 #ifdef DEV_NETMAP 1122 o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq, 1123 pi->nnmrxq); 1124 MPASS(n > o); 1125 #endif 1126 device_printf(dev, "%s\n", s); 1127 free(s, M_CXGBE); 1128 1129 #ifdef DEV_NETMAP 1130 /* nm_media handled here to keep implementation private to this file */ 1131 ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change, 1132 cxgbe_media_status); 1133 build_medialist(pi, &pi->nm_media); 1134 create_netmap_ifnet(pi); /* logs errors it something fails */ 1135 #endif 1136 cxgbe_sysctls(pi); 1137 1138 return (0); 1139 } 1140 1141 static int 1142 cxgbe_detach(device_t dev) 1143 { 1144 struct port_info *pi = device_get_softc(dev); 1145 struct adapter *sc = pi->adapter; 1146 struct ifnet *ifp = pi->ifp; 1147 1148 /* Tell if_ioctl and if_init that the port is going away */ 1149 ADAPTER_LOCK(sc); 1150 SET_DOOMED(pi); 1151 wakeup(&sc->flags); 1152 while (IS_BUSY(sc)) 1153 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 1154 SET_BUSY(sc); 1155 #ifdef INVARIANTS 1156 sc->last_op = "t4detach"; 1157 sc->last_op_thr = curthread; 1158 #endif 1159 ADAPTER_UNLOCK(sc); 1160 1161 if (pi->flags & HAS_TRACEQ) { 1162 sc->traceq = -1; /* cloner should not create ifnet */ 1163 t4_tracer_port_detach(sc); 1164 } 1165 1166 if (pi->vlan_c) 1167 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c); 1168 1169 PORT_LOCK(pi); 1170 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1171 callout_stop(&pi->tick); 1172 PORT_UNLOCK(pi); 1173 callout_drain(&pi->tick); 1174 1175 /* Let detach proceed even if these fail. */ 1176 cxgbe_uninit_synchronized(pi); 1177 port_full_uninit(pi); 1178 1179 ifmedia_removeall(&pi->media); 1180 ether_ifdetach(pi->ifp); 1181 if_free(pi->ifp); 1182 1183 #ifdef DEV_NETMAP 1184 /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */ 1185 destroy_netmap_ifnet(pi); 1186 #endif 1187 1188 ADAPTER_LOCK(sc); 1189 CLR_BUSY(sc); 1190 wakeup(&sc->flags); 1191 ADAPTER_UNLOCK(sc); 1192 1193 return (0); 1194 } 1195 1196 static void 1197 cxgbe_init(void *arg) 1198 { 1199 struct port_info *pi = arg; 1200 struct adapter *sc = pi->adapter; 1201 1202 if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0) 1203 return; 1204 cxgbe_init_synchronized(pi); 1205 end_synchronized_op(sc, 0); 1206 } 1207 1208 static int 1209 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1210 { 1211 int rc = 0, mtu, flags, can_sleep; 1212 struct port_info *pi = ifp->if_softc; 1213 struct adapter *sc = pi->adapter; 1214 struct ifreq *ifr = (struct ifreq *)data; 1215 uint32_t mask; 1216 1217 switch (cmd) { 1218 case SIOCSIFMTU: 1219 mtu = ifr->ifr_mtu; 1220 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1221 return (EINVAL); 1222 1223 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu"); 1224 if (rc) 1225 return (rc); 1226 ifp->if_mtu = mtu; 1227 if (pi->flags & PORT_INIT_DONE) { 1228 t4_update_fl_bufsize(ifp); 1229 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1230 rc = update_mac_settings(ifp, XGMAC_MTU); 1231 } 1232 end_synchronized_op(sc, 0); 1233 break; 1234 1235 case SIOCSIFFLAGS: 1236 can_sleep = 0; 1237 redo_sifflags: 1238 rc = begin_synchronized_op(sc, pi, 1239 can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg"); 1240 if (rc) 1241 return (rc); 1242 1243 if (ifp->if_flags & IFF_UP) { 1244 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1245 flags = pi->if_flags; 1246 if ((ifp->if_flags ^ flags) & 1247 (IFF_PROMISC | IFF_ALLMULTI)) { 1248 if (can_sleep == 1) { 1249 end_synchronized_op(sc, 0); 1250 can_sleep = 0; 1251 goto redo_sifflags; 1252 } 1253 rc = update_mac_settings(ifp, 1254 XGMAC_PROMISC | XGMAC_ALLMULTI); 1255 } 1256 } else { 1257 if (can_sleep == 0) { 1258 end_synchronized_op(sc, LOCK_HELD); 1259 can_sleep = 1; 1260 goto redo_sifflags; 1261 } 1262 rc = cxgbe_init_synchronized(pi); 1263 } 1264 pi->if_flags = ifp->if_flags; 1265 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1266 if (can_sleep == 0) { 1267 end_synchronized_op(sc, LOCK_HELD); 1268 can_sleep = 1; 1269 goto redo_sifflags; 1270 } 1271 rc = cxgbe_uninit_synchronized(pi); 1272 } 1273 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD); 1274 break; 1275 1276 case SIOCADDMULTI: 1277 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1278 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi"); 1279 if (rc) 1280 return (rc); 1281 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1282 rc = update_mac_settings(ifp, XGMAC_MCADDRS); 1283 end_synchronized_op(sc, LOCK_HELD); 1284 break; 1285 1286 case SIOCSIFCAP: 1287 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap"); 1288 if (rc) 1289 return (rc); 1290 1291 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1292 if (mask & IFCAP_TXCSUM) { 1293 ifp->if_capenable ^= IFCAP_TXCSUM; 1294 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1295 1296 if (IFCAP_TSO4 & ifp->if_capenable && 1297 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1298 ifp->if_capenable &= ~IFCAP_TSO4; 1299 if_printf(ifp, 1300 "tso4 disabled due to -txcsum.\n"); 1301 } 1302 } 1303 if (mask & IFCAP_TXCSUM_IPV6) { 1304 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1305 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1306 1307 if (IFCAP_TSO6 & ifp->if_capenable && 1308 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1309 ifp->if_capenable &= ~IFCAP_TSO6; 1310 if_printf(ifp, 1311 "tso6 disabled due to -txcsum6.\n"); 1312 } 1313 } 1314 if (mask & IFCAP_RXCSUM) 1315 ifp->if_capenable ^= IFCAP_RXCSUM; 1316 if (mask & IFCAP_RXCSUM_IPV6) 1317 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1318 1319 /* 1320 * Note that we leave CSUM_TSO alone (it is always set). The 1321 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1322 * sending a TSO request our way, so it's sufficient to toggle 1323 * IFCAP_TSOx only. 1324 */ 1325 if (mask & IFCAP_TSO4) { 1326 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1327 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1328 if_printf(ifp, "enable txcsum first.\n"); 1329 rc = EAGAIN; 1330 goto fail; 1331 } 1332 ifp->if_capenable ^= IFCAP_TSO4; 1333 } 1334 if (mask & IFCAP_TSO6) { 1335 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1336 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1337 if_printf(ifp, "enable txcsum6 first.\n"); 1338 rc = EAGAIN; 1339 goto fail; 1340 } 1341 ifp->if_capenable ^= IFCAP_TSO6; 1342 } 1343 if (mask & IFCAP_LRO) { 1344 #if defined(INET) || defined(INET6) 1345 int i; 1346 struct sge_rxq *rxq; 1347 1348 ifp->if_capenable ^= IFCAP_LRO; 1349 for_each_rxq(pi, i, rxq) { 1350 if (ifp->if_capenable & IFCAP_LRO) 1351 rxq->iq.flags |= IQ_LRO_ENABLED; 1352 else 1353 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1354 } 1355 #endif 1356 } 1357 #ifdef TCP_OFFLOAD 1358 if (mask & IFCAP_TOE) { 1359 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1360 1361 rc = toe_capability(pi, enable); 1362 if (rc != 0) 1363 goto fail; 1364 1365 ifp->if_capenable ^= mask; 1366 } 1367 #endif 1368 if (mask & IFCAP_VLAN_HWTAGGING) { 1369 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1370 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1371 rc = update_mac_settings(ifp, XGMAC_VLANEX); 1372 } 1373 if (mask & IFCAP_VLAN_MTU) { 1374 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1375 1376 /* Need to find out how to disable auto-mtu-inflation */ 1377 } 1378 if (mask & IFCAP_VLAN_HWTSO) 1379 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1380 if (mask & IFCAP_VLAN_HWCSUM) 1381 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1382 1383 #ifdef VLAN_CAPABILITIES 1384 VLAN_CAPABILITIES(ifp); 1385 #endif 1386 fail: 1387 end_synchronized_op(sc, 0); 1388 break; 1389 1390 case SIOCSIFMEDIA: 1391 case SIOCGIFMEDIA: 1392 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1393 break; 1394 1395 case SIOCGI2C: { 1396 struct ifi2creq i2c; 1397 1398 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); 1399 if (rc != 0) 1400 break; 1401 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) { 1402 rc = EPERM; 1403 break; 1404 } 1405 if (i2c.len > sizeof(i2c.data)) { 1406 rc = EINVAL; 1407 break; 1408 } 1409 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4i2c"); 1410 if (rc) 1411 return (rc); 1412 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr, 1413 i2c.offset, i2c.len, &i2c.data[0]); 1414 end_synchronized_op(sc, 0); 1415 if (rc == 0) 1416 rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); 1417 break; 1418 } 1419 1420 default: 1421 rc = ether_ioctl(ifp, cmd, data); 1422 } 1423 1424 return (rc); 1425 } 1426 1427 static int 1428 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1429 { 1430 struct port_info *pi = ifp->if_softc; 1431 struct adapter *sc = pi->adapter; 1432 struct sge_txq *txq = &sc->sge.txq[pi->first_txq]; 1433 struct buf_ring *br; 1434 int rc; 1435 1436 M_ASSERTPKTHDR(m); 1437 1438 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1439 m_freem(m); 1440 return (ENETDOWN); 1441 } 1442 1443 if (m->m_flags & M_FLOWID) 1444 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) 1445 + pi->rsrv_noflowq); 1446 br = txq->br; 1447 1448 if (TXQ_TRYLOCK(txq) == 0) { 1449 struct sge_eq *eq = &txq->eq; 1450 1451 /* 1452 * It is possible that t4_eth_tx finishes up and releases the 1453 * lock between the TRYLOCK above and the drbr_enqueue here. We 1454 * need to make sure that this mbuf doesn't just sit there in 1455 * the drbr. 1456 */ 1457 1458 rc = drbr_enqueue(ifp, br, m); 1459 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 && 1460 !(eq->flags & EQ_DOOMED)) 1461 callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq); 1462 return (rc); 1463 } 1464 1465 /* 1466 * txq->m is the mbuf that is held up due to a temporary shortage of 1467 * resources and it should be put on the wire first. Then what's in 1468 * drbr and finally the mbuf that was just passed in to us. 1469 * 1470 * Return code should indicate the fate of the mbuf that was passed in 1471 * this time. 1472 */ 1473 1474 TXQ_LOCK_ASSERT_OWNED(txq); 1475 if (drbr_needs_enqueue(ifp, br) || txq->m) { 1476 1477 /* Queued for transmission. */ 1478 1479 rc = drbr_enqueue(ifp, br, m); 1480 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 1481 (void) t4_eth_tx(ifp, txq, m); 1482 TXQ_UNLOCK(txq); 1483 return (rc); 1484 } 1485 1486 /* Direct transmission. */ 1487 rc = t4_eth_tx(ifp, txq, m); 1488 if (rc != 0 && txq->m) 1489 rc = 0; /* held, will be transmitted soon (hopefully) */ 1490 1491 TXQ_UNLOCK(txq); 1492 return (rc); 1493 } 1494 1495 static void 1496 cxgbe_qflush(struct ifnet *ifp) 1497 { 1498 struct port_info *pi = ifp->if_softc; 1499 struct sge_txq *txq; 1500 int i; 1501 struct mbuf *m; 1502 1503 /* queues do not exist if !PORT_INIT_DONE. */ 1504 if (pi->flags & PORT_INIT_DONE) { 1505 for_each_txq(pi, i, txq) { 1506 TXQ_LOCK(txq); 1507 m_freem(txq->m); 1508 txq->m = NULL; 1509 while ((m = buf_ring_dequeue_sc(txq->br)) != NULL) 1510 m_freem(m); 1511 TXQ_UNLOCK(txq); 1512 } 1513 } 1514 if_qflush(ifp); 1515 } 1516 1517 static uint64_t 1518 cxgbe_get_counter(struct ifnet *ifp, ift_counter c) 1519 { 1520 struct port_info *pi = ifp->if_softc; 1521 struct adapter *sc = pi->adapter; 1522 struct port_stats *s = &pi->stats; 1523 1524 cxgbe_refresh_stats(sc, pi); 1525 1526 switch (c) { 1527 case IFCOUNTER_IPACKETS: 1528 return (s->rx_frames - s->rx_pause); 1529 1530 case IFCOUNTER_IERRORS: 1531 return (s->rx_jabber + s->rx_runt + s->rx_too_long + 1532 s->rx_fcs_err + s->rx_len_err); 1533 1534 case IFCOUNTER_OPACKETS: 1535 return (s->tx_frames - s->tx_pause); 1536 1537 case IFCOUNTER_OERRORS: 1538 return (s->tx_error_frames); 1539 1540 case IFCOUNTER_IBYTES: 1541 return (s->rx_octets - s->rx_pause * 64); 1542 1543 case IFCOUNTER_OBYTES: 1544 return (s->tx_octets - s->tx_pause * 64); 1545 1546 case IFCOUNTER_IMCASTS: 1547 return (s->rx_mcast_frames - s->rx_pause); 1548 1549 case IFCOUNTER_OMCASTS: 1550 return (s->tx_mcast_frames - s->tx_pause); 1551 1552 case IFCOUNTER_IQDROPS: 1553 return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 1554 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 1555 s->rx_trunc3 + pi->tnl_cong_drops); 1556 1557 case IFCOUNTER_OQDROPS: { 1558 uint64_t drops; 1559 1560 drops = s->tx_drop; 1561 if (pi->flags & PORT_INIT_DONE) { 1562 int i; 1563 struct sge_txq *txq; 1564 1565 for_each_txq(pi, i, txq) 1566 drops += txq->br->br_drops; 1567 } 1568 1569 return (drops); 1570 1571 } 1572 1573 default: 1574 return (if_get_counter_default(ifp, c)); 1575 } 1576 } 1577 1578 static int 1579 cxgbe_media_change(struct ifnet *ifp) 1580 { 1581 struct port_info *pi = ifp->if_softc; 1582 1583 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1584 1585 return (EOPNOTSUPP); 1586 } 1587 1588 static void 1589 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1590 { 1591 struct port_info *pi = ifp->if_softc; 1592 struct ifmedia *media = NULL; 1593 struct ifmedia_entry *cur; 1594 int speed = pi->link_cfg.speed; 1595 #ifdef INVARIANTS 1596 int data = (pi->port_type << 8) | pi->mod_type; 1597 #endif 1598 1599 if (ifp == pi->ifp) 1600 media = &pi->media; 1601 #ifdef DEV_NETMAP 1602 else if (ifp == pi->nm_ifp) 1603 media = &pi->nm_media; 1604 #endif 1605 MPASS(media != NULL); 1606 1607 cur = media->ifm_cur; 1608 MPASS(cur->ifm_data == data); 1609 1610 ifmr->ifm_status = IFM_AVALID; 1611 if (!pi->link_cfg.link_ok) 1612 return; 1613 1614 ifmr->ifm_status |= IFM_ACTIVE; 1615 1616 /* active and current will differ iff current media is autoselect. */ 1617 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1618 return; 1619 1620 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1621 if (speed == SPEED_10000) 1622 ifmr->ifm_active |= IFM_10G_T; 1623 else if (speed == SPEED_1000) 1624 ifmr->ifm_active |= IFM_1000_T; 1625 else if (speed == SPEED_100) 1626 ifmr->ifm_active |= IFM_100_TX; 1627 else if (speed == SPEED_10) 1628 ifmr->ifm_active |= IFM_10_T; 1629 else 1630 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1631 speed)); 1632 } 1633 1634 void 1635 t4_fatal_err(struct adapter *sc) 1636 { 1637 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1638 t4_intr_disable(sc); 1639 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1640 device_get_nameunit(sc->dev)); 1641 } 1642 1643 static int 1644 map_bars_0_and_4(struct adapter *sc) 1645 { 1646 sc->regs_rid = PCIR_BAR(0); 1647 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1648 &sc->regs_rid, RF_ACTIVE); 1649 if (sc->regs_res == NULL) { 1650 device_printf(sc->dev, "cannot map registers.\n"); 1651 return (ENXIO); 1652 } 1653 sc->bt = rman_get_bustag(sc->regs_res); 1654 sc->bh = rman_get_bushandle(sc->regs_res); 1655 sc->mmio_len = rman_get_size(sc->regs_res); 1656 setbit(&sc->doorbells, DOORBELL_KDB); 1657 1658 sc->msix_rid = PCIR_BAR(4); 1659 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1660 &sc->msix_rid, RF_ACTIVE); 1661 if (sc->msix_res == NULL) { 1662 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1663 return (ENXIO); 1664 } 1665 1666 return (0); 1667 } 1668 1669 static int 1670 map_bar_2(struct adapter *sc) 1671 { 1672 1673 /* 1674 * T4: only iWARP driver uses the userspace doorbells. There is no need 1675 * to map it if RDMA is disabled. 1676 */ 1677 if (is_t4(sc) && sc->rdmacaps == 0) 1678 return (0); 1679 1680 sc->udbs_rid = PCIR_BAR(2); 1681 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1682 &sc->udbs_rid, RF_ACTIVE); 1683 if (sc->udbs_res == NULL) { 1684 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1685 return (ENXIO); 1686 } 1687 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1688 1689 if (is_t5(sc)) { 1690 setbit(&sc->doorbells, DOORBELL_UDB); 1691 #if defined(__i386__) || defined(__amd64__) 1692 if (t5_write_combine) { 1693 int rc; 1694 1695 /* 1696 * Enable write combining on BAR2. This is the 1697 * userspace doorbell BAR and is split into 128B 1698 * (UDBS_SEG_SIZE) doorbell regions, each associated 1699 * with an egress queue. The first 64B has the doorbell 1700 * and the second 64B can be used to submit a tx work 1701 * request with an implicit doorbell. 1702 */ 1703 1704 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1705 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1706 if (rc == 0) { 1707 clrbit(&sc->doorbells, DOORBELL_UDB); 1708 setbit(&sc->doorbells, DOORBELL_WCWR); 1709 setbit(&sc->doorbells, DOORBELL_UDBWC); 1710 } else { 1711 device_printf(sc->dev, 1712 "couldn't enable write combining: %d\n", 1713 rc); 1714 } 1715 1716 t4_write_reg(sc, A_SGE_STAT_CFG, 1717 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1718 } 1719 #endif 1720 } 1721 1722 return (0); 1723 } 1724 1725 static const struct memwin t4_memwin[] = { 1726 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1727 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1728 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1729 }; 1730 1731 static const struct memwin t5_memwin[] = { 1732 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1733 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1734 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1735 }; 1736 1737 static void 1738 setup_memwin(struct adapter *sc) 1739 { 1740 const struct memwin *mw; 1741 int i, n; 1742 uint32_t bar0; 1743 1744 if (is_t4(sc)) { 1745 /* 1746 * Read low 32b of bar0 indirectly via the hardware backdoor 1747 * mechanism. Works from within PCI passthrough environments 1748 * too, where rman_get_start() can return a different value. We 1749 * need to program the T4 memory window decoders with the actual 1750 * addresses that will be coming across the PCIe link. 1751 */ 1752 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1753 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1754 1755 mw = &t4_memwin[0]; 1756 n = nitems(t4_memwin); 1757 } else { 1758 /* T5 uses the relative offset inside the PCIe BAR */ 1759 bar0 = 0; 1760 1761 mw = &t5_memwin[0]; 1762 n = nitems(t5_memwin); 1763 } 1764 1765 for (i = 0; i < n; i++, mw++) { 1766 t4_write_reg(sc, 1767 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 1768 (mw->base + bar0) | V_BIR(0) | 1769 V_WINDOW(ilog2(mw->aperture) - 10)); 1770 } 1771 1772 /* flush */ 1773 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1774 } 1775 1776 /* 1777 * Verify that the memory range specified by the addr/len pair is valid and lies 1778 * entirely within a single region (EDCx or MCx). 1779 */ 1780 static int 1781 validate_mem_range(struct adapter *sc, uint32_t addr, int len) 1782 { 1783 uint32_t em, addr_len, maddr, mlen; 1784 1785 /* Memory can only be accessed in naturally aligned 4 byte units */ 1786 if (addr & 3 || len & 3 || len == 0) 1787 return (EINVAL); 1788 1789 /* Enabled memories */ 1790 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1791 if (em & F_EDRAM0_ENABLE) { 1792 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1793 maddr = G_EDRAM0_BASE(addr_len) << 20; 1794 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1795 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1796 addr + len <= maddr + mlen) 1797 return (0); 1798 } 1799 if (em & F_EDRAM1_ENABLE) { 1800 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1801 maddr = G_EDRAM1_BASE(addr_len) << 20; 1802 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1803 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1804 addr + len <= maddr + mlen) 1805 return (0); 1806 } 1807 if (em & F_EXT_MEM_ENABLE) { 1808 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1809 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1810 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1811 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1812 addr + len <= maddr + mlen) 1813 return (0); 1814 } 1815 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 1816 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1817 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1818 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1819 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1820 addr + len <= maddr + mlen) 1821 return (0); 1822 } 1823 1824 return (EFAULT); 1825 } 1826 1827 static int 1828 fwmtype_to_hwmtype(int mtype) 1829 { 1830 1831 switch (mtype) { 1832 case FW_MEMTYPE_EDC0: 1833 return (MEM_EDC0); 1834 case FW_MEMTYPE_EDC1: 1835 return (MEM_EDC1); 1836 case FW_MEMTYPE_EXTMEM: 1837 return (MEM_MC0); 1838 case FW_MEMTYPE_EXTMEM1: 1839 return (MEM_MC1); 1840 default: 1841 panic("%s: cannot translate fw mtype %d.", __func__, mtype); 1842 } 1843 } 1844 1845 /* 1846 * Verify that the memory range specified by the memtype/offset/len pair is 1847 * valid and lies entirely within the memtype specified. The global address of 1848 * the start of the range is returned in addr. 1849 */ 1850 static int 1851 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 1852 uint32_t *addr) 1853 { 1854 uint32_t em, addr_len, maddr, mlen; 1855 1856 /* Memory can only be accessed in naturally aligned 4 byte units */ 1857 if (off & 3 || len & 3 || len == 0) 1858 return (EINVAL); 1859 1860 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1861 switch (fwmtype_to_hwmtype(mtype)) { 1862 case MEM_EDC0: 1863 if (!(em & F_EDRAM0_ENABLE)) 1864 return (EINVAL); 1865 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1866 maddr = G_EDRAM0_BASE(addr_len) << 20; 1867 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1868 break; 1869 case MEM_EDC1: 1870 if (!(em & F_EDRAM1_ENABLE)) 1871 return (EINVAL); 1872 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1873 maddr = G_EDRAM1_BASE(addr_len) << 20; 1874 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1875 break; 1876 case MEM_MC: 1877 if (!(em & F_EXT_MEM_ENABLE)) 1878 return (EINVAL); 1879 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1880 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1881 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1882 break; 1883 case MEM_MC1: 1884 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 1885 return (EINVAL); 1886 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1887 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1888 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1889 break; 1890 default: 1891 return (EINVAL); 1892 } 1893 1894 if (mlen > 0 && off < mlen && off + len <= mlen) { 1895 *addr = maddr + off; /* global address */ 1896 return (0); 1897 } 1898 1899 return (EFAULT); 1900 } 1901 1902 static void 1903 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 1904 { 1905 const struct memwin *mw; 1906 1907 if (is_t4(sc)) { 1908 KASSERT(win >= 0 && win < nitems(t4_memwin), 1909 ("%s: incorrect memwin# (%d)", __func__, win)); 1910 mw = &t4_memwin[win]; 1911 } else { 1912 KASSERT(win >= 0 && win < nitems(t5_memwin), 1913 ("%s: incorrect memwin# (%d)", __func__, win)); 1914 mw = &t5_memwin[win]; 1915 } 1916 1917 if (base != NULL) 1918 *base = mw->base; 1919 if (aperture != NULL) 1920 *aperture = mw->aperture; 1921 } 1922 1923 /* 1924 * Positions the memory window such that it can be used to access the specified 1925 * address in the chip's address space. The return value is the offset of addr 1926 * from the start of the window. 1927 */ 1928 static uint32_t 1929 position_memwin(struct adapter *sc, int n, uint32_t addr) 1930 { 1931 uint32_t start, pf; 1932 uint32_t reg; 1933 1934 KASSERT(n >= 0 && n <= 3, 1935 ("%s: invalid window %d.", __func__, n)); 1936 KASSERT((addr & 3) == 0, 1937 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 1938 1939 if (is_t4(sc)) { 1940 pf = 0; 1941 start = addr & ~0xf; /* start must be 16B aligned */ 1942 } else { 1943 pf = V_PFNUM(sc->pf); 1944 start = addr & ~0x7f; /* start must be 128B aligned */ 1945 } 1946 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 1947 1948 t4_write_reg(sc, reg, start | pf); 1949 t4_read_reg(sc, reg); 1950 1951 return (addr - start); 1952 } 1953 1954 static int 1955 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1956 struct intrs_and_queues *iaq) 1957 { 1958 int rc, itype, navail, nrxq10g, nrxq1g, n; 1959 int nofldrxq10g = 0, nofldrxq1g = 0; 1960 int nnmrxq10g = 0, nnmrxq1g = 0; 1961 1962 bzero(iaq, sizeof(*iaq)); 1963 1964 iaq->ntxq10g = t4_ntxq10g; 1965 iaq->ntxq1g = t4_ntxq1g; 1966 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1967 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1968 iaq->rsrv_noflowq = t4_rsrv_noflowq; 1969 #ifdef TCP_OFFLOAD 1970 if (is_offload(sc)) { 1971 iaq->nofldtxq10g = t4_nofldtxq10g; 1972 iaq->nofldtxq1g = t4_nofldtxq1g; 1973 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1974 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1975 } 1976 #endif 1977 #ifdef DEV_NETMAP 1978 iaq->nnmtxq10g = t4_nnmtxq10g; 1979 iaq->nnmtxq1g = t4_nnmtxq1g; 1980 iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g; 1981 iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g; 1982 #endif 1983 1984 for (itype = INTR_MSIX; itype; itype >>= 1) { 1985 1986 if ((itype & t4_intr_types) == 0) 1987 continue; /* not allowed */ 1988 1989 if (itype == INTR_MSIX) 1990 navail = pci_msix_count(sc->dev); 1991 else if (itype == INTR_MSI) 1992 navail = pci_msi_count(sc->dev); 1993 else 1994 navail = 1; 1995 restart: 1996 if (navail == 0) 1997 continue; 1998 1999 iaq->intr_type = itype; 2000 iaq->intr_flags_10g = 0; 2001 iaq->intr_flags_1g = 0; 2002 2003 /* 2004 * Best option: an interrupt vector for errors, one for the 2005 * firmware event queue, and one for every rxq (NIC, TOE, and 2006 * netmap). 2007 */ 2008 iaq->nirq = T4_EXTRA_INTR; 2009 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g); 2010 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g); 2011 if (iaq->nirq <= navail && 2012 (itype != INTR_MSI || powerof2(iaq->nirq))) { 2013 iaq->intr_flags_10g = INTR_ALL; 2014 iaq->intr_flags_1g = INTR_ALL; 2015 goto allocate; 2016 } 2017 2018 /* 2019 * Second best option: a vector for errors, one for the firmware 2020 * event queue, and vectors for either all the NIC rx queues or 2021 * all the TOE rx queues. The queues that don't get vectors 2022 * will forward their interrupts to those that do. 2023 * 2024 * Note: netmap rx queues cannot be created early and so they 2025 * can't be setup to receive forwarded interrupts for others. 2026 */ 2027 iaq->nirq = T4_EXTRA_INTR; 2028 if (nrxq10g >= nofldrxq10g) { 2029 iaq->intr_flags_10g = INTR_RXQ; 2030 iaq->nirq += n10g * nrxq10g; 2031 #ifdef DEV_NETMAP 2032 iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g); 2033 #endif 2034 } else { 2035 iaq->intr_flags_10g = INTR_OFLD_RXQ; 2036 iaq->nirq += n10g * nofldrxq10g; 2037 #ifdef DEV_NETMAP 2038 iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g); 2039 #endif 2040 } 2041 if (nrxq1g >= nofldrxq1g) { 2042 iaq->intr_flags_1g = INTR_RXQ; 2043 iaq->nirq += n1g * nrxq1g; 2044 #ifdef DEV_NETMAP 2045 iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g); 2046 #endif 2047 } else { 2048 iaq->intr_flags_1g = INTR_OFLD_RXQ; 2049 iaq->nirq += n1g * nofldrxq1g; 2050 #ifdef DEV_NETMAP 2051 iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g); 2052 #endif 2053 } 2054 if (iaq->nirq <= navail && 2055 (itype != INTR_MSI || powerof2(iaq->nirq))) 2056 goto allocate; 2057 2058 /* 2059 * Next best option: an interrupt vector for errors, one for the 2060 * firmware event queue, and at least one per port. At this 2061 * point we know we'll have to downsize nrxq and/or nofldrxq 2062 * and/or nnmrxq to fit what's available to us. 2063 */ 2064 iaq->nirq = T4_EXTRA_INTR; 2065 iaq->nirq += n10g + n1g; 2066 if (iaq->nirq <= navail) { 2067 int leftover = navail - iaq->nirq; 2068 2069 if (n10g > 0) { 2070 int target = max(nrxq10g, nofldrxq10g); 2071 2072 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ? 2073 INTR_RXQ : INTR_OFLD_RXQ; 2074 2075 n = 1; 2076 while (n < target && leftover >= n10g) { 2077 leftover -= n10g; 2078 iaq->nirq += n10g; 2079 n++; 2080 } 2081 iaq->nrxq10g = min(n, nrxq10g); 2082 #ifdef TCP_OFFLOAD 2083 iaq->nofldrxq10g = min(n, nofldrxq10g); 2084 #endif 2085 #ifdef DEV_NETMAP 2086 iaq->nnmrxq10g = min(n, nnmrxq10g); 2087 #endif 2088 } 2089 2090 if (n1g > 0) { 2091 int target = max(nrxq1g, nofldrxq1g); 2092 2093 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ? 2094 INTR_RXQ : INTR_OFLD_RXQ; 2095 2096 n = 1; 2097 while (n < target && leftover >= n1g) { 2098 leftover -= n1g; 2099 iaq->nirq += n1g; 2100 n++; 2101 } 2102 iaq->nrxq1g = min(n, nrxq1g); 2103 #ifdef TCP_OFFLOAD 2104 iaq->nofldrxq1g = min(n, nofldrxq1g); 2105 #endif 2106 #ifdef DEV_NETMAP 2107 iaq->nnmrxq1g = min(n, nnmrxq1g); 2108 #endif 2109 } 2110 2111 if (itype != INTR_MSI || powerof2(iaq->nirq)) 2112 goto allocate; 2113 } 2114 2115 /* 2116 * Least desirable option: one interrupt vector for everything. 2117 */ 2118 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 2119 iaq->intr_flags_10g = iaq->intr_flags_1g = 0; 2120 #ifdef TCP_OFFLOAD 2121 if (is_offload(sc)) 2122 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 2123 #endif 2124 #ifdef DEV_NETMAP 2125 iaq->nnmrxq10g = iaq->nnmrxq1g = 1; 2126 #endif 2127 2128 allocate: 2129 navail = iaq->nirq; 2130 rc = 0; 2131 if (itype == INTR_MSIX) 2132 rc = pci_alloc_msix(sc->dev, &navail); 2133 else if (itype == INTR_MSI) 2134 rc = pci_alloc_msi(sc->dev, &navail); 2135 2136 if (rc == 0) { 2137 if (navail == iaq->nirq) 2138 return (0); 2139 2140 /* 2141 * Didn't get the number requested. Use whatever number 2142 * the kernel is willing to allocate (it's in navail). 2143 */ 2144 device_printf(sc->dev, "fewer vectors than requested, " 2145 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 2146 itype, iaq->nirq, navail); 2147 pci_release_msi(sc->dev); 2148 goto restart; 2149 } 2150 2151 device_printf(sc->dev, 2152 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 2153 itype, rc, iaq->nirq, navail); 2154 } 2155 2156 device_printf(sc->dev, 2157 "failed to find a usable interrupt type. " 2158 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 2159 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 2160 2161 return (ENXIO); 2162 } 2163 2164 #define FW_VERSION(chip) ( \ 2165 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ 2166 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ 2167 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ 2168 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) 2169 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) 2170 2171 struct fw_info { 2172 uint8_t chip; 2173 char *kld_name; 2174 char *fw_mod_name; 2175 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 2176 } fw_info[] = { 2177 { 2178 .chip = CHELSIO_T4, 2179 .kld_name = "t4fw_cfg", 2180 .fw_mod_name = "t4fw", 2181 .fw_hdr = { 2182 .chip = FW_HDR_CHIP_T4, 2183 .fw_ver = htobe32_const(FW_VERSION(T4)), 2184 .intfver_nic = FW_INTFVER(T4, NIC), 2185 .intfver_vnic = FW_INTFVER(T4, VNIC), 2186 .intfver_ofld = FW_INTFVER(T4, OFLD), 2187 .intfver_ri = FW_INTFVER(T4, RI), 2188 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 2189 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 2190 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 2191 .intfver_fcoe = FW_INTFVER(T4, FCOE), 2192 }, 2193 }, { 2194 .chip = CHELSIO_T5, 2195 .kld_name = "t5fw_cfg", 2196 .fw_mod_name = "t5fw", 2197 .fw_hdr = { 2198 .chip = FW_HDR_CHIP_T5, 2199 .fw_ver = htobe32_const(FW_VERSION(T5)), 2200 .intfver_nic = FW_INTFVER(T5, NIC), 2201 .intfver_vnic = FW_INTFVER(T5, VNIC), 2202 .intfver_ofld = FW_INTFVER(T5, OFLD), 2203 .intfver_ri = FW_INTFVER(T5, RI), 2204 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 2205 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 2206 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 2207 .intfver_fcoe = FW_INTFVER(T5, FCOE), 2208 }, 2209 } 2210 }; 2211 2212 static struct fw_info * 2213 find_fw_info(int chip) 2214 { 2215 int i; 2216 2217 for (i = 0; i < nitems(fw_info); i++) { 2218 if (fw_info[i].chip == chip) 2219 return (&fw_info[i]); 2220 } 2221 return (NULL); 2222 } 2223 2224 /* 2225 * Is the given firmware API compatible with the one the driver was compiled 2226 * with? 2227 */ 2228 static int 2229 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 2230 { 2231 2232 /* short circuit if it's the exact same firmware version */ 2233 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 2234 return (1); 2235 2236 /* 2237 * XXX: Is this too conservative? Perhaps I should limit this to the 2238 * features that are supported in the driver. 2239 */ 2240 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 2241 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 2242 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 2243 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 2244 return (1); 2245 #undef SAME_INTF 2246 2247 return (0); 2248 } 2249 2250 /* 2251 * The firmware in the KLD is usable, but should it be installed? This routine 2252 * explains itself in detail if it indicates the KLD firmware should be 2253 * installed. 2254 */ 2255 static int 2256 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 2257 { 2258 const char *reason; 2259 2260 if (!card_fw_usable) { 2261 reason = "incompatible or unusable"; 2262 goto install; 2263 } 2264 2265 if (k > c) { 2266 reason = "older than the version bundled with this driver"; 2267 goto install; 2268 } 2269 2270 if (t4_fw_install == 2 && k != c) { 2271 reason = "different than the version bundled with this driver"; 2272 goto install; 2273 } 2274 2275 return (0); 2276 2277 install: 2278 if (t4_fw_install == 0) { 2279 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2280 "but the driver is prohibited from installing a different " 2281 "firmware on the card.\n", 2282 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2283 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 2284 2285 return (0); 2286 } 2287 2288 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2289 "installing firmware %u.%u.%u.%u on card.\n", 2290 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2291 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 2292 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2293 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2294 2295 return (1); 2296 } 2297 /* 2298 * Establish contact with the firmware and determine if we are the master driver 2299 * or not, and whether we are responsible for chip initialization. 2300 */ 2301 static int 2302 prep_firmware(struct adapter *sc) 2303 { 2304 const struct firmware *fw = NULL, *default_cfg; 2305 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 2306 enum dev_state state; 2307 struct fw_info *fw_info; 2308 struct fw_hdr *card_fw; /* fw on the card */ 2309 const struct fw_hdr *kld_fw; /* fw in the KLD */ 2310 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 2311 against */ 2312 2313 /* Contact firmware. */ 2314 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 2315 if (rc < 0 || state == DEV_STATE_ERR) { 2316 rc = -rc; 2317 device_printf(sc->dev, 2318 "failed to connect to the firmware: %d, %d.\n", rc, state); 2319 return (rc); 2320 } 2321 pf = rc; 2322 if (pf == sc->mbox) 2323 sc->flags |= MASTER_PF; 2324 else if (state == DEV_STATE_UNINIT) { 2325 /* 2326 * We didn't get to be the master so we definitely won't be 2327 * configuring the chip. It's a bug if someone else hasn't 2328 * configured it already. 2329 */ 2330 device_printf(sc->dev, "couldn't be master(%d), " 2331 "device not already initialized either(%d).\n", rc, state); 2332 return (EDOOFUS); 2333 } 2334 2335 /* This is the firmware whose headers the driver was compiled against */ 2336 fw_info = find_fw_info(chip_id(sc)); 2337 if (fw_info == NULL) { 2338 device_printf(sc->dev, 2339 "unable to look up firmware information for chip %d.\n", 2340 chip_id(sc)); 2341 return (EINVAL); 2342 } 2343 drv_fw = &fw_info->fw_hdr; 2344 2345 /* 2346 * The firmware KLD contains many modules. The KLD name is also the 2347 * name of the module that contains the default config file. 2348 */ 2349 default_cfg = firmware_get(fw_info->kld_name); 2350 2351 /* Read the header of the firmware on the card */ 2352 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 2353 rc = -t4_read_flash(sc, FLASH_FW_START, 2354 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 2355 if (rc == 0) 2356 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 2357 else { 2358 device_printf(sc->dev, 2359 "Unable to read card's firmware header: %d\n", rc); 2360 card_fw_usable = 0; 2361 } 2362 2363 /* This is the firmware in the KLD */ 2364 fw = firmware_get(fw_info->fw_mod_name); 2365 if (fw != NULL) { 2366 kld_fw = (const void *)fw->data; 2367 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 2368 } else { 2369 kld_fw = NULL; 2370 kld_fw_usable = 0; 2371 } 2372 2373 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2374 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) { 2375 /* 2376 * Common case: the firmware on the card is an exact match and 2377 * the KLD is an exact match too, or the KLD is 2378 * absent/incompatible. Note that t4_fw_install = 2 is ignored 2379 * here -- use cxgbetool loadfw if you want to reinstall the 2380 * same firmware as the one on the card. 2381 */ 2382 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2383 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2384 be32toh(card_fw->fw_ver))) { 2385 2386 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 2387 if (rc != 0) { 2388 device_printf(sc->dev, 2389 "failed to install firmware: %d\n", rc); 2390 goto done; 2391 } 2392 2393 /* Installed successfully, update the cached header too. */ 2394 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2395 card_fw_usable = 1; 2396 need_fw_reset = 0; /* already reset as part of load_fw */ 2397 } 2398 2399 if (!card_fw_usable) { 2400 uint32_t d, c, k; 2401 2402 d = ntohl(drv_fw->fw_ver); 2403 c = ntohl(card_fw->fw_ver); 2404 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2405 2406 device_printf(sc->dev, "Cannot find a usable firmware: " 2407 "fw_install %d, chip state %d, " 2408 "driver compiled with %d.%d.%d.%d, " 2409 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2410 t4_fw_install, state, 2411 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2412 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2413 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2414 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2415 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2416 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2417 rc = EINVAL; 2418 goto done; 2419 } 2420 2421 /* We're using whatever's on the card and it's known to be good. */ 2422 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2423 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2424 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2425 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2426 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2427 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2428 t4_get_tp_version(sc, &sc->params.tp_vers); 2429 2430 /* Reset device */ 2431 if (need_fw_reset && 2432 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2433 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2434 if (rc != ETIMEDOUT && rc != EIO) 2435 t4_fw_bye(sc, sc->mbox); 2436 goto done; 2437 } 2438 sc->flags |= FW_OK; 2439 2440 rc = get_params__pre_init(sc); 2441 if (rc != 0) 2442 goto done; /* error message displayed already */ 2443 2444 /* Partition adapter resources as specified in the config file. */ 2445 if (state == DEV_STATE_UNINIT) { 2446 2447 KASSERT(sc->flags & MASTER_PF, 2448 ("%s: trying to change chip settings when not master.", 2449 __func__)); 2450 2451 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2452 if (rc != 0) 2453 goto done; /* error message displayed already */ 2454 2455 t4_tweak_chip_settings(sc); 2456 2457 /* get basic stuff going */ 2458 rc = -t4_fw_initialize(sc, sc->mbox); 2459 if (rc != 0) { 2460 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2461 goto done; 2462 } 2463 } else { 2464 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2465 sc->cfcsum = 0; 2466 } 2467 2468 done: 2469 free(card_fw, M_CXGBE); 2470 if (fw != NULL) 2471 firmware_put(fw, FIRMWARE_UNLOAD); 2472 if (default_cfg != NULL) 2473 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2474 2475 return (rc); 2476 } 2477 2478 #define FW_PARAM_DEV(param) \ 2479 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2480 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2481 #define FW_PARAM_PFVF(param) \ 2482 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2483 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2484 2485 /* 2486 * Partition chip resources for use between various PFs, VFs, etc. 2487 */ 2488 static int 2489 partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2490 const char *name_prefix) 2491 { 2492 const struct firmware *cfg = NULL; 2493 int rc = 0; 2494 struct fw_caps_config_cmd caps; 2495 uint32_t mtype, moff, finicsum, cfcsum; 2496 2497 /* 2498 * Figure out what configuration file to use. Pick the default config 2499 * file for the card if the user hasn't specified one explicitly. 2500 */ 2501 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2502 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2503 /* Card specific overrides go here. */ 2504 if (pci_get_device(sc->dev) == 0x440a) 2505 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2506 if (is_fpga(sc)) 2507 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2508 } 2509 2510 /* 2511 * We need to load another module if the profile is anything except 2512 * "default" or "flash". 2513 */ 2514 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2515 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2516 char s[32]; 2517 2518 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2519 cfg = firmware_get(s); 2520 if (cfg == NULL) { 2521 if (default_cfg != NULL) { 2522 device_printf(sc->dev, 2523 "unable to load module \"%s\" for " 2524 "configuration profile \"%s\", will use " 2525 "the default config file instead.\n", 2526 s, sc->cfg_file); 2527 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2528 "%s", DEFAULT_CF); 2529 } else { 2530 device_printf(sc->dev, 2531 "unable to load module \"%s\" for " 2532 "configuration profile \"%s\", will use " 2533 "the config file on the card's flash " 2534 "instead.\n", s, sc->cfg_file); 2535 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2536 "%s", FLASH_CF); 2537 } 2538 } 2539 } 2540 2541 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2542 default_cfg == NULL) { 2543 device_printf(sc->dev, 2544 "default config file not available, will use the config " 2545 "file on the card's flash instead.\n"); 2546 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2547 } 2548 2549 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2550 u_int cflen, i, n; 2551 const uint32_t *cfdata; 2552 uint32_t param, val, addr, off, mw_base, mw_aperture; 2553 2554 KASSERT(cfg != NULL || default_cfg != NULL, 2555 ("%s: no config to upload", __func__)); 2556 2557 /* 2558 * Ask the firmware where it wants us to upload the config file. 2559 */ 2560 param = FW_PARAM_DEV(CF); 2561 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2562 if (rc != 0) { 2563 /* No support for config file? Shouldn't happen. */ 2564 device_printf(sc->dev, 2565 "failed to query config file location: %d.\n", rc); 2566 goto done; 2567 } 2568 mtype = G_FW_PARAMS_PARAM_Y(val); 2569 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2570 2571 /* 2572 * XXX: sheer laziness. We deliberately added 4 bytes of 2573 * useless stuffing/comments at the end of the config file so 2574 * it's ok to simply throw away the last remaining bytes when 2575 * the config file is not an exact multiple of 4. This also 2576 * helps with the validate_mt_off_len check. 2577 */ 2578 if (cfg != NULL) { 2579 cflen = cfg->datasize & ~3; 2580 cfdata = cfg->data; 2581 } else { 2582 cflen = default_cfg->datasize & ~3; 2583 cfdata = default_cfg->data; 2584 } 2585 2586 if (cflen > FLASH_CFG_MAX_SIZE) { 2587 device_printf(sc->dev, 2588 "config file too long (%d, max allowed is %d). " 2589 "Will try to use the config on the card, if any.\n", 2590 cflen, FLASH_CFG_MAX_SIZE); 2591 goto use_config_on_flash; 2592 } 2593 2594 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2595 if (rc != 0) { 2596 device_printf(sc->dev, 2597 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2598 "Will try to use the config on the card, if any.\n", 2599 __func__, mtype, moff, cflen, rc); 2600 goto use_config_on_flash; 2601 } 2602 2603 memwin_info(sc, 2, &mw_base, &mw_aperture); 2604 while (cflen) { 2605 off = position_memwin(sc, 2, addr); 2606 n = min(cflen, mw_aperture - off); 2607 for (i = 0; i < n; i += 4) 2608 t4_write_reg(sc, mw_base + off + i, *cfdata++); 2609 cflen -= n; 2610 addr += n; 2611 } 2612 } else { 2613 use_config_on_flash: 2614 mtype = FW_MEMTYPE_FLASH; 2615 moff = t4_flash_cfg_addr(sc); 2616 } 2617 2618 bzero(&caps, sizeof(caps)); 2619 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2620 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2621 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2622 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2623 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2624 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2625 if (rc != 0) { 2626 device_printf(sc->dev, 2627 "failed to pre-process config file: %d " 2628 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2629 goto done; 2630 } 2631 2632 finicsum = be32toh(caps.finicsum); 2633 cfcsum = be32toh(caps.cfcsum); 2634 if (finicsum != cfcsum) { 2635 device_printf(sc->dev, 2636 "WARNING: config file checksum mismatch: %08x %08x\n", 2637 finicsum, cfcsum); 2638 } 2639 sc->cfcsum = cfcsum; 2640 2641 #define LIMIT_CAPS(x) do { \ 2642 caps.x &= htobe16(t4_##x##_allowed); \ 2643 } while (0) 2644 2645 /* 2646 * Let the firmware know what features will (not) be used so it can tune 2647 * things accordingly. 2648 */ 2649 LIMIT_CAPS(linkcaps); 2650 LIMIT_CAPS(niccaps); 2651 LIMIT_CAPS(toecaps); 2652 LIMIT_CAPS(rdmacaps); 2653 LIMIT_CAPS(iscsicaps); 2654 LIMIT_CAPS(fcoecaps); 2655 #undef LIMIT_CAPS 2656 2657 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2658 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2659 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2660 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2661 if (rc != 0) { 2662 device_printf(sc->dev, 2663 "failed to process config file: %d.\n", rc); 2664 } 2665 done: 2666 if (cfg != NULL) 2667 firmware_put(cfg, FIRMWARE_UNLOAD); 2668 return (rc); 2669 } 2670 2671 /* 2672 * Retrieve parameters that are needed (or nice to have) very early. 2673 */ 2674 static int 2675 get_params__pre_init(struct adapter *sc) 2676 { 2677 int rc; 2678 uint32_t param[2], val[2]; 2679 struct fw_devlog_cmd cmd; 2680 struct devlog_params *dlog = &sc->params.devlog; 2681 2682 param[0] = FW_PARAM_DEV(PORTVEC); 2683 param[1] = FW_PARAM_DEV(CCLK); 2684 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2685 if (rc != 0) { 2686 device_printf(sc->dev, 2687 "failed to query parameters (pre_init): %d.\n", rc); 2688 return (rc); 2689 } 2690 2691 sc->params.portvec = val[0]; 2692 sc->params.nports = bitcount32(val[0]); 2693 sc->params.vpd.cclk = val[1]; 2694 2695 /* Read device log parameters. */ 2696 bzero(&cmd, sizeof(cmd)); 2697 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2698 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2699 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2700 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2701 if (rc != 0) { 2702 device_printf(sc->dev, 2703 "failed to get devlog parameters: %d.\n", rc); 2704 bzero(dlog, sizeof (*dlog)); 2705 rc = 0; /* devlog isn't critical for device operation */ 2706 } else { 2707 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2708 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2709 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2710 dlog->size = be32toh(cmd.memsize_devlog); 2711 } 2712 2713 return (rc); 2714 } 2715 2716 /* 2717 * Retrieve various parameters that are of interest to the driver. The device 2718 * has been initialized by the firmware at this point. 2719 */ 2720 static int 2721 get_params__post_init(struct adapter *sc) 2722 { 2723 int rc; 2724 uint32_t param[7], val[7]; 2725 struct fw_caps_config_cmd caps; 2726 2727 param[0] = FW_PARAM_PFVF(IQFLINT_START); 2728 param[1] = FW_PARAM_PFVF(EQ_START); 2729 param[2] = FW_PARAM_PFVF(FILTER_START); 2730 param[3] = FW_PARAM_PFVF(FILTER_END); 2731 param[4] = FW_PARAM_PFVF(L2T_START); 2732 param[5] = FW_PARAM_PFVF(L2T_END); 2733 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2734 if (rc != 0) { 2735 device_printf(sc->dev, 2736 "failed to query parameters (post_init): %d.\n", rc); 2737 return (rc); 2738 } 2739 2740 sc->sge.iq_start = val[0]; 2741 sc->sge.eq_start = val[1]; 2742 sc->tids.ftid_base = val[2]; 2743 sc->tids.nftids = val[3] - val[2] + 1; 2744 sc->params.ftid_min = val[2]; 2745 sc->params.ftid_max = val[3]; 2746 sc->vres.l2t.start = val[4]; 2747 sc->vres.l2t.size = val[5] - val[4] + 1; 2748 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2749 ("%s: L2 table size (%u) larger than expected (%u)", 2750 __func__, sc->vres.l2t.size, L2T_SIZE)); 2751 2752 /* get capabilites */ 2753 bzero(&caps, sizeof(caps)); 2754 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2755 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2756 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2757 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2758 if (rc != 0) { 2759 device_printf(sc->dev, 2760 "failed to get card capabilities: %d.\n", rc); 2761 return (rc); 2762 } 2763 2764 #define READ_CAPS(x) do { \ 2765 sc->x = htobe16(caps.x); \ 2766 } while (0) 2767 READ_CAPS(linkcaps); 2768 READ_CAPS(niccaps); 2769 READ_CAPS(toecaps); 2770 READ_CAPS(rdmacaps); 2771 READ_CAPS(iscsicaps); 2772 READ_CAPS(fcoecaps); 2773 2774 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { 2775 param[0] = FW_PARAM_PFVF(ETHOFLD_START); 2776 param[1] = FW_PARAM_PFVF(ETHOFLD_END); 2777 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2778 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); 2779 if (rc != 0) { 2780 device_printf(sc->dev, 2781 "failed to query NIC parameters: %d.\n", rc); 2782 return (rc); 2783 } 2784 sc->tids.etid_base = val[0]; 2785 sc->params.etid_min = val[0]; 2786 sc->tids.netids = val[1] - val[0] + 1; 2787 sc->params.netids = sc->tids.netids; 2788 sc->params.eo_wr_cred = val[2]; 2789 sc->params.ethoffload = 1; 2790 } 2791 2792 if (sc->toecaps) { 2793 /* query offload-related parameters */ 2794 param[0] = FW_PARAM_DEV(NTID); 2795 param[1] = FW_PARAM_PFVF(SERVER_START); 2796 param[2] = FW_PARAM_PFVF(SERVER_END); 2797 param[3] = FW_PARAM_PFVF(TDDP_START); 2798 param[4] = FW_PARAM_PFVF(TDDP_END); 2799 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2800 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2801 if (rc != 0) { 2802 device_printf(sc->dev, 2803 "failed to query TOE parameters: %d.\n", rc); 2804 return (rc); 2805 } 2806 sc->tids.ntids = val[0]; 2807 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 2808 sc->tids.stid_base = val[1]; 2809 sc->tids.nstids = val[2] - val[1] + 1; 2810 sc->vres.ddp.start = val[3]; 2811 sc->vres.ddp.size = val[4] - val[3] + 1; 2812 sc->params.ofldq_wr_cred = val[5]; 2813 sc->params.offload = 1; 2814 } 2815 if (sc->rdmacaps) { 2816 param[0] = FW_PARAM_PFVF(STAG_START); 2817 param[1] = FW_PARAM_PFVF(STAG_END); 2818 param[2] = FW_PARAM_PFVF(RQ_START); 2819 param[3] = FW_PARAM_PFVF(RQ_END); 2820 param[4] = FW_PARAM_PFVF(PBL_START); 2821 param[5] = FW_PARAM_PFVF(PBL_END); 2822 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2823 if (rc != 0) { 2824 device_printf(sc->dev, 2825 "failed to query RDMA parameters(1): %d.\n", rc); 2826 return (rc); 2827 } 2828 sc->vres.stag.start = val[0]; 2829 sc->vres.stag.size = val[1] - val[0] + 1; 2830 sc->vres.rq.start = val[2]; 2831 sc->vres.rq.size = val[3] - val[2] + 1; 2832 sc->vres.pbl.start = val[4]; 2833 sc->vres.pbl.size = val[5] - val[4] + 1; 2834 2835 param[0] = FW_PARAM_PFVF(SQRQ_START); 2836 param[1] = FW_PARAM_PFVF(SQRQ_END); 2837 param[2] = FW_PARAM_PFVF(CQ_START); 2838 param[3] = FW_PARAM_PFVF(CQ_END); 2839 param[4] = FW_PARAM_PFVF(OCQ_START); 2840 param[5] = FW_PARAM_PFVF(OCQ_END); 2841 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2842 if (rc != 0) { 2843 device_printf(sc->dev, 2844 "failed to query RDMA parameters(2): %d.\n", rc); 2845 return (rc); 2846 } 2847 sc->vres.qp.start = val[0]; 2848 sc->vres.qp.size = val[1] - val[0] + 1; 2849 sc->vres.cq.start = val[2]; 2850 sc->vres.cq.size = val[3] - val[2] + 1; 2851 sc->vres.ocq.start = val[4]; 2852 sc->vres.ocq.size = val[5] - val[4] + 1; 2853 } 2854 if (sc->iscsicaps) { 2855 param[0] = FW_PARAM_PFVF(ISCSI_START); 2856 param[1] = FW_PARAM_PFVF(ISCSI_END); 2857 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2858 if (rc != 0) { 2859 device_printf(sc->dev, 2860 "failed to query iSCSI parameters: %d.\n", rc); 2861 return (rc); 2862 } 2863 sc->vres.iscsi.start = val[0]; 2864 sc->vres.iscsi.size = val[1] - val[0] + 1; 2865 } 2866 2867 /* 2868 * We've got the params we wanted to query via the firmware. Now grab 2869 * some others directly from the chip. 2870 */ 2871 rc = t4_read_chip_settings(sc); 2872 2873 return (rc); 2874 } 2875 2876 static int 2877 set_params__post_init(struct adapter *sc) 2878 { 2879 uint32_t param, val; 2880 2881 /* ask for encapsulated CPLs */ 2882 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 2883 val = 1; 2884 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2885 2886 return (0); 2887 } 2888 2889 #undef FW_PARAM_PFVF 2890 #undef FW_PARAM_DEV 2891 2892 static void 2893 t4_set_desc(struct adapter *sc) 2894 { 2895 char buf[128]; 2896 struct adapter_params *p = &sc->params; 2897 2898 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, " 2899 "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "", 2900 chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec); 2901 2902 device_set_desc_copy(sc->dev, buf); 2903 } 2904 2905 static void 2906 build_medialist(struct port_info *pi, struct ifmedia *media) 2907 { 2908 int data, m; 2909 2910 PORT_LOCK(pi); 2911 2912 ifmedia_removeall(media); 2913 2914 m = IFM_ETHER | IFM_FDX; 2915 data = (pi->port_type << 8) | pi->mod_type; 2916 2917 switch(pi->port_type) { 2918 case FW_PORT_TYPE_BT_XFI: 2919 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2920 break; 2921 2922 case FW_PORT_TYPE_BT_XAUI: 2923 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2924 /* fall through */ 2925 2926 case FW_PORT_TYPE_BT_SGMII: 2927 ifmedia_add(media, m | IFM_1000_T, data, NULL); 2928 ifmedia_add(media, m | IFM_100_TX, data, NULL); 2929 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL); 2930 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2931 break; 2932 2933 case FW_PORT_TYPE_CX4: 2934 ifmedia_add(media, m | IFM_10G_CX4, data, NULL); 2935 ifmedia_set(media, m | IFM_10G_CX4); 2936 break; 2937 2938 case FW_PORT_TYPE_QSFP_10G: 2939 case FW_PORT_TYPE_SFP: 2940 case FW_PORT_TYPE_FIBER_XFI: 2941 case FW_PORT_TYPE_FIBER_XAUI: 2942 switch (pi->mod_type) { 2943 2944 case FW_PORT_MOD_TYPE_LR: 2945 ifmedia_add(media, m | IFM_10G_LR, data, NULL); 2946 ifmedia_set(media, m | IFM_10G_LR); 2947 break; 2948 2949 case FW_PORT_MOD_TYPE_SR: 2950 ifmedia_add(media, m | IFM_10G_SR, data, NULL); 2951 ifmedia_set(media, m | IFM_10G_SR); 2952 break; 2953 2954 case FW_PORT_MOD_TYPE_LRM: 2955 ifmedia_add(media, m | IFM_10G_LRM, data, NULL); 2956 ifmedia_set(media, m | IFM_10G_LRM); 2957 break; 2958 2959 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2960 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2961 ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL); 2962 ifmedia_set(media, m | IFM_10G_TWINAX); 2963 break; 2964 2965 case FW_PORT_MOD_TYPE_NONE: 2966 m &= ~IFM_FDX; 2967 ifmedia_add(media, m | IFM_NONE, data, NULL); 2968 ifmedia_set(media, m | IFM_NONE); 2969 break; 2970 2971 case FW_PORT_MOD_TYPE_NA: 2972 case FW_PORT_MOD_TYPE_ER: 2973 default: 2974 device_printf(pi->dev, 2975 "unknown port_type (%d), mod_type (%d)\n", 2976 pi->port_type, pi->mod_type); 2977 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2978 ifmedia_set(media, m | IFM_UNKNOWN); 2979 break; 2980 } 2981 break; 2982 2983 case FW_PORT_TYPE_QSFP: 2984 switch (pi->mod_type) { 2985 2986 case FW_PORT_MOD_TYPE_LR: 2987 ifmedia_add(media, m | IFM_40G_LR4, data, NULL); 2988 ifmedia_set(media, m | IFM_40G_LR4); 2989 break; 2990 2991 case FW_PORT_MOD_TYPE_SR: 2992 ifmedia_add(media, m | IFM_40G_SR4, data, NULL); 2993 ifmedia_set(media, m | IFM_40G_SR4); 2994 break; 2995 2996 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2997 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2998 ifmedia_add(media, m | IFM_40G_CR4, data, NULL); 2999 ifmedia_set(media, m | IFM_40G_CR4); 3000 break; 3001 3002 case FW_PORT_MOD_TYPE_NONE: 3003 m &= ~IFM_FDX; 3004 ifmedia_add(media, m | IFM_NONE, data, NULL); 3005 ifmedia_set(media, m | IFM_NONE); 3006 break; 3007 3008 default: 3009 device_printf(pi->dev, 3010 "unknown port_type (%d), mod_type (%d)\n", 3011 pi->port_type, pi->mod_type); 3012 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 3013 ifmedia_set(media, m | IFM_UNKNOWN); 3014 break; 3015 } 3016 break; 3017 3018 default: 3019 device_printf(pi->dev, 3020 "unknown port_type (%d), mod_type (%d)\n", pi->port_type, 3021 pi->mod_type); 3022 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 3023 ifmedia_set(media, m | IFM_UNKNOWN); 3024 break; 3025 } 3026 3027 PORT_UNLOCK(pi); 3028 } 3029 3030 #define FW_MAC_EXACT_CHUNK 7 3031 3032 /* 3033 * Program the port's XGMAC based on parameters in ifnet. The caller also 3034 * indicates which parameters should be programmed (the rest are left alone). 3035 */ 3036 int 3037 update_mac_settings(struct ifnet *ifp, int flags) 3038 { 3039 int rc = 0; 3040 struct port_info *pi = ifp->if_softc; 3041 struct adapter *sc = pi->adapter; 3042 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 3043 uint16_t viid = 0xffff; 3044 int16_t *xact_addr_filt = NULL; 3045 3046 ASSERT_SYNCHRONIZED_OP(sc); 3047 KASSERT(flags, ("%s: not told what to update.", __func__)); 3048 3049 if (ifp == pi->ifp) { 3050 viid = pi->viid; 3051 xact_addr_filt = &pi->xact_addr_filt; 3052 } 3053 #ifdef DEV_NETMAP 3054 else if (ifp == pi->nm_ifp) { 3055 viid = pi->nm_viid; 3056 xact_addr_filt = &pi->nm_xact_addr_filt; 3057 } 3058 #endif 3059 if (flags & XGMAC_MTU) 3060 mtu = ifp->if_mtu; 3061 3062 if (flags & XGMAC_PROMISC) 3063 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 3064 3065 if (flags & XGMAC_ALLMULTI) 3066 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 3067 3068 if (flags & XGMAC_VLANEX) 3069 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 3070 3071 if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) { 3072 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti, 3073 1, vlanex, false); 3074 if (rc) { 3075 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, 3076 rc); 3077 return (rc); 3078 } 3079 } 3080 3081 if (flags & XGMAC_UCADDR) { 3082 uint8_t ucaddr[ETHER_ADDR_LEN]; 3083 3084 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 3085 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr, 3086 true, true); 3087 if (rc < 0) { 3088 rc = -rc; 3089 if_printf(ifp, "change_mac failed: %d\n", rc); 3090 return (rc); 3091 } else { 3092 *xact_addr_filt = rc; 3093 rc = 0; 3094 } 3095 } 3096 3097 if (flags & XGMAC_MCADDRS) { 3098 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 3099 int del = 1; 3100 uint64_t hash = 0; 3101 struct ifmultiaddr *ifma; 3102 int i = 0, j; 3103 3104 if_maddr_rlock(ifp); 3105 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3106 if (ifma->ifma_addr->sa_family != AF_LINK) 3107 continue; 3108 mcaddr[i] = 3109 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 3110 MPASS(ETHER_IS_MULTICAST(mcaddr[i])); 3111 i++; 3112 3113 if (i == FW_MAC_EXACT_CHUNK) { 3114 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, 3115 i, mcaddr, NULL, &hash, 0); 3116 if (rc < 0) { 3117 rc = -rc; 3118 for (j = 0; j < i; j++) { 3119 if_printf(ifp, 3120 "failed to add mc address" 3121 " %02x:%02x:%02x:" 3122 "%02x:%02x:%02x rc=%d\n", 3123 mcaddr[j][0], mcaddr[j][1], 3124 mcaddr[j][2], mcaddr[j][3], 3125 mcaddr[j][4], mcaddr[j][5], 3126 rc); 3127 } 3128 goto mcfail; 3129 } 3130 del = 0; 3131 i = 0; 3132 } 3133 } 3134 if (i > 0) { 3135 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i, 3136 mcaddr, NULL, &hash, 0); 3137 if (rc < 0) { 3138 rc = -rc; 3139 for (j = 0; j < i; j++) { 3140 if_printf(ifp, 3141 "failed to add mc address" 3142 " %02x:%02x:%02x:" 3143 "%02x:%02x:%02x rc=%d\n", 3144 mcaddr[j][0], mcaddr[j][1], 3145 mcaddr[j][2], mcaddr[j][3], 3146 mcaddr[j][4], mcaddr[j][5], 3147 rc); 3148 } 3149 goto mcfail; 3150 } 3151 } 3152 3153 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0); 3154 if (rc != 0) 3155 if_printf(ifp, "failed to set mc address hash: %d", rc); 3156 mcfail: 3157 if_maddr_runlock(ifp); 3158 } 3159 3160 return (rc); 3161 } 3162 3163 int 3164 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags, 3165 char *wmesg) 3166 { 3167 int rc, pri; 3168 3169 #ifdef WITNESS 3170 /* the caller thinks it's ok to sleep, but is it really? */ 3171 if (flags & SLEEP_OK) 3172 pause("t4slptst", 1); 3173 #endif 3174 3175 if (INTR_OK) 3176 pri = PCATCH; 3177 else 3178 pri = 0; 3179 3180 ADAPTER_LOCK(sc); 3181 for (;;) { 3182 3183 if (pi && IS_DOOMED(pi)) { 3184 rc = ENXIO; 3185 goto done; 3186 } 3187 3188 if (!IS_BUSY(sc)) { 3189 rc = 0; 3190 break; 3191 } 3192 3193 if (!(flags & SLEEP_OK)) { 3194 rc = EBUSY; 3195 goto done; 3196 } 3197 3198 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 3199 rc = EINTR; 3200 goto done; 3201 } 3202 } 3203 3204 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 3205 SET_BUSY(sc); 3206 #ifdef INVARIANTS 3207 sc->last_op = wmesg; 3208 sc->last_op_thr = curthread; 3209 #endif 3210 3211 done: 3212 if (!(flags & HOLD_LOCK) || rc) 3213 ADAPTER_UNLOCK(sc); 3214 3215 return (rc); 3216 } 3217 3218 void 3219 end_synchronized_op(struct adapter *sc, int flags) 3220 { 3221 3222 if (flags & LOCK_HELD) 3223 ADAPTER_LOCK_ASSERT_OWNED(sc); 3224 else 3225 ADAPTER_LOCK(sc); 3226 3227 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 3228 CLR_BUSY(sc); 3229 wakeup(&sc->flags); 3230 ADAPTER_UNLOCK(sc); 3231 } 3232 3233 static int 3234 cxgbe_init_synchronized(struct port_info *pi) 3235 { 3236 struct adapter *sc = pi->adapter; 3237 struct ifnet *ifp = pi->ifp; 3238 int rc = 0; 3239 3240 ASSERT_SYNCHRONIZED_OP(sc); 3241 3242 if (isset(&sc->open_device_map, pi->port_id)) { 3243 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 3244 ("mismatch between open_device_map and if_drv_flags")); 3245 return (0); /* already running */ 3246 } 3247 3248 if (!(sc->flags & FULL_INIT_DONE) && 3249 ((rc = adapter_full_init(sc)) != 0)) 3250 return (rc); /* error message displayed already */ 3251 3252 if (!(pi->flags & PORT_INIT_DONE) && 3253 ((rc = port_full_init(pi)) != 0)) 3254 return (rc); /* error message displayed already */ 3255 3256 rc = update_mac_settings(ifp, XGMAC_ALL); 3257 if (rc) 3258 goto done; /* error message displayed already */ 3259 3260 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 3261 if (rc != 0) { 3262 if_printf(ifp, "enable_vi failed: %d\n", rc); 3263 goto done; 3264 } 3265 3266 /* 3267 * The first iq of the first port to come up is used for tracing. 3268 */ 3269 if (sc->traceq < 0) { 3270 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id; 3271 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : 3272 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | 3273 V_QUEUENUMBER(sc->traceq)); 3274 pi->flags |= HAS_TRACEQ; 3275 } 3276 3277 /* all ok */ 3278 setbit(&sc->open_device_map, pi->port_id); 3279 PORT_LOCK(pi); 3280 ifp->if_drv_flags |= IFF_DRV_RUNNING; 3281 PORT_UNLOCK(pi); 3282 3283 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 3284 done: 3285 if (rc != 0) 3286 cxgbe_uninit_synchronized(pi); 3287 3288 return (rc); 3289 } 3290 3291 /* 3292 * Idempotent. 3293 */ 3294 static int 3295 cxgbe_uninit_synchronized(struct port_info *pi) 3296 { 3297 struct adapter *sc = pi->adapter; 3298 struct ifnet *ifp = pi->ifp; 3299 int rc; 3300 3301 ASSERT_SYNCHRONIZED_OP(sc); 3302 3303 /* 3304 * Disable the VI so that all its data in either direction is discarded 3305 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 3306 * tick) intact as the TP can deliver negative advice or data that it's 3307 * holding in its RAM (for an offloaded connection) even after the VI is 3308 * disabled. 3309 */ 3310 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 3311 if (rc) { 3312 if_printf(ifp, "disable_vi failed: %d\n", rc); 3313 return (rc); 3314 } 3315 3316 clrbit(&sc->open_device_map, pi->port_id); 3317 PORT_LOCK(pi); 3318 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3319 PORT_UNLOCK(pi); 3320 3321 pi->link_cfg.link_ok = 0; 3322 pi->link_cfg.speed = 0; 3323 pi->linkdnrc = -1; 3324 t4_os_link_changed(sc, pi->port_id, 0, -1); 3325 3326 return (0); 3327 } 3328 3329 /* 3330 * It is ok for this function to fail midway and return right away. t4_detach 3331 * will walk the entire sc->irq list and clean up whatever is valid. 3332 */ 3333 static int 3334 setup_intr_handlers(struct adapter *sc) 3335 { 3336 int rc, rid, p, q; 3337 char s[8]; 3338 struct irq *irq; 3339 struct port_info *pi; 3340 struct sge_rxq *rxq; 3341 #ifdef TCP_OFFLOAD 3342 struct sge_ofld_rxq *ofld_rxq; 3343 #endif 3344 #ifdef DEV_NETMAP 3345 struct sge_nm_rxq *nm_rxq; 3346 #endif 3347 3348 /* 3349 * Setup interrupts. 3350 */ 3351 irq = &sc->irq[0]; 3352 rid = sc->intr_type == INTR_INTX ? 0 : 1; 3353 if (sc->intr_count == 1) 3354 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all")); 3355 3356 /* Multiple interrupts. */ 3357 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 3358 ("%s: too few intr.", __func__)); 3359 3360 /* The first one is always error intr */ 3361 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 3362 if (rc != 0) 3363 return (rc); 3364 irq++; 3365 rid++; 3366 3367 /* The second one is always the firmware event queue */ 3368 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt"); 3369 if (rc != 0) 3370 return (rc); 3371 irq++; 3372 rid++; 3373 3374 for_each_port(sc, p) { 3375 pi = sc->port[p]; 3376 3377 if (pi->flags & INTR_RXQ) { 3378 for_each_rxq(pi, q, rxq) { 3379 snprintf(s, sizeof(s), "%d.%d", p, q); 3380 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq, 3381 s); 3382 if (rc != 0) 3383 return (rc); 3384 irq++; 3385 rid++; 3386 } 3387 } 3388 #ifdef TCP_OFFLOAD 3389 if (pi->flags & INTR_OFLD_RXQ) { 3390 for_each_ofld_rxq(pi, q, ofld_rxq) { 3391 snprintf(s, sizeof(s), "%d,%d", p, q); 3392 rc = t4_alloc_irq(sc, irq, rid, t4_intr, 3393 ofld_rxq, s); 3394 if (rc != 0) 3395 return (rc); 3396 irq++; 3397 rid++; 3398 } 3399 } 3400 #endif 3401 #ifdef DEV_NETMAP 3402 if (pi->flags & INTR_NM_RXQ) { 3403 for_each_nm_rxq(pi, q, nm_rxq) { 3404 snprintf(s, sizeof(s), "%d-%d", p, q); 3405 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr, 3406 nm_rxq, s); 3407 if (rc != 0) 3408 return (rc); 3409 irq++; 3410 rid++; 3411 } 3412 } 3413 #endif 3414 } 3415 MPASS(irq == &sc->irq[sc->intr_count]); 3416 3417 return (0); 3418 } 3419 3420 int 3421 adapter_full_init(struct adapter *sc) 3422 { 3423 int rc, i; 3424 3425 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3426 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 3427 ("%s: FULL_INIT_DONE already", __func__)); 3428 3429 /* 3430 * queues that belong to the adapter (not any particular port). 3431 */ 3432 rc = t4_setup_adapter_queues(sc); 3433 if (rc != 0) 3434 goto done; 3435 3436 for (i = 0; i < nitems(sc->tq); i++) { 3437 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 3438 taskqueue_thread_enqueue, &sc->tq[i]); 3439 if (sc->tq[i] == NULL) { 3440 device_printf(sc->dev, 3441 "failed to allocate task queue %d\n", i); 3442 rc = ENOMEM; 3443 goto done; 3444 } 3445 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3446 device_get_nameunit(sc->dev), i); 3447 } 3448 3449 t4_intr_enable(sc); 3450 sc->flags |= FULL_INIT_DONE; 3451 done: 3452 if (rc != 0) 3453 adapter_full_uninit(sc); 3454 3455 return (rc); 3456 } 3457 3458 int 3459 adapter_full_uninit(struct adapter *sc) 3460 { 3461 int i; 3462 3463 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3464 3465 t4_teardown_adapter_queues(sc); 3466 3467 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3468 taskqueue_free(sc->tq[i]); 3469 sc->tq[i] = NULL; 3470 } 3471 3472 sc->flags &= ~FULL_INIT_DONE; 3473 3474 return (0); 3475 } 3476 3477 int 3478 port_full_init(struct port_info *pi) 3479 { 3480 struct adapter *sc = pi->adapter; 3481 struct ifnet *ifp = pi->ifp; 3482 uint16_t *rss; 3483 struct sge_rxq *rxq; 3484 int rc, i, j; 3485 3486 ASSERT_SYNCHRONIZED_OP(sc); 3487 KASSERT((pi->flags & PORT_INIT_DONE) == 0, 3488 ("%s: PORT_INIT_DONE already", __func__)); 3489 3490 sysctl_ctx_init(&pi->ctx); 3491 pi->flags |= PORT_SYSCTL_CTX; 3492 3493 /* 3494 * Allocate tx/rx/fl queues for this port. 3495 */ 3496 rc = t4_setup_port_queues(pi); 3497 if (rc != 0) 3498 goto done; /* error message displayed already */ 3499 3500 /* 3501 * Setup RSS for this port. Save a copy of the RSS table for later use. 3502 */ 3503 rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); 3504 for (i = 0; i < pi->rss_size;) { 3505 for_each_rxq(pi, j, rxq) { 3506 rss[i++] = rxq->iq.abs_id; 3507 if (i == pi->rss_size) 3508 break; 3509 } 3510 } 3511 3512 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss, 3513 pi->rss_size); 3514 if (rc != 0) { 3515 if_printf(ifp, "rss_config failed: %d\n", rc); 3516 goto done; 3517 } 3518 3519 pi->rss = rss; 3520 pi->flags |= PORT_INIT_DONE; 3521 done: 3522 if (rc != 0) 3523 port_full_uninit(pi); 3524 3525 return (rc); 3526 } 3527 3528 /* 3529 * Idempotent. 3530 */ 3531 int 3532 port_full_uninit(struct port_info *pi) 3533 { 3534 struct adapter *sc = pi->adapter; 3535 int i; 3536 struct sge_rxq *rxq; 3537 struct sge_txq *txq; 3538 #ifdef TCP_OFFLOAD 3539 struct sge_ofld_rxq *ofld_rxq; 3540 struct sge_wrq *ofld_txq; 3541 #endif 3542 3543 if (pi->flags & PORT_INIT_DONE) { 3544 3545 /* Need to quiesce queues. XXX: ctrl queues? */ 3546 3547 for_each_txq(pi, i, txq) { 3548 quiesce_eq(sc, &txq->eq); 3549 } 3550 3551 #ifdef TCP_OFFLOAD 3552 for_each_ofld_txq(pi, i, ofld_txq) { 3553 quiesce_eq(sc, &ofld_txq->eq); 3554 } 3555 #endif 3556 3557 for_each_rxq(pi, i, rxq) { 3558 quiesce_iq(sc, &rxq->iq); 3559 quiesce_fl(sc, &rxq->fl); 3560 } 3561 3562 #ifdef TCP_OFFLOAD 3563 for_each_ofld_rxq(pi, i, ofld_rxq) { 3564 quiesce_iq(sc, &ofld_rxq->iq); 3565 quiesce_fl(sc, &ofld_rxq->fl); 3566 } 3567 #endif 3568 free(pi->rss, M_CXGBE); 3569 } 3570 3571 t4_teardown_port_queues(pi); 3572 pi->flags &= ~PORT_INIT_DONE; 3573 3574 return (0); 3575 } 3576 3577 static void 3578 quiesce_eq(struct adapter *sc, struct sge_eq *eq) 3579 { 3580 EQ_LOCK(eq); 3581 eq->flags |= EQ_DOOMED; 3582 3583 /* 3584 * Wait for the response to a credit flush if one's 3585 * pending. 3586 */ 3587 while (eq->flags & EQ_CRFLUSHED) 3588 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0); 3589 EQ_UNLOCK(eq); 3590 3591 callout_drain(&eq->tx_callout); /* XXX: iffy */ 3592 pause("callout", 10); /* Still iffy */ 3593 3594 taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task); 3595 } 3596 3597 static void 3598 quiesce_iq(struct adapter *sc, struct sge_iq *iq) 3599 { 3600 (void) sc; /* unused */ 3601 3602 /* Synchronize with the interrupt handler */ 3603 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 3604 pause("iqfree", 1); 3605 } 3606 3607 static void 3608 quiesce_fl(struct adapter *sc, struct sge_fl *fl) 3609 { 3610 mtx_lock(&sc->sfl_lock); 3611 FL_LOCK(fl); 3612 fl->flags |= FL_DOOMED; 3613 FL_UNLOCK(fl); 3614 mtx_unlock(&sc->sfl_lock); 3615 3616 callout_drain(&sc->sfl_callout); 3617 KASSERT((fl->flags & FL_STARVING) == 0, 3618 ("%s: still starving", __func__)); 3619 } 3620 3621 static int 3622 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 3623 driver_intr_t *handler, void *arg, char *name) 3624 { 3625 int rc; 3626 3627 irq->rid = rid; 3628 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 3629 RF_SHAREABLE | RF_ACTIVE); 3630 if (irq->res == NULL) { 3631 device_printf(sc->dev, 3632 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 3633 return (ENOMEM); 3634 } 3635 3636 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 3637 NULL, handler, arg, &irq->tag); 3638 if (rc != 0) { 3639 device_printf(sc->dev, 3640 "failed to setup interrupt for rid %d, name %s: %d\n", 3641 rid, name, rc); 3642 } else if (name) 3643 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 3644 3645 return (rc); 3646 } 3647 3648 static int 3649 t4_free_irq(struct adapter *sc, struct irq *irq) 3650 { 3651 if (irq->tag) 3652 bus_teardown_intr(sc->dev, irq->res, irq->tag); 3653 if (irq->res) 3654 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 3655 3656 bzero(irq, sizeof(*irq)); 3657 3658 return (0); 3659 } 3660 3661 static void 3662 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 3663 unsigned int end) 3664 { 3665 uint32_t *p = (uint32_t *)(buf + start); 3666 3667 for ( ; start <= end; start += sizeof(uint32_t)) 3668 *p++ = t4_read_reg(sc, start); 3669 } 3670 3671 static void 3672 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 3673 { 3674 int i, n; 3675 const unsigned int *reg_ranges; 3676 static const unsigned int t4_reg_ranges[] = { 3677 0x1008, 0x1108, 3678 0x1180, 0x11b4, 3679 0x11fc, 0x123c, 3680 0x1300, 0x173c, 3681 0x1800, 0x18fc, 3682 0x3000, 0x30d8, 3683 0x30e0, 0x5924, 3684 0x5960, 0x59d4, 3685 0x5a00, 0x5af8, 3686 0x6000, 0x6098, 3687 0x6100, 0x6150, 3688 0x6200, 0x6208, 3689 0x6240, 0x6248, 3690 0x6280, 0x6338, 3691 0x6370, 0x638c, 3692 0x6400, 0x643c, 3693 0x6500, 0x6524, 3694 0x6a00, 0x6a38, 3695 0x6a60, 0x6a78, 3696 0x6b00, 0x6b84, 3697 0x6bf0, 0x6c84, 3698 0x6cf0, 0x6d84, 3699 0x6df0, 0x6e84, 3700 0x6ef0, 0x6f84, 3701 0x6ff0, 0x7084, 3702 0x70f0, 0x7184, 3703 0x71f0, 0x7284, 3704 0x72f0, 0x7384, 3705 0x73f0, 0x7450, 3706 0x7500, 0x7530, 3707 0x7600, 0x761c, 3708 0x7680, 0x76cc, 3709 0x7700, 0x7798, 3710 0x77c0, 0x77fc, 3711 0x7900, 0x79fc, 3712 0x7b00, 0x7c38, 3713 0x7d00, 0x7efc, 3714 0x8dc0, 0x8e1c, 3715 0x8e30, 0x8e78, 3716 0x8ea0, 0x8f6c, 3717 0x8fc0, 0x9074, 3718 0x90fc, 0x90fc, 3719 0x9400, 0x9458, 3720 0x9600, 0x96bc, 3721 0x9800, 0x9808, 3722 0x9820, 0x983c, 3723 0x9850, 0x9864, 3724 0x9c00, 0x9c6c, 3725 0x9c80, 0x9cec, 3726 0x9d00, 0x9d6c, 3727 0x9d80, 0x9dec, 3728 0x9e00, 0x9e6c, 3729 0x9e80, 0x9eec, 3730 0x9f00, 0x9f6c, 3731 0x9f80, 0x9fec, 3732 0xd004, 0xd03c, 3733 0xdfc0, 0xdfe0, 3734 0xe000, 0xea7c, 3735 0xf000, 0x11110, 3736 0x11118, 0x11190, 3737 0x19040, 0x1906c, 3738 0x19078, 0x19080, 3739 0x1908c, 0x19124, 3740 0x19150, 0x191b0, 3741 0x191d0, 0x191e8, 3742 0x19238, 0x1924c, 3743 0x193f8, 0x19474, 3744 0x19490, 0x194f8, 3745 0x19800, 0x19f30, 3746 0x1a000, 0x1a06c, 3747 0x1a0b0, 0x1a120, 3748 0x1a128, 0x1a138, 3749 0x1a190, 0x1a1c4, 3750 0x1a1fc, 0x1a1fc, 3751 0x1e040, 0x1e04c, 3752 0x1e284, 0x1e28c, 3753 0x1e2c0, 0x1e2c0, 3754 0x1e2e0, 0x1e2e0, 3755 0x1e300, 0x1e384, 3756 0x1e3c0, 0x1e3c8, 3757 0x1e440, 0x1e44c, 3758 0x1e684, 0x1e68c, 3759 0x1e6c0, 0x1e6c0, 3760 0x1e6e0, 0x1e6e0, 3761 0x1e700, 0x1e784, 3762 0x1e7c0, 0x1e7c8, 3763 0x1e840, 0x1e84c, 3764 0x1ea84, 0x1ea8c, 3765 0x1eac0, 0x1eac0, 3766 0x1eae0, 0x1eae0, 3767 0x1eb00, 0x1eb84, 3768 0x1ebc0, 0x1ebc8, 3769 0x1ec40, 0x1ec4c, 3770 0x1ee84, 0x1ee8c, 3771 0x1eec0, 0x1eec0, 3772 0x1eee0, 0x1eee0, 3773 0x1ef00, 0x1ef84, 3774 0x1efc0, 0x1efc8, 3775 0x1f040, 0x1f04c, 3776 0x1f284, 0x1f28c, 3777 0x1f2c0, 0x1f2c0, 3778 0x1f2e0, 0x1f2e0, 3779 0x1f300, 0x1f384, 3780 0x1f3c0, 0x1f3c8, 3781 0x1f440, 0x1f44c, 3782 0x1f684, 0x1f68c, 3783 0x1f6c0, 0x1f6c0, 3784 0x1f6e0, 0x1f6e0, 3785 0x1f700, 0x1f784, 3786 0x1f7c0, 0x1f7c8, 3787 0x1f840, 0x1f84c, 3788 0x1fa84, 0x1fa8c, 3789 0x1fac0, 0x1fac0, 3790 0x1fae0, 0x1fae0, 3791 0x1fb00, 0x1fb84, 3792 0x1fbc0, 0x1fbc8, 3793 0x1fc40, 0x1fc4c, 3794 0x1fe84, 0x1fe8c, 3795 0x1fec0, 0x1fec0, 3796 0x1fee0, 0x1fee0, 3797 0x1ff00, 0x1ff84, 3798 0x1ffc0, 0x1ffc8, 3799 0x20000, 0x2002c, 3800 0x20100, 0x2013c, 3801 0x20190, 0x201c8, 3802 0x20200, 0x20318, 3803 0x20400, 0x20528, 3804 0x20540, 0x20614, 3805 0x21000, 0x21040, 3806 0x2104c, 0x21060, 3807 0x210c0, 0x210ec, 3808 0x21200, 0x21268, 3809 0x21270, 0x21284, 3810 0x212fc, 0x21388, 3811 0x21400, 0x21404, 3812 0x21500, 0x21518, 3813 0x2152c, 0x2153c, 3814 0x21550, 0x21554, 3815 0x21600, 0x21600, 3816 0x21608, 0x21628, 3817 0x21630, 0x2163c, 3818 0x21700, 0x2171c, 3819 0x21780, 0x2178c, 3820 0x21800, 0x21c38, 3821 0x21c80, 0x21d7c, 3822 0x21e00, 0x21e04, 3823 0x22000, 0x2202c, 3824 0x22100, 0x2213c, 3825 0x22190, 0x221c8, 3826 0x22200, 0x22318, 3827 0x22400, 0x22528, 3828 0x22540, 0x22614, 3829 0x23000, 0x23040, 3830 0x2304c, 0x23060, 3831 0x230c0, 0x230ec, 3832 0x23200, 0x23268, 3833 0x23270, 0x23284, 3834 0x232fc, 0x23388, 3835 0x23400, 0x23404, 3836 0x23500, 0x23518, 3837 0x2352c, 0x2353c, 3838 0x23550, 0x23554, 3839 0x23600, 0x23600, 3840 0x23608, 0x23628, 3841 0x23630, 0x2363c, 3842 0x23700, 0x2371c, 3843 0x23780, 0x2378c, 3844 0x23800, 0x23c38, 3845 0x23c80, 0x23d7c, 3846 0x23e00, 0x23e04, 3847 0x24000, 0x2402c, 3848 0x24100, 0x2413c, 3849 0x24190, 0x241c8, 3850 0x24200, 0x24318, 3851 0x24400, 0x24528, 3852 0x24540, 0x24614, 3853 0x25000, 0x25040, 3854 0x2504c, 0x25060, 3855 0x250c0, 0x250ec, 3856 0x25200, 0x25268, 3857 0x25270, 0x25284, 3858 0x252fc, 0x25388, 3859 0x25400, 0x25404, 3860 0x25500, 0x25518, 3861 0x2552c, 0x2553c, 3862 0x25550, 0x25554, 3863 0x25600, 0x25600, 3864 0x25608, 0x25628, 3865 0x25630, 0x2563c, 3866 0x25700, 0x2571c, 3867 0x25780, 0x2578c, 3868 0x25800, 0x25c38, 3869 0x25c80, 0x25d7c, 3870 0x25e00, 0x25e04, 3871 0x26000, 0x2602c, 3872 0x26100, 0x2613c, 3873 0x26190, 0x261c8, 3874 0x26200, 0x26318, 3875 0x26400, 0x26528, 3876 0x26540, 0x26614, 3877 0x27000, 0x27040, 3878 0x2704c, 0x27060, 3879 0x270c0, 0x270ec, 3880 0x27200, 0x27268, 3881 0x27270, 0x27284, 3882 0x272fc, 0x27388, 3883 0x27400, 0x27404, 3884 0x27500, 0x27518, 3885 0x2752c, 0x2753c, 3886 0x27550, 0x27554, 3887 0x27600, 0x27600, 3888 0x27608, 0x27628, 3889 0x27630, 0x2763c, 3890 0x27700, 0x2771c, 3891 0x27780, 0x2778c, 3892 0x27800, 0x27c38, 3893 0x27c80, 0x27d7c, 3894 0x27e00, 0x27e04 3895 }; 3896 static const unsigned int t5_reg_ranges[] = { 3897 0x1008, 0x1148, 3898 0x1180, 0x11b4, 3899 0x11fc, 0x123c, 3900 0x1280, 0x173c, 3901 0x1800, 0x18fc, 3902 0x3000, 0x3028, 3903 0x3060, 0x30d8, 3904 0x30e0, 0x30fc, 3905 0x3140, 0x357c, 3906 0x35a8, 0x35cc, 3907 0x35ec, 0x35ec, 3908 0x3600, 0x5624, 3909 0x56cc, 0x575c, 3910 0x580c, 0x5814, 3911 0x5890, 0x58bc, 3912 0x5940, 0x59dc, 3913 0x59fc, 0x5a18, 3914 0x5a60, 0x5a9c, 3915 0x5b94, 0x5bfc, 3916 0x6000, 0x6040, 3917 0x6058, 0x614c, 3918 0x7700, 0x7798, 3919 0x77c0, 0x78fc, 3920 0x7b00, 0x7c54, 3921 0x7d00, 0x7efc, 3922 0x8dc0, 0x8de0, 3923 0x8df8, 0x8e84, 3924 0x8ea0, 0x8f84, 3925 0x8fc0, 0x90f8, 3926 0x9400, 0x9470, 3927 0x9600, 0x96f4, 3928 0x9800, 0x9808, 3929 0x9820, 0x983c, 3930 0x9850, 0x9864, 3931 0x9c00, 0x9c6c, 3932 0x9c80, 0x9cec, 3933 0x9d00, 0x9d6c, 3934 0x9d80, 0x9dec, 3935 0x9e00, 0x9e6c, 3936 0x9e80, 0x9eec, 3937 0x9f00, 0x9f6c, 3938 0x9f80, 0xa020, 3939 0xd004, 0xd03c, 3940 0xdfc0, 0xdfe0, 3941 0xe000, 0x11088, 3942 0x1109c, 0x11110, 3943 0x11118, 0x1117c, 3944 0x11190, 0x11204, 3945 0x19040, 0x1906c, 3946 0x19078, 0x19080, 3947 0x1908c, 0x19124, 3948 0x19150, 0x191b0, 3949 0x191d0, 0x191e8, 3950 0x19238, 0x19290, 3951 0x193f8, 0x19474, 3952 0x19490, 0x194cc, 3953 0x194f0, 0x194f8, 3954 0x19c00, 0x19c60, 3955 0x19c94, 0x19e10, 3956 0x19e50, 0x19f34, 3957 0x19f40, 0x19f50, 3958 0x19f90, 0x19fe4, 3959 0x1a000, 0x1a06c, 3960 0x1a0b0, 0x1a120, 3961 0x1a128, 0x1a138, 3962 0x1a190, 0x1a1c4, 3963 0x1a1fc, 0x1a1fc, 3964 0x1e008, 0x1e00c, 3965 0x1e040, 0x1e04c, 3966 0x1e284, 0x1e290, 3967 0x1e2c0, 0x1e2c0, 3968 0x1e2e0, 0x1e2e0, 3969 0x1e300, 0x1e384, 3970 0x1e3c0, 0x1e3c8, 3971 0x1e408, 0x1e40c, 3972 0x1e440, 0x1e44c, 3973 0x1e684, 0x1e690, 3974 0x1e6c0, 0x1e6c0, 3975 0x1e6e0, 0x1e6e0, 3976 0x1e700, 0x1e784, 3977 0x1e7c0, 0x1e7c8, 3978 0x1e808, 0x1e80c, 3979 0x1e840, 0x1e84c, 3980 0x1ea84, 0x1ea90, 3981 0x1eac0, 0x1eac0, 3982 0x1eae0, 0x1eae0, 3983 0x1eb00, 0x1eb84, 3984 0x1ebc0, 0x1ebc8, 3985 0x1ec08, 0x1ec0c, 3986 0x1ec40, 0x1ec4c, 3987 0x1ee84, 0x1ee90, 3988 0x1eec0, 0x1eec0, 3989 0x1eee0, 0x1eee0, 3990 0x1ef00, 0x1ef84, 3991 0x1efc0, 0x1efc8, 3992 0x1f008, 0x1f00c, 3993 0x1f040, 0x1f04c, 3994 0x1f284, 0x1f290, 3995 0x1f2c0, 0x1f2c0, 3996 0x1f2e0, 0x1f2e0, 3997 0x1f300, 0x1f384, 3998 0x1f3c0, 0x1f3c8, 3999 0x1f408, 0x1f40c, 4000 0x1f440, 0x1f44c, 4001 0x1f684, 0x1f690, 4002 0x1f6c0, 0x1f6c0, 4003 0x1f6e0, 0x1f6e0, 4004 0x1f700, 0x1f784, 4005 0x1f7c0, 0x1f7c8, 4006 0x1f808, 0x1f80c, 4007 0x1f840, 0x1f84c, 4008 0x1fa84, 0x1fa90, 4009 0x1fac0, 0x1fac0, 4010 0x1fae0, 0x1fae0, 4011 0x1fb00, 0x1fb84, 4012 0x1fbc0, 0x1fbc8, 4013 0x1fc08, 0x1fc0c, 4014 0x1fc40, 0x1fc4c, 4015 0x1fe84, 0x1fe90, 4016 0x1fec0, 0x1fec0, 4017 0x1fee0, 0x1fee0, 4018 0x1ff00, 0x1ff84, 4019 0x1ffc0, 0x1ffc8, 4020 0x30000, 0x30030, 4021 0x30100, 0x30144, 4022 0x30190, 0x301d0, 4023 0x30200, 0x30318, 4024 0x30400, 0x3052c, 4025 0x30540, 0x3061c, 4026 0x30800, 0x30834, 4027 0x308c0, 0x30908, 4028 0x30910, 0x309ac, 4029 0x30a00, 0x30a2c, 4030 0x30a44, 0x30a50, 4031 0x30a74, 0x30c24, 4032 0x30d00, 0x30d00, 4033 0x30d08, 0x30d14, 4034 0x30d1c, 0x30d20, 4035 0x30d3c, 0x30d50, 4036 0x31200, 0x3120c, 4037 0x31220, 0x31220, 4038 0x31240, 0x31240, 4039 0x31600, 0x3160c, 4040 0x31a00, 0x31a1c, 4041 0x31e00, 0x31e20, 4042 0x31e38, 0x31e3c, 4043 0x31e80, 0x31e80, 4044 0x31e88, 0x31ea8, 4045 0x31eb0, 0x31eb4, 4046 0x31ec8, 0x31ed4, 4047 0x31fb8, 0x32004, 4048 0x32200, 0x32200, 4049 0x32208, 0x32240, 4050 0x32248, 0x32280, 4051 0x32288, 0x322c0, 4052 0x322c8, 0x322fc, 4053 0x32600, 0x32630, 4054 0x32a00, 0x32abc, 4055 0x32b00, 0x32b70, 4056 0x33000, 0x33048, 4057 0x33060, 0x3309c, 4058 0x330f0, 0x33148, 4059 0x33160, 0x3319c, 4060 0x331f0, 0x332e4, 4061 0x332f8, 0x333e4, 4062 0x333f8, 0x33448, 4063 0x33460, 0x3349c, 4064 0x334f0, 0x33548, 4065 0x33560, 0x3359c, 4066 0x335f0, 0x336e4, 4067 0x336f8, 0x337e4, 4068 0x337f8, 0x337fc, 4069 0x33814, 0x33814, 4070 0x3382c, 0x3382c, 4071 0x33880, 0x3388c, 4072 0x338e8, 0x338ec, 4073 0x33900, 0x33948, 4074 0x33960, 0x3399c, 4075 0x339f0, 0x33ae4, 4076 0x33af8, 0x33b10, 4077 0x33b28, 0x33b28, 4078 0x33b3c, 0x33b50, 4079 0x33bf0, 0x33c10, 4080 0x33c28, 0x33c28, 4081 0x33c3c, 0x33c50, 4082 0x33cf0, 0x33cfc, 4083 0x34000, 0x34030, 4084 0x34100, 0x34144, 4085 0x34190, 0x341d0, 4086 0x34200, 0x34318, 4087 0x34400, 0x3452c, 4088 0x34540, 0x3461c, 4089 0x34800, 0x34834, 4090 0x348c0, 0x34908, 4091 0x34910, 0x349ac, 4092 0x34a00, 0x34a2c, 4093 0x34a44, 0x34a50, 4094 0x34a74, 0x34c24, 4095 0x34d00, 0x34d00, 4096 0x34d08, 0x34d14, 4097 0x34d1c, 0x34d20, 4098 0x34d3c, 0x34d50, 4099 0x35200, 0x3520c, 4100 0x35220, 0x35220, 4101 0x35240, 0x35240, 4102 0x35600, 0x3560c, 4103 0x35a00, 0x35a1c, 4104 0x35e00, 0x35e20, 4105 0x35e38, 0x35e3c, 4106 0x35e80, 0x35e80, 4107 0x35e88, 0x35ea8, 4108 0x35eb0, 0x35eb4, 4109 0x35ec8, 0x35ed4, 4110 0x35fb8, 0x36004, 4111 0x36200, 0x36200, 4112 0x36208, 0x36240, 4113 0x36248, 0x36280, 4114 0x36288, 0x362c0, 4115 0x362c8, 0x362fc, 4116 0x36600, 0x36630, 4117 0x36a00, 0x36abc, 4118 0x36b00, 0x36b70, 4119 0x37000, 0x37048, 4120 0x37060, 0x3709c, 4121 0x370f0, 0x37148, 4122 0x37160, 0x3719c, 4123 0x371f0, 0x372e4, 4124 0x372f8, 0x373e4, 4125 0x373f8, 0x37448, 4126 0x37460, 0x3749c, 4127 0x374f0, 0x37548, 4128 0x37560, 0x3759c, 4129 0x375f0, 0x376e4, 4130 0x376f8, 0x377e4, 4131 0x377f8, 0x377fc, 4132 0x37814, 0x37814, 4133 0x3782c, 0x3782c, 4134 0x37880, 0x3788c, 4135 0x378e8, 0x378ec, 4136 0x37900, 0x37948, 4137 0x37960, 0x3799c, 4138 0x379f0, 0x37ae4, 4139 0x37af8, 0x37b10, 4140 0x37b28, 0x37b28, 4141 0x37b3c, 0x37b50, 4142 0x37bf0, 0x37c10, 4143 0x37c28, 0x37c28, 4144 0x37c3c, 0x37c50, 4145 0x37cf0, 0x37cfc, 4146 0x38000, 0x38030, 4147 0x38100, 0x38144, 4148 0x38190, 0x381d0, 4149 0x38200, 0x38318, 4150 0x38400, 0x3852c, 4151 0x38540, 0x3861c, 4152 0x38800, 0x38834, 4153 0x388c0, 0x38908, 4154 0x38910, 0x389ac, 4155 0x38a00, 0x38a2c, 4156 0x38a44, 0x38a50, 4157 0x38a74, 0x38c24, 4158 0x38d00, 0x38d00, 4159 0x38d08, 0x38d14, 4160 0x38d1c, 0x38d20, 4161 0x38d3c, 0x38d50, 4162 0x39200, 0x3920c, 4163 0x39220, 0x39220, 4164 0x39240, 0x39240, 4165 0x39600, 0x3960c, 4166 0x39a00, 0x39a1c, 4167 0x39e00, 0x39e20, 4168 0x39e38, 0x39e3c, 4169 0x39e80, 0x39e80, 4170 0x39e88, 0x39ea8, 4171 0x39eb0, 0x39eb4, 4172 0x39ec8, 0x39ed4, 4173 0x39fb8, 0x3a004, 4174 0x3a200, 0x3a200, 4175 0x3a208, 0x3a240, 4176 0x3a248, 0x3a280, 4177 0x3a288, 0x3a2c0, 4178 0x3a2c8, 0x3a2fc, 4179 0x3a600, 0x3a630, 4180 0x3aa00, 0x3aabc, 4181 0x3ab00, 0x3ab70, 4182 0x3b000, 0x3b048, 4183 0x3b060, 0x3b09c, 4184 0x3b0f0, 0x3b148, 4185 0x3b160, 0x3b19c, 4186 0x3b1f0, 0x3b2e4, 4187 0x3b2f8, 0x3b3e4, 4188 0x3b3f8, 0x3b448, 4189 0x3b460, 0x3b49c, 4190 0x3b4f0, 0x3b548, 4191 0x3b560, 0x3b59c, 4192 0x3b5f0, 0x3b6e4, 4193 0x3b6f8, 0x3b7e4, 4194 0x3b7f8, 0x3b7fc, 4195 0x3b814, 0x3b814, 4196 0x3b82c, 0x3b82c, 4197 0x3b880, 0x3b88c, 4198 0x3b8e8, 0x3b8ec, 4199 0x3b900, 0x3b948, 4200 0x3b960, 0x3b99c, 4201 0x3b9f0, 0x3bae4, 4202 0x3baf8, 0x3bb10, 4203 0x3bb28, 0x3bb28, 4204 0x3bb3c, 0x3bb50, 4205 0x3bbf0, 0x3bc10, 4206 0x3bc28, 0x3bc28, 4207 0x3bc3c, 0x3bc50, 4208 0x3bcf0, 0x3bcfc, 4209 0x3c000, 0x3c030, 4210 0x3c100, 0x3c144, 4211 0x3c190, 0x3c1d0, 4212 0x3c200, 0x3c318, 4213 0x3c400, 0x3c52c, 4214 0x3c540, 0x3c61c, 4215 0x3c800, 0x3c834, 4216 0x3c8c0, 0x3c908, 4217 0x3c910, 0x3c9ac, 4218 0x3ca00, 0x3ca2c, 4219 0x3ca44, 0x3ca50, 4220 0x3ca74, 0x3cc24, 4221 0x3cd00, 0x3cd00, 4222 0x3cd08, 0x3cd14, 4223 0x3cd1c, 0x3cd20, 4224 0x3cd3c, 0x3cd50, 4225 0x3d200, 0x3d20c, 4226 0x3d220, 0x3d220, 4227 0x3d240, 0x3d240, 4228 0x3d600, 0x3d60c, 4229 0x3da00, 0x3da1c, 4230 0x3de00, 0x3de20, 4231 0x3de38, 0x3de3c, 4232 0x3de80, 0x3de80, 4233 0x3de88, 0x3dea8, 4234 0x3deb0, 0x3deb4, 4235 0x3dec8, 0x3ded4, 4236 0x3dfb8, 0x3e004, 4237 0x3e200, 0x3e200, 4238 0x3e208, 0x3e240, 4239 0x3e248, 0x3e280, 4240 0x3e288, 0x3e2c0, 4241 0x3e2c8, 0x3e2fc, 4242 0x3e600, 0x3e630, 4243 0x3ea00, 0x3eabc, 4244 0x3eb00, 0x3eb70, 4245 0x3f000, 0x3f048, 4246 0x3f060, 0x3f09c, 4247 0x3f0f0, 0x3f148, 4248 0x3f160, 0x3f19c, 4249 0x3f1f0, 0x3f2e4, 4250 0x3f2f8, 0x3f3e4, 4251 0x3f3f8, 0x3f448, 4252 0x3f460, 0x3f49c, 4253 0x3f4f0, 0x3f548, 4254 0x3f560, 0x3f59c, 4255 0x3f5f0, 0x3f6e4, 4256 0x3f6f8, 0x3f7e4, 4257 0x3f7f8, 0x3f7fc, 4258 0x3f814, 0x3f814, 4259 0x3f82c, 0x3f82c, 4260 0x3f880, 0x3f88c, 4261 0x3f8e8, 0x3f8ec, 4262 0x3f900, 0x3f948, 4263 0x3f960, 0x3f99c, 4264 0x3f9f0, 0x3fae4, 4265 0x3faf8, 0x3fb10, 4266 0x3fb28, 0x3fb28, 4267 0x3fb3c, 0x3fb50, 4268 0x3fbf0, 0x3fc10, 4269 0x3fc28, 0x3fc28, 4270 0x3fc3c, 0x3fc50, 4271 0x3fcf0, 0x3fcfc, 4272 0x40000, 0x4000c, 4273 0x40040, 0x40068, 4274 0x4007c, 0x40144, 4275 0x40180, 0x4018c, 4276 0x40200, 0x40298, 4277 0x402ac, 0x4033c, 4278 0x403f8, 0x403fc, 4279 0x41304, 0x413c4, 4280 0x41400, 0x4141c, 4281 0x41480, 0x414d0, 4282 0x44000, 0x44078, 4283 0x440c0, 0x44278, 4284 0x442c0, 0x44478, 4285 0x444c0, 0x44678, 4286 0x446c0, 0x44878, 4287 0x448c0, 0x449fc, 4288 0x45000, 0x45068, 4289 0x45080, 0x45084, 4290 0x450a0, 0x450b0, 4291 0x45200, 0x45268, 4292 0x45280, 0x45284, 4293 0x452a0, 0x452b0, 4294 0x460c0, 0x460e4, 4295 0x47000, 0x4708c, 4296 0x47200, 0x47250, 4297 0x47400, 0x47420, 4298 0x47600, 0x47618, 4299 0x47800, 0x47814, 4300 0x48000, 0x4800c, 4301 0x48040, 0x48068, 4302 0x4807c, 0x48144, 4303 0x48180, 0x4818c, 4304 0x48200, 0x48298, 4305 0x482ac, 0x4833c, 4306 0x483f8, 0x483fc, 4307 0x49304, 0x493c4, 4308 0x49400, 0x4941c, 4309 0x49480, 0x494d0, 4310 0x4c000, 0x4c078, 4311 0x4c0c0, 0x4c278, 4312 0x4c2c0, 0x4c478, 4313 0x4c4c0, 0x4c678, 4314 0x4c6c0, 0x4c878, 4315 0x4c8c0, 0x4c9fc, 4316 0x4d000, 0x4d068, 4317 0x4d080, 0x4d084, 4318 0x4d0a0, 0x4d0b0, 4319 0x4d200, 0x4d268, 4320 0x4d280, 0x4d284, 4321 0x4d2a0, 0x4d2b0, 4322 0x4e0c0, 0x4e0e4, 4323 0x4f000, 0x4f08c, 4324 0x4f200, 0x4f250, 4325 0x4f400, 0x4f420, 4326 0x4f600, 0x4f618, 4327 0x4f800, 0x4f814, 4328 0x50000, 0x500cc, 4329 0x50400, 0x50400, 4330 0x50800, 0x508cc, 4331 0x50c00, 0x50c00, 4332 0x51000, 0x5101c, 4333 0x51300, 0x51308, 4334 }; 4335 4336 if (is_t4(sc)) { 4337 reg_ranges = &t4_reg_ranges[0]; 4338 n = nitems(t4_reg_ranges); 4339 } else { 4340 reg_ranges = &t5_reg_ranges[0]; 4341 n = nitems(t5_reg_ranges); 4342 } 4343 4344 regs->version = chip_id(sc) | chip_rev(sc) << 10; 4345 for (i = 0; i < n; i += 2) 4346 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 4347 } 4348 4349 static void 4350 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi) 4351 { 4352 int i; 4353 u_int v, tnl_cong_drops; 4354 struct timeval tv; 4355 const struct timeval interval = {0, 250000}; /* 250ms */ 4356 4357 getmicrotime(&tv); 4358 timevalsub(&tv, &interval); 4359 if (timevalcmp(&tv, &pi->last_refreshed, <)) 4360 return; 4361 4362 tnl_cong_drops = 0; 4363 t4_get_port_stats(sc, pi->tx_chan, &pi->stats); 4364 for (i = 0; i < NCHAN; i++) { 4365 if (pi->rx_chan_map & (1 << i)) { 4366 mtx_lock(&sc->regwin_lock); 4367 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 4368 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 4369 mtx_unlock(&sc->regwin_lock); 4370 tnl_cong_drops += v; 4371 } 4372 } 4373 pi->tnl_cong_drops = tnl_cong_drops; 4374 getmicrotime(&pi->last_refreshed); 4375 } 4376 4377 static void 4378 cxgbe_tick(void *arg) 4379 { 4380 struct port_info *pi = arg; 4381 struct adapter *sc = pi->adapter; 4382 struct ifnet *ifp = pi->ifp; 4383 4384 PORT_LOCK(pi); 4385 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 4386 PORT_UNLOCK(pi); 4387 return; /* without scheduling another callout */ 4388 } 4389 4390 cxgbe_refresh_stats(sc, pi); 4391 4392 callout_schedule(&pi->tick, hz); 4393 PORT_UNLOCK(pi); 4394 } 4395 4396 static void 4397 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 4398 { 4399 struct ifnet *vlan; 4400 4401 if (arg != ifp || ifp->if_type != IFT_ETHER) 4402 return; 4403 4404 vlan = VLAN_DEVAT(ifp, vid); 4405 VLAN_SETCOOKIE(vlan, ifp); 4406 } 4407 4408 static int 4409 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 4410 { 4411 4412 #ifdef INVARIANTS 4413 panic("%s: opcode 0x%02x on iq %p with payload %p", 4414 __func__, rss->opcode, iq, m); 4415 #else 4416 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 4417 __func__, rss->opcode, iq, m); 4418 m_freem(m); 4419 #endif 4420 return (EDOOFUS); 4421 } 4422 4423 int 4424 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 4425 { 4426 uintptr_t *loc, new; 4427 4428 if (opcode >= nitems(sc->cpl_handler)) 4429 return (EINVAL); 4430 4431 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 4432 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 4433 atomic_store_rel_ptr(loc, new); 4434 4435 return (0); 4436 } 4437 4438 static int 4439 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 4440 { 4441 4442 #ifdef INVARIANTS 4443 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 4444 #else 4445 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 4446 __func__, iq, ctrl); 4447 #endif 4448 return (EDOOFUS); 4449 } 4450 4451 int 4452 t4_register_an_handler(struct adapter *sc, an_handler_t h) 4453 { 4454 uintptr_t *loc, new; 4455 4456 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 4457 loc = (uintptr_t *) &sc->an_handler; 4458 atomic_store_rel_ptr(loc, new); 4459 4460 return (0); 4461 } 4462 4463 static int 4464 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 4465 { 4466 const struct cpl_fw6_msg *cpl = 4467 __containerof(rpl, struct cpl_fw6_msg, data[0]); 4468 4469 #ifdef INVARIANTS 4470 panic("%s: fw_msg type %d", __func__, cpl->type); 4471 #else 4472 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 4473 #endif 4474 return (EDOOFUS); 4475 } 4476 4477 int 4478 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 4479 { 4480 uintptr_t *loc, new; 4481 4482 if (type >= nitems(sc->fw_msg_handler)) 4483 return (EINVAL); 4484 4485 /* 4486 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 4487 * handler dispatch table. Reject any attempt to install a handler for 4488 * this subtype. 4489 */ 4490 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 4491 return (EINVAL); 4492 4493 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 4494 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 4495 atomic_store_rel_ptr(loc, new); 4496 4497 return (0); 4498 } 4499 4500 static int 4501 t4_sysctls(struct adapter *sc) 4502 { 4503 struct sysctl_ctx_list *ctx; 4504 struct sysctl_oid *oid; 4505 struct sysctl_oid_list *children, *c0; 4506 static char *caps[] = { 4507 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 4508 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL" /* caps[1] niccaps */ 4509 "\6HASHFILTER\7ETHOFLD", 4510 "\20\1TOE", /* caps[2] toecaps */ 4511 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 4512 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 4513 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 4514 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 4515 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 4516 "\4PO_INITIAOR\5PO_TARGET" 4517 }; 4518 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 4519 4520 ctx = device_get_sysctl_ctx(sc->dev); 4521 4522 /* 4523 * dev.t4nex.X. 4524 */ 4525 oid = device_get_sysctl_tree(sc->dev); 4526 c0 = children = SYSCTL_CHILDREN(oid); 4527 4528 sc->sc_do_rxcopy = 1; 4529 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, 4530 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); 4531 4532 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 4533 sc->params.nports, "# of ports"); 4534 4535 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 4536 NULL, chip_rev(sc), "chip hardware revision"); 4537 4538 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 4539 CTLFLAG_RD, sc->fw_version, 0, "firmware version"); 4540 4541 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 4542 CTLFLAG_RD, sc->cfg_file, 0, "configuration file"); 4543 4544 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 4545 sc->cfcsum, "config file checksum"); 4546 4547 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 4548 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 4549 sysctl_bitfield, "A", "available doorbells"); 4550 4551 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 4552 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 4553 sysctl_bitfield, "A", "available link capabilities"); 4554 4555 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 4556 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 4557 sysctl_bitfield, "A", "available NIC capabilities"); 4558 4559 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 4560 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 4561 sysctl_bitfield, "A", "available TCP offload capabilities"); 4562 4563 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 4564 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 4565 sysctl_bitfield, "A", "available RDMA capabilities"); 4566 4567 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 4568 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 4569 sysctl_bitfield, "A", "available iSCSI capabilities"); 4570 4571 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 4572 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 4573 sysctl_bitfield, "A", "available FCoE capabilities"); 4574 4575 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 4576 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 4577 4578 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 4579 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 4580 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 4581 "interrupt holdoff timer values (us)"); 4582 4583 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 4584 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 4585 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 4586 "interrupt holdoff packet counter values"); 4587 4588 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, 4589 NULL, sc->tids.nftids, "number of filters"); 4590 4591 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | 4592 CTLFLAG_RD, sc, 0, sysctl_temperature, "I", 4593 "chip temperature (in Celsius)"); 4594 4595 t4_sge_sysctls(sc, ctx, children); 4596 4597 sc->lro_timeout = 100; 4598 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, 4599 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); 4600 4601 #ifdef SBUF_DRAIN 4602 /* 4603 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 4604 */ 4605 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 4606 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 4607 "logs and miscellaneous information"); 4608 children = SYSCTL_CHILDREN(oid); 4609 4610 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 4611 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4612 sysctl_cctrl, "A", "congestion control"); 4613 4614 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 4615 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4616 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 4617 4618 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 4619 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 4620 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 4621 4622 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 4623 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 4624 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 4625 4626 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 4627 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 4628 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 4629 4630 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 4631 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 4632 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 4633 4634 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 4635 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 4636 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 4637 4638 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 4639 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4640 sysctl_cim_la, "A", "CIM logic analyzer"); 4641 4642 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", 4643 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4644 sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); 4645 4646 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 4647 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 4648 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 4649 4650 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 4651 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 4652 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 4653 4654 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 4655 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 4656 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 4657 4658 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 4659 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 4660 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 4661 4662 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 4663 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 4664 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 4665 4666 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 4667 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 4668 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 4669 4670 if (is_t5(sc)) { 4671 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 4672 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 4673 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 4674 4675 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 4676 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 4677 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 4678 } 4679 4680 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", 4681 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4682 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); 4683 4684 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 4685 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4686 sysctl_cim_qcfg, "A", "CIM queue configuration"); 4687 4688 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 4689 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4690 sysctl_cpl_stats, "A", "CPL statistics"); 4691 4692 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 4693 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4694 sysctl_ddp_stats, "A", "DDP statistics"); 4695 4696 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 4697 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4698 sysctl_devlog, "A", "firmware's device log"); 4699 4700 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 4701 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4702 sysctl_fcoe_stats, "A", "FCoE statistics"); 4703 4704 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 4705 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4706 sysctl_hw_sched, "A", "hardware scheduler "); 4707 4708 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 4709 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4710 sysctl_l2t, "A", "hardware L2 table"); 4711 4712 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 4713 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4714 sysctl_lb_stats, "A", "loopback statistics"); 4715 4716 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 4717 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4718 sysctl_meminfo, "A", "memory regions"); 4719 4720 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", 4721 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4722 sysctl_mps_tcam, "A", "MPS TCAM entries"); 4723 4724 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 4725 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4726 sysctl_path_mtus, "A", "path MTUs"); 4727 4728 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 4729 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4730 sysctl_pm_stats, "A", "PM statistics"); 4731 4732 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 4733 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4734 sysctl_rdma_stats, "A", "RDMA statistics"); 4735 4736 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 4737 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4738 sysctl_tcp_stats, "A", "TCP statistics"); 4739 4740 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 4741 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4742 sysctl_tids, "A", "TID information"); 4743 4744 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 4745 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4746 sysctl_tp_err_stats, "A", "TP error statistics"); 4747 4748 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", 4749 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4750 sysctl_tp_la, "A", "TP logic analyzer"); 4751 4752 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 4753 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4754 sysctl_tx_rate, "A", "Tx rate"); 4755 4756 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", 4757 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4758 sysctl_ulprx_la, "A", "ULPRX logic analyzer"); 4759 4760 if (is_t5(sc)) { 4761 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 4762 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4763 sysctl_wcwr_stats, "A", "write combined work requests"); 4764 } 4765 #endif 4766 4767 #ifdef TCP_OFFLOAD 4768 if (is_offload(sc)) { 4769 /* 4770 * dev.t4nex.X.toe. 4771 */ 4772 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 4773 NULL, "TOE parameters"); 4774 children = SYSCTL_CHILDREN(oid); 4775 4776 sc->tt.sndbuf = 256 * 1024; 4777 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 4778 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 4779 4780 sc->tt.ddp = 0; 4781 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 4782 &sc->tt.ddp, 0, "DDP allowed"); 4783 4784 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 4785 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 4786 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 4787 4788 sc->tt.ddp_thres = 4789 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 4790 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 4791 &sc->tt.ddp_thres, 0, "DDP threshold"); 4792 4793 sc->tt.rx_coalesce = 1; 4794 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", 4795 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); 4796 } 4797 #endif 4798 4799 4800 return (0); 4801 } 4802 4803 static int 4804 cxgbe_sysctls(struct port_info *pi) 4805 { 4806 struct sysctl_ctx_list *ctx; 4807 struct sysctl_oid *oid; 4808 struct sysctl_oid_list *children; 4809 struct adapter *sc = pi->adapter; 4810 4811 ctx = device_get_sysctl_ctx(pi->dev); 4812 4813 /* 4814 * dev.cxgbe.X. 4815 */ 4816 oid = device_get_sysctl_tree(pi->dev); 4817 children = SYSCTL_CHILDREN(oid); 4818 4819 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | 4820 CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); 4821 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { 4822 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 4823 CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I", 4824 "PHY temperature (in Celsius)"); 4825 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", 4826 CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I", 4827 "PHY firmware version"); 4828 } 4829 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 4830 &pi->nrxq, 0, "# of rx queues"); 4831 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 4832 &pi->ntxq, 0, "# of tx queues"); 4833 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 4834 &pi->first_rxq, 0, "index of first rx queue"); 4835 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 4836 &pi->first_txq, 0, "index of first tx queue"); 4837 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT | 4838 CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU", 4839 "Reserve queue 0 for non-flowid packets"); 4840 4841 #ifdef TCP_OFFLOAD 4842 if (is_offload(sc)) { 4843 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 4844 &pi->nofldrxq, 0, 4845 "# of rx queues for offloaded TCP connections"); 4846 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 4847 &pi->nofldtxq, 0, 4848 "# of tx queues for offloaded TCP connections"); 4849 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 4850 CTLFLAG_RD, &pi->first_ofld_rxq, 0, 4851 "index of first TOE rx queue"); 4852 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 4853 CTLFLAG_RD, &pi->first_ofld_txq, 0, 4854 "index of first TOE tx queue"); 4855 } 4856 #endif 4857 #ifdef DEV_NETMAP 4858 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD, 4859 &pi->nnmrxq, 0, "# of rx queues for netmap"); 4860 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD, 4861 &pi->nnmtxq, 0, "# of tx queues for netmap"); 4862 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq", 4863 CTLFLAG_RD, &pi->first_nm_rxq, 0, 4864 "index of first netmap rx queue"); 4865 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq", 4866 CTLFLAG_RD, &pi->first_nm_txq, 0, 4867 "index of first netmap tx queue"); 4868 #endif 4869 4870 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 4871 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 4872 "holdoff timer index"); 4873 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 4874 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 4875 "holdoff packet counter index"); 4876 4877 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 4878 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 4879 "rx queue size"); 4880 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 4881 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 4882 "tx queue size"); 4883 4884 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings", 4885 CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings, 4886 "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)"); 4887 4888 /* 4889 * dev.cxgbe.X.stats. 4890 */ 4891 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 4892 NULL, "port statistics"); 4893 children = SYSCTL_CHILDREN(oid); 4894 4895 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 4896 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 4897 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ 4898 sysctl_handle_t4_reg64, "QU", desc) 4899 4900 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 4901 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 4902 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 4903 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 4904 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 4905 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 4906 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 4907 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 4908 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 4909 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 4910 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 4911 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 4912 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 4913 "# of tx frames in this range", 4914 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 4915 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 4916 "# of tx frames in this range", 4917 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 4918 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 4919 "# of tx frames in this range", 4920 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 4921 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 4922 "# of tx frames in this range", 4923 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 4924 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 4925 "# of tx frames in this range", 4926 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 4927 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 4928 "# of tx frames in this range", 4929 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 4930 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 4931 "# of tx frames in this range", 4932 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 4933 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 4934 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 4935 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 4936 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 4937 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 4938 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 4939 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 4940 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 4941 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 4942 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 4943 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 4944 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 4945 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 4946 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 4947 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 4948 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 4949 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 4950 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 4951 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 4952 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 4953 4954 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 4955 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 4956 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 4957 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 4958 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 4959 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 4960 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 4961 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 4962 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 4963 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 4964 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 4965 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 4966 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 4967 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 4968 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 4969 "# of frames received with bad FCS", 4970 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 4971 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 4972 "# of frames received with length error", 4973 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 4974 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 4975 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 4976 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 4977 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 4978 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 4979 "# of rx frames in this range", 4980 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 4981 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 4982 "# of rx frames in this range", 4983 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 4984 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 4985 "# of rx frames in this range", 4986 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 4987 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 4988 "# of rx frames in this range", 4989 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 4990 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 4991 "# of rx frames in this range", 4992 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 4993 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 4994 "# of rx frames in this range", 4995 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 4996 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 4997 "# of rx frames in this range", 4998 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 4999 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 5000 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 5001 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 5002 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 5003 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 5004 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 5005 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 5006 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 5007 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 5008 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 5009 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 5010 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 5011 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 5012 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 5013 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 5014 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 5015 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 5016 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 5017 5018 #undef SYSCTL_ADD_T4_REG64 5019 5020 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 5021 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 5022 &pi->stats.name, desc) 5023 5024 /* We get these from port_stats and they may be stale by upto 1s */ 5025 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 5026 "# drops due to buffer-group 0 overflows"); 5027 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 5028 "# drops due to buffer-group 1 overflows"); 5029 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 5030 "# drops due to buffer-group 2 overflows"); 5031 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 5032 "# drops due to buffer-group 3 overflows"); 5033 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 5034 "# of buffer-group 0 truncated packets"); 5035 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 5036 "# of buffer-group 1 truncated packets"); 5037 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 5038 "# of buffer-group 2 truncated packets"); 5039 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 5040 "# of buffer-group 3 truncated packets"); 5041 5042 #undef SYSCTL_ADD_T4_PORTSTAT 5043 5044 return (0); 5045 } 5046 5047 static int 5048 sysctl_int_array(SYSCTL_HANDLER_ARGS) 5049 { 5050 int rc, *i; 5051 struct sbuf sb; 5052 5053 sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND); 5054 for (i = arg1; arg2; arg2 -= sizeof(int), i++) 5055 sbuf_printf(&sb, "%d ", *i); 5056 sbuf_trim(&sb); 5057 sbuf_finish(&sb); 5058 rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 5059 sbuf_delete(&sb); 5060 return (rc); 5061 } 5062 5063 static int 5064 sysctl_bitfield(SYSCTL_HANDLER_ARGS) 5065 { 5066 int rc; 5067 struct sbuf *sb; 5068 5069 rc = sysctl_wire_old_buffer(req, 0); 5070 if (rc != 0) 5071 return(rc); 5072 5073 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5074 if (sb == NULL) 5075 return (ENOMEM); 5076 5077 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 5078 rc = sbuf_finish(sb); 5079 sbuf_delete(sb); 5080 5081 return (rc); 5082 } 5083 5084 static int 5085 sysctl_btphy(SYSCTL_HANDLER_ARGS) 5086 { 5087 struct port_info *pi = arg1; 5088 int op = arg2; 5089 struct adapter *sc = pi->adapter; 5090 u_int v; 5091 int rc; 5092 5093 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt"); 5094 if (rc) 5095 return (rc); 5096 /* XXX: magic numbers */ 5097 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, 5098 &v); 5099 end_synchronized_op(sc, 0); 5100 if (rc) 5101 return (rc); 5102 if (op == 0) 5103 v /= 256; 5104 5105 rc = sysctl_handle_int(oidp, &v, 0, req); 5106 return (rc); 5107 } 5108 5109 static int 5110 sysctl_noflowq(SYSCTL_HANDLER_ARGS) 5111 { 5112 struct port_info *pi = arg1; 5113 int rc, val; 5114 5115 val = pi->rsrv_noflowq; 5116 rc = sysctl_handle_int(oidp, &val, 0, req); 5117 if (rc != 0 || req->newptr == NULL) 5118 return (rc); 5119 5120 if ((val >= 1) && (pi->ntxq > 1)) 5121 pi->rsrv_noflowq = 1; 5122 else 5123 pi->rsrv_noflowq = 0; 5124 5125 return (rc); 5126 } 5127 5128 static int 5129 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 5130 { 5131 struct port_info *pi = arg1; 5132 struct adapter *sc = pi->adapter; 5133 int idx, rc, i; 5134 struct sge_rxq *rxq; 5135 #ifdef TCP_OFFLOAD 5136 struct sge_ofld_rxq *ofld_rxq; 5137 #endif 5138 uint8_t v; 5139 5140 idx = pi->tmr_idx; 5141 5142 rc = sysctl_handle_int(oidp, &idx, 0, req); 5143 if (rc != 0 || req->newptr == NULL) 5144 return (rc); 5145 5146 if (idx < 0 || idx >= SGE_NTIMERS) 5147 return (EINVAL); 5148 5149 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5150 "t4tmr"); 5151 if (rc) 5152 return (rc); 5153 5154 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 5155 for_each_rxq(pi, i, rxq) { 5156 #ifdef atomic_store_rel_8 5157 atomic_store_rel_8(&rxq->iq.intr_params, v); 5158 #else 5159 rxq->iq.intr_params = v; 5160 #endif 5161 } 5162 #ifdef TCP_OFFLOAD 5163 for_each_ofld_rxq(pi, i, ofld_rxq) { 5164 #ifdef atomic_store_rel_8 5165 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); 5166 #else 5167 ofld_rxq->iq.intr_params = v; 5168 #endif 5169 } 5170 #endif 5171 pi->tmr_idx = idx; 5172 5173 end_synchronized_op(sc, LOCK_HELD); 5174 return (0); 5175 } 5176 5177 static int 5178 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 5179 { 5180 struct port_info *pi = arg1; 5181 struct adapter *sc = pi->adapter; 5182 int idx, rc; 5183 5184 idx = pi->pktc_idx; 5185 5186 rc = sysctl_handle_int(oidp, &idx, 0, req); 5187 if (rc != 0 || req->newptr == NULL) 5188 return (rc); 5189 5190 if (idx < -1 || idx >= SGE_NCOUNTERS) 5191 return (EINVAL); 5192 5193 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5194 "t4pktc"); 5195 if (rc) 5196 return (rc); 5197 5198 if (pi->flags & PORT_INIT_DONE) 5199 rc = EBUSY; /* cannot be changed once the queues are created */ 5200 else 5201 pi->pktc_idx = idx; 5202 5203 end_synchronized_op(sc, LOCK_HELD); 5204 return (rc); 5205 } 5206 5207 static int 5208 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 5209 { 5210 struct port_info *pi = arg1; 5211 struct adapter *sc = pi->adapter; 5212 int qsize, rc; 5213 5214 qsize = pi->qsize_rxq; 5215 5216 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5217 if (rc != 0 || req->newptr == NULL) 5218 return (rc); 5219 5220 if (qsize < 128 || (qsize & 7)) 5221 return (EINVAL); 5222 5223 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5224 "t4rxqs"); 5225 if (rc) 5226 return (rc); 5227 5228 if (pi->flags & PORT_INIT_DONE) 5229 rc = EBUSY; /* cannot be changed once the queues are created */ 5230 else 5231 pi->qsize_rxq = qsize; 5232 5233 end_synchronized_op(sc, LOCK_HELD); 5234 return (rc); 5235 } 5236 5237 static int 5238 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 5239 { 5240 struct port_info *pi = arg1; 5241 struct adapter *sc = pi->adapter; 5242 int qsize, rc; 5243 5244 qsize = pi->qsize_txq; 5245 5246 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5247 if (rc != 0 || req->newptr == NULL) 5248 return (rc); 5249 5250 /* bufring size must be powerof2 */ 5251 if (qsize < 128 || !powerof2(qsize)) 5252 return (EINVAL); 5253 5254 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5255 "t4txqs"); 5256 if (rc) 5257 return (rc); 5258 5259 if (pi->flags & PORT_INIT_DONE) 5260 rc = EBUSY; /* cannot be changed once the queues are created */ 5261 else 5262 pi->qsize_txq = qsize; 5263 5264 end_synchronized_op(sc, LOCK_HELD); 5265 return (rc); 5266 } 5267 5268 static int 5269 sysctl_pause_settings(SYSCTL_HANDLER_ARGS) 5270 { 5271 struct port_info *pi = arg1; 5272 struct adapter *sc = pi->adapter; 5273 struct link_config *lc = &pi->link_cfg; 5274 int rc; 5275 5276 if (req->newptr == NULL) { 5277 struct sbuf *sb; 5278 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX"; 5279 5280 rc = sysctl_wire_old_buffer(req, 0); 5281 if (rc != 0) 5282 return(rc); 5283 5284 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5285 if (sb == NULL) 5286 return (ENOMEM); 5287 5288 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits); 5289 rc = sbuf_finish(sb); 5290 sbuf_delete(sb); 5291 } else { 5292 char s[2]; 5293 int n; 5294 5295 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX)); 5296 s[1] = 0; 5297 5298 rc = sysctl_handle_string(oidp, s, sizeof(s), req); 5299 if (rc != 0) 5300 return(rc); 5301 5302 if (s[1] != 0) 5303 return (EINVAL); 5304 if (s[0] < '0' || s[0] > '9') 5305 return (EINVAL); /* not a number */ 5306 n = s[0] - '0'; 5307 if (n & ~(PAUSE_TX | PAUSE_RX)) 5308 return (EINVAL); /* some other bit is set too */ 5309 5310 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE"); 5311 if (rc) 5312 return (rc); 5313 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) { 5314 int link_ok = lc->link_ok; 5315 5316 lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX); 5317 lc->requested_fc |= n; 5318 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc); 5319 lc->link_ok = link_ok; /* restore */ 5320 } 5321 end_synchronized_op(sc, 0); 5322 } 5323 5324 return (rc); 5325 } 5326 5327 static int 5328 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 5329 { 5330 struct adapter *sc = arg1; 5331 int reg = arg2; 5332 uint64_t val; 5333 5334 val = t4_read_reg64(sc, reg); 5335 5336 return (sysctl_handle_64(oidp, &val, 0, req)); 5337 } 5338 5339 static int 5340 sysctl_temperature(SYSCTL_HANDLER_ARGS) 5341 { 5342 struct adapter *sc = arg1; 5343 int rc, t; 5344 uint32_t param, val; 5345 5346 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); 5347 if (rc) 5348 return (rc); 5349 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 5350 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 5351 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); 5352 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 5353 end_synchronized_op(sc, 0); 5354 if (rc) 5355 return (rc); 5356 5357 /* unknown is returned as 0 but we display -1 in that case */ 5358 t = val == 0 ? -1 : val; 5359 5360 rc = sysctl_handle_int(oidp, &t, 0, req); 5361 return (rc); 5362 } 5363 5364 #ifdef SBUF_DRAIN 5365 static int 5366 sysctl_cctrl(SYSCTL_HANDLER_ARGS) 5367 { 5368 struct adapter *sc = arg1; 5369 struct sbuf *sb; 5370 int rc, i; 5371 uint16_t incr[NMTUS][NCCTRL_WIN]; 5372 static const char *dec_fac[] = { 5373 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 5374 "0.9375" 5375 }; 5376 5377 rc = sysctl_wire_old_buffer(req, 0); 5378 if (rc != 0) 5379 return (rc); 5380 5381 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5382 if (sb == NULL) 5383 return (ENOMEM); 5384 5385 t4_read_cong_tbl(sc, incr); 5386 5387 for (i = 0; i < NCCTRL_WIN; ++i) { 5388 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 5389 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 5390 incr[5][i], incr[6][i], incr[7][i]); 5391 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 5392 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 5393 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 5394 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 5395 } 5396 5397 rc = sbuf_finish(sb); 5398 sbuf_delete(sb); 5399 5400 return (rc); 5401 } 5402 5403 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 5404 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 5405 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 5406 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 5407 }; 5408 5409 static int 5410 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 5411 { 5412 struct adapter *sc = arg1; 5413 struct sbuf *sb; 5414 int rc, i, n, qid = arg2; 5415 uint32_t *buf, *p; 5416 char *qtype; 5417 u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5; 5418 5419 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 5420 ("%s: bad qid %d\n", __func__, qid)); 5421 5422 if (qid < CIM_NUM_IBQ) { 5423 /* inbound queue */ 5424 qtype = "IBQ"; 5425 n = 4 * CIM_IBQ_SIZE; 5426 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5427 rc = t4_read_cim_ibq(sc, qid, buf, n); 5428 } else { 5429 /* outbound queue */ 5430 qtype = "OBQ"; 5431 qid -= CIM_NUM_IBQ; 5432 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 5433 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5434 rc = t4_read_cim_obq(sc, qid, buf, n); 5435 } 5436 5437 if (rc < 0) { 5438 rc = -rc; 5439 goto done; 5440 } 5441 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 5442 5443 rc = sysctl_wire_old_buffer(req, 0); 5444 if (rc != 0) 5445 goto done; 5446 5447 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5448 if (sb == NULL) { 5449 rc = ENOMEM; 5450 goto done; 5451 } 5452 5453 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 5454 for (i = 0, p = buf; i < n; i += 16, p += 4) 5455 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 5456 p[2], p[3]); 5457 5458 rc = sbuf_finish(sb); 5459 sbuf_delete(sb); 5460 done: 5461 free(buf, M_CXGBE); 5462 return (rc); 5463 } 5464 5465 static int 5466 sysctl_cim_la(SYSCTL_HANDLER_ARGS) 5467 { 5468 struct adapter *sc = arg1; 5469 u_int cfg; 5470 struct sbuf *sb; 5471 uint32_t *buf, *p; 5472 int rc; 5473 5474 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 5475 if (rc != 0) 5476 return (rc); 5477 5478 rc = sysctl_wire_old_buffer(req, 0); 5479 if (rc != 0) 5480 return (rc); 5481 5482 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5483 if (sb == NULL) 5484 return (ENOMEM); 5485 5486 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 5487 M_ZERO | M_WAITOK); 5488 5489 rc = -t4_cim_read_la(sc, buf, NULL); 5490 if (rc != 0) 5491 goto done; 5492 5493 sbuf_printf(sb, "Status Data PC%s", 5494 cfg & F_UPDBGLACAPTPCONLY ? "" : 5495 " LS0Stat LS0Addr LS0Data"); 5496 5497 KASSERT((sc->params.cim_la_size & 7) == 0, 5498 ("%s: p will walk off the end of buf", __func__)); 5499 5500 for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) { 5501 if (cfg & F_UPDBGLACAPTPCONLY) { 5502 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 5503 p[6], p[7]); 5504 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 5505 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 5506 p[4] & 0xff, p[5] >> 8); 5507 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 5508 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5509 p[1] & 0xf, p[2] >> 4); 5510 } else { 5511 sbuf_printf(sb, 5512 "\n %02x %x%07x %x%07x %08x %08x " 5513 "%08x%08x%08x%08x", 5514 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5515 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 5516 p[6], p[7]); 5517 } 5518 } 5519 5520 rc = sbuf_finish(sb); 5521 sbuf_delete(sb); 5522 done: 5523 free(buf, M_CXGBE); 5524 return (rc); 5525 } 5526 5527 static int 5528 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) 5529 { 5530 struct adapter *sc = arg1; 5531 u_int i; 5532 struct sbuf *sb; 5533 uint32_t *buf, *p; 5534 int rc; 5535 5536 rc = sysctl_wire_old_buffer(req, 0); 5537 if (rc != 0) 5538 return (rc); 5539 5540 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5541 if (sb == NULL) 5542 return (ENOMEM); 5543 5544 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, 5545 M_ZERO | M_WAITOK); 5546 5547 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); 5548 p = buf; 5549 5550 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5551 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], 5552 p[1], p[0]); 5553 } 5554 5555 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); 5556 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5557 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", 5558 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, 5559 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, 5560 (p[1] >> 2) | ((p[2] & 3) << 30), 5561 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, 5562 p[0] & 1); 5563 } 5564 5565 rc = sbuf_finish(sb); 5566 sbuf_delete(sb); 5567 free(buf, M_CXGBE); 5568 return (rc); 5569 } 5570 5571 static int 5572 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) 5573 { 5574 struct adapter *sc = arg1; 5575 u_int i; 5576 struct sbuf *sb; 5577 uint32_t *buf, *p; 5578 int rc; 5579 5580 rc = sysctl_wire_old_buffer(req, 0); 5581 if (rc != 0) 5582 return (rc); 5583 5584 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5585 if (sb == NULL) 5586 return (ENOMEM); 5587 5588 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, 5589 M_ZERO | M_WAITOK); 5590 5591 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); 5592 p = buf; 5593 5594 sbuf_printf(sb, "Cntl ID DataBE Addr Data"); 5595 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5596 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", 5597 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, 5598 p[4], p[3], p[2], p[1], p[0]); 5599 } 5600 5601 sbuf_printf(sb, "\n\nCntl ID Data"); 5602 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5603 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", 5604 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); 5605 } 5606 5607 rc = sbuf_finish(sb); 5608 sbuf_delete(sb); 5609 free(buf, M_CXGBE); 5610 return (rc); 5611 } 5612 5613 static int 5614 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 5615 { 5616 struct adapter *sc = arg1; 5617 struct sbuf *sb; 5618 int rc, i; 5619 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5620 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5621 uint16_t thres[CIM_NUM_IBQ]; 5622 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 5623 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 5624 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 5625 5626 if (is_t4(sc)) { 5627 cim_num_obq = CIM_NUM_OBQ; 5628 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 5629 obq_rdaddr = A_UP_OBQ_0_REALADDR; 5630 } else { 5631 cim_num_obq = CIM_NUM_OBQ_T5; 5632 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 5633 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 5634 } 5635 nq = CIM_NUM_IBQ + cim_num_obq; 5636 5637 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 5638 if (rc == 0) 5639 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 5640 if (rc != 0) 5641 return (rc); 5642 5643 t4_read_cimq_cfg(sc, base, size, thres); 5644 5645 rc = sysctl_wire_old_buffer(req, 0); 5646 if (rc != 0) 5647 return (rc); 5648 5649 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5650 if (sb == NULL) 5651 return (ENOMEM); 5652 5653 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 5654 5655 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 5656 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 5657 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 5658 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5659 G_QUEREMFLITS(p[2]) * 16); 5660 for ( ; i < nq; i++, p += 4, wr += 2) 5661 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 5662 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 5663 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5664 G_QUEREMFLITS(p[2]) * 16); 5665 5666 rc = sbuf_finish(sb); 5667 sbuf_delete(sb); 5668 5669 return (rc); 5670 } 5671 5672 static int 5673 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 5674 { 5675 struct adapter *sc = arg1; 5676 struct sbuf *sb; 5677 int rc; 5678 struct tp_cpl_stats stats; 5679 5680 rc = sysctl_wire_old_buffer(req, 0); 5681 if (rc != 0) 5682 return (rc); 5683 5684 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5685 if (sb == NULL) 5686 return (ENOMEM); 5687 5688 t4_tp_get_cpl_stats(sc, &stats); 5689 5690 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 5691 "channel 3\n"); 5692 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 5693 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 5694 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 5695 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 5696 5697 rc = sbuf_finish(sb); 5698 sbuf_delete(sb); 5699 5700 return (rc); 5701 } 5702 5703 static int 5704 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 5705 { 5706 struct adapter *sc = arg1; 5707 struct sbuf *sb; 5708 int rc; 5709 struct tp_usm_stats stats; 5710 5711 rc = sysctl_wire_old_buffer(req, 0); 5712 if (rc != 0) 5713 return(rc); 5714 5715 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5716 if (sb == NULL) 5717 return (ENOMEM); 5718 5719 t4_get_usm_stats(sc, &stats); 5720 5721 sbuf_printf(sb, "Frames: %u\n", stats.frames); 5722 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 5723 sbuf_printf(sb, "Drops: %u", stats.drops); 5724 5725 rc = sbuf_finish(sb); 5726 sbuf_delete(sb); 5727 5728 return (rc); 5729 } 5730 5731 const char *devlog_level_strings[] = { 5732 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 5733 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 5734 [FW_DEVLOG_LEVEL_ERR] = "ERR", 5735 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 5736 [FW_DEVLOG_LEVEL_INFO] = "INFO", 5737 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 5738 }; 5739 5740 const char *devlog_facility_strings[] = { 5741 [FW_DEVLOG_FACILITY_CORE] = "CORE", 5742 [FW_DEVLOG_FACILITY_CF] = "CF", 5743 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 5744 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 5745 [FW_DEVLOG_FACILITY_RES] = "RES", 5746 [FW_DEVLOG_FACILITY_HW] = "HW", 5747 [FW_DEVLOG_FACILITY_FLR] = "FLR", 5748 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 5749 [FW_DEVLOG_FACILITY_PHY] = "PHY", 5750 [FW_DEVLOG_FACILITY_MAC] = "MAC", 5751 [FW_DEVLOG_FACILITY_PORT] = "PORT", 5752 [FW_DEVLOG_FACILITY_VI] = "VI", 5753 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 5754 [FW_DEVLOG_FACILITY_ACL] = "ACL", 5755 [FW_DEVLOG_FACILITY_TM] = "TM", 5756 [FW_DEVLOG_FACILITY_QFC] = "QFC", 5757 [FW_DEVLOG_FACILITY_DCB] = "DCB", 5758 [FW_DEVLOG_FACILITY_ETH] = "ETH", 5759 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 5760 [FW_DEVLOG_FACILITY_RI] = "RI", 5761 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 5762 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 5763 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 5764 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 5765 }; 5766 5767 static int 5768 sysctl_devlog(SYSCTL_HANDLER_ARGS) 5769 { 5770 struct adapter *sc = arg1; 5771 struct devlog_params *dparams = &sc->params.devlog; 5772 struct fw_devlog_e *buf, *e; 5773 int i, j, rc, nentries, first = 0, m; 5774 struct sbuf *sb; 5775 uint64_t ftstamp = UINT64_MAX; 5776 5777 if (dparams->start == 0) { 5778 dparams->memtype = FW_MEMTYPE_EDC0; 5779 dparams->start = 0x84000; 5780 dparams->size = 32768; 5781 } 5782 5783 nentries = dparams->size / sizeof(struct fw_devlog_e); 5784 5785 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 5786 if (buf == NULL) 5787 return (ENOMEM); 5788 5789 m = fwmtype_to_hwmtype(dparams->memtype); 5790 rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf); 5791 if (rc != 0) 5792 goto done; 5793 5794 for (i = 0; i < nentries; i++) { 5795 e = &buf[i]; 5796 5797 if (e->timestamp == 0) 5798 break; /* end */ 5799 5800 e->timestamp = be64toh(e->timestamp); 5801 e->seqno = be32toh(e->seqno); 5802 for (j = 0; j < 8; j++) 5803 e->params[j] = be32toh(e->params[j]); 5804 5805 if (e->timestamp < ftstamp) { 5806 ftstamp = e->timestamp; 5807 first = i; 5808 } 5809 } 5810 5811 if (buf[first].timestamp == 0) 5812 goto done; /* nothing in the log */ 5813 5814 rc = sysctl_wire_old_buffer(req, 0); 5815 if (rc != 0) 5816 goto done; 5817 5818 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5819 if (sb == NULL) { 5820 rc = ENOMEM; 5821 goto done; 5822 } 5823 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 5824 "Seq#", "Tstamp", "Level", "Facility", "Message"); 5825 5826 i = first; 5827 do { 5828 e = &buf[i]; 5829 if (e->timestamp == 0) 5830 break; /* end */ 5831 5832 sbuf_printf(sb, "%10d %15ju %8s %8s ", 5833 e->seqno, e->timestamp, 5834 (e->level < nitems(devlog_level_strings) ? 5835 devlog_level_strings[e->level] : "UNKNOWN"), 5836 (e->facility < nitems(devlog_facility_strings) ? 5837 devlog_facility_strings[e->facility] : "UNKNOWN")); 5838 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 5839 e->params[2], e->params[3], e->params[4], 5840 e->params[5], e->params[6], e->params[7]); 5841 5842 if (++i == nentries) 5843 i = 0; 5844 } while (i != first); 5845 5846 rc = sbuf_finish(sb); 5847 sbuf_delete(sb); 5848 done: 5849 free(buf, M_CXGBE); 5850 return (rc); 5851 } 5852 5853 static int 5854 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 5855 { 5856 struct adapter *sc = arg1; 5857 struct sbuf *sb; 5858 int rc; 5859 struct tp_fcoe_stats stats[4]; 5860 5861 rc = sysctl_wire_old_buffer(req, 0); 5862 if (rc != 0) 5863 return (rc); 5864 5865 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5866 if (sb == NULL) 5867 return (ENOMEM); 5868 5869 t4_get_fcoe_stats(sc, 0, &stats[0]); 5870 t4_get_fcoe_stats(sc, 1, &stats[1]); 5871 t4_get_fcoe_stats(sc, 2, &stats[2]); 5872 t4_get_fcoe_stats(sc, 3, &stats[3]); 5873 5874 sbuf_printf(sb, " channel 0 channel 1 " 5875 "channel 2 channel 3\n"); 5876 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 5877 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 5878 stats[3].octetsDDP); 5879 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 5880 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 5881 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 5882 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 5883 stats[3].framesDrop); 5884 5885 rc = sbuf_finish(sb); 5886 sbuf_delete(sb); 5887 5888 return (rc); 5889 } 5890 5891 static int 5892 sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 5893 { 5894 struct adapter *sc = arg1; 5895 struct sbuf *sb; 5896 int rc, i; 5897 unsigned int map, kbps, ipg, mode; 5898 unsigned int pace_tab[NTX_SCHED]; 5899 5900 rc = sysctl_wire_old_buffer(req, 0); 5901 if (rc != 0) 5902 return (rc); 5903 5904 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5905 if (sb == NULL) 5906 return (ENOMEM); 5907 5908 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 5909 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 5910 t4_read_pace_tbl(sc, pace_tab); 5911 5912 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 5913 "Class IPG (0.1 ns) Flow IPG (us)"); 5914 5915 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 5916 t4_get_tx_sched(sc, i, &kbps, &ipg); 5917 sbuf_printf(sb, "\n %u %-5s %u ", i, 5918 (mode & (1 << i)) ? "flow" : "class", map & 3); 5919 if (kbps) 5920 sbuf_printf(sb, "%9u ", kbps); 5921 else 5922 sbuf_printf(sb, " disabled "); 5923 5924 if (ipg) 5925 sbuf_printf(sb, "%13u ", ipg); 5926 else 5927 sbuf_printf(sb, " disabled "); 5928 5929 if (pace_tab[i]) 5930 sbuf_printf(sb, "%10u", pace_tab[i]); 5931 else 5932 sbuf_printf(sb, " disabled"); 5933 } 5934 5935 rc = sbuf_finish(sb); 5936 sbuf_delete(sb); 5937 5938 return (rc); 5939 } 5940 5941 static int 5942 sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 5943 { 5944 struct adapter *sc = arg1; 5945 struct sbuf *sb; 5946 int rc, i, j; 5947 uint64_t *p0, *p1; 5948 struct lb_port_stats s[2]; 5949 static const char *stat_name[] = { 5950 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 5951 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 5952 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 5953 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 5954 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 5955 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 5956 "BG2FramesTrunc:", "BG3FramesTrunc:" 5957 }; 5958 5959 rc = sysctl_wire_old_buffer(req, 0); 5960 if (rc != 0) 5961 return (rc); 5962 5963 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5964 if (sb == NULL) 5965 return (ENOMEM); 5966 5967 memset(s, 0, sizeof(s)); 5968 5969 for (i = 0; i < 4; i += 2) { 5970 t4_get_lb_stats(sc, i, &s[0]); 5971 t4_get_lb_stats(sc, i + 1, &s[1]); 5972 5973 p0 = &s[0].octets; 5974 p1 = &s[1].octets; 5975 sbuf_printf(sb, "%s Loopback %u" 5976 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 5977 5978 for (j = 0; j < nitems(stat_name); j++) 5979 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 5980 *p0++, *p1++); 5981 } 5982 5983 rc = sbuf_finish(sb); 5984 sbuf_delete(sb); 5985 5986 return (rc); 5987 } 5988 5989 static int 5990 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) 5991 { 5992 int rc = 0; 5993 struct port_info *pi = arg1; 5994 struct sbuf *sb; 5995 static const char *linkdnreasons[] = { 5996 "non-specific", "remote fault", "autoneg failed", "reserved3", 5997 "PHY overheated", "unknown", "rx los", "reserved7" 5998 }; 5999 6000 rc = sysctl_wire_old_buffer(req, 0); 6001 if (rc != 0) 6002 return(rc); 6003 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); 6004 if (sb == NULL) 6005 return (ENOMEM); 6006 6007 if (pi->linkdnrc < 0) 6008 sbuf_printf(sb, "n/a"); 6009 else if (pi->linkdnrc < nitems(linkdnreasons)) 6010 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]); 6011 else 6012 sbuf_printf(sb, "%d", pi->linkdnrc); 6013 6014 rc = sbuf_finish(sb); 6015 sbuf_delete(sb); 6016 6017 return (rc); 6018 } 6019 6020 struct mem_desc { 6021 unsigned int base; 6022 unsigned int limit; 6023 unsigned int idx; 6024 }; 6025 6026 static int 6027 mem_desc_cmp(const void *a, const void *b) 6028 { 6029 return ((const struct mem_desc *)a)->base - 6030 ((const struct mem_desc *)b)->base; 6031 } 6032 6033 static void 6034 mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 6035 unsigned int to) 6036 { 6037 unsigned int size; 6038 6039 size = to - from + 1; 6040 if (size == 0) 6041 return; 6042 6043 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 6044 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 6045 } 6046 6047 static int 6048 sysctl_meminfo(SYSCTL_HANDLER_ARGS) 6049 { 6050 struct adapter *sc = arg1; 6051 struct sbuf *sb; 6052 int rc, i, n; 6053 uint32_t lo, hi, used, alloc; 6054 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 6055 static const char *region[] = { 6056 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 6057 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 6058 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 6059 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 6060 "RQUDP region:", "PBL region:", "TXPBL region:", 6061 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 6062 "On-chip queues:" 6063 }; 6064 struct mem_desc avail[4]; 6065 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 6066 struct mem_desc *md = mem; 6067 6068 rc = sysctl_wire_old_buffer(req, 0); 6069 if (rc != 0) 6070 return (rc); 6071 6072 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6073 if (sb == NULL) 6074 return (ENOMEM); 6075 6076 for (i = 0; i < nitems(mem); i++) { 6077 mem[i].limit = 0; 6078 mem[i].idx = i; 6079 } 6080 6081 /* Find and sort the populated memory ranges */ 6082 i = 0; 6083 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 6084 if (lo & F_EDRAM0_ENABLE) { 6085 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 6086 avail[i].base = G_EDRAM0_BASE(hi) << 20; 6087 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 6088 avail[i].idx = 0; 6089 i++; 6090 } 6091 if (lo & F_EDRAM1_ENABLE) { 6092 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 6093 avail[i].base = G_EDRAM1_BASE(hi) << 20; 6094 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 6095 avail[i].idx = 1; 6096 i++; 6097 } 6098 if (lo & F_EXT_MEM_ENABLE) { 6099 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 6100 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 6101 avail[i].limit = avail[i].base + 6102 (G_EXT_MEM_SIZE(hi) << 20); 6103 avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */ 6104 i++; 6105 } 6106 if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) { 6107 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 6108 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 6109 avail[i].limit = avail[i].base + 6110 (G_EXT_MEM1_SIZE(hi) << 20); 6111 avail[i].idx = 4; 6112 i++; 6113 } 6114 if (!i) /* no memory available */ 6115 return 0; 6116 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 6117 6118 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 6119 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 6120 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 6121 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 6122 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 6123 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 6124 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 6125 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 6126 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 6127 6128 /* the next few have explicit upper bounds */ 6129 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 6130 md->limit = md->base - 1 + 6131 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 6132 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 6133 md++; 6134 6135 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 6136 md->limit = md->base - 1 + 6137 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 6138 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 6139 md++; 6140 6141 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6142 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 6143 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 6144 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 6145 } else { 6146 md->base = 0; 6147 md->idx = nitems(region); /* hide it */ 6148 } 6149 md++; 6150 6151 #define ulp_region(reg) \ 6152 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 6153 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 6154 6155 ulp_region(RX_ISCSI); 6156 ulp_region(RX_TDDP); 6157 ulp_region(TX_TPT); 6158 ulp_region(RX_STAG); 6159 ulp_region(RX_RQ); 6160 ulp_region(RX_RQUDP); 6161 ulp_region(RX_PBL); 6162 ulp_region(TX_PBL); 6163 #undef ulp_region 6164 6165 md->base = 0; 6166 md->idx = nitems(region); 6167 if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) { 6168 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); 6169 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc, 6170 A_SGE_DBVFIFO_SIZE))) << 2) - 1; 6171 } 6172 md++; 6173 6174 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 6175 md->limit = md->base + sc->tids.ntids - 1; 6176 md++; 6177 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 6178 md->limit = md->base + sc->tids.ntids - 1; 6179 md++; 6180 6181 md->base = sc->vres.ocq.start; 6182 if (sc->vres.ocq.size) 6183 md->limit = md->base + sc->vres.ocq.size - 1; 6184 else 6185 md->idx = nitems(region); /* hide it */ 6186 md++; 6187 6188 /* add any address-space holes, there can be up to 3 */ 6189 for (n = 0; n < i - 1; n++) 6190 if (avail[n].limit < avail[n + 1].base) 6191 (md++)->base = avail[n].limit; 6192 if (avail[n].limit) 6193 (md++)->base = avail[n].limit; 6194 6195 n = md - mem; 6196 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 6197 6198 for (lo = 0; lo < i; lo++) 6199 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 6200 avail[lo].limit - 1); 6201 6202 sbuf_printf(sb, "\n"); 6203 for (i = 0; i < n; i++) { 6204 if (mem[i].idx >= nitems(region)) 6205 continue; /* skip holes */ 6206 if (!mem[i].limit) 6207 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 6208 mem_region_show(sb, region[mem[i].idx], mem[i].base, 6209 mem[i].limit); 6210 } 6211 6212 sbuf_printf(sb, "\n"); 6213 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 6214 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 6215 mem_region_show(sb, "uP RAM:", lo, hi); 6216 6217 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 6218 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 6219 mem_region_show(sb, "uP Extmem2:", lo, hi); 6220 6221 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 6222 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 6223 G_PMRXMAXPAGE(lo), 6224 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 6225 (lo & F_PMRXNUMCHN) ? 2 : 1); 6226 6227 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 6228 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 6229 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 6230 G_PMTXMAXPAGE(lo), 6231 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 6232 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 6233 sbuf_printf(sb, "%u p-structs\n", 6234 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 6235 6236 for (i = 0; i < 4; i++) { 6237 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 6238 if (is_t4(sc)) { 6239 used = G_USED(lo); 6240 alloc = G_ALLOC(lo); 6241 } else { 6242 used = G_T5_USED(lo); 6243 alloc = G_T5_ALLOC(lo); 6244 } 6245 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 6246 i, used, alloc); 6247 } 6248 for (i = 0; i < 4; i++) { 6249 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 6250 if (is_t4(sc)) { 6251 used = G_USED(lo); 6252 alloc = G_ALLOC(lo); 6253 } else { 6254 used = G_T5_USED(lo); 6255 alloc = G_T5_ALLOC(lo); 6256 } 6257 sbuf_printf(sb, 6258 "\nLoopback %d using %u pages out of %u allocated", 6259 i, used, alloc); 6260 } 6261 6262 rc = sbuf_finish(sb); 6263 sbuf_delete(sb); 6264 6265 return (rc); 6266 } 6267 6268 static inline void 6269 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) 6270 { 6271 *mask = x | y; 6272 y = htobe64(y); 6273 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); 6274 } 6275 6276 static int 6277 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) 6278 { 6279 struct adapter *sc = arg1; 6280 struct sbuf *sb; 6281 int rc, i, n; 6282 6283 rc = sysctl_wire_old_buffer(req, 0); 6284 if (rc != 0) 6285 return (rc); 6286 6287 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6288 if (sb == NULL) 6289 return (ENOMEM); 6290 6291 sbuf_printf(sb, 6292 "Idx Ethernet address Mask Vld Ports PF" 6293 " VF Replication P0 P1 P2 P3 ML"); 6294 n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES : 6295 NUM_MPS_T5_CLS_SRAM_L_INSTANCES; 6296 for (i = 0; i < n; i++) { 6297 uint64_t tcamx, tcamy, mask; 6298 uint32_t cls_lo, cls_hi; 6299 uint8_t addr[ETHER_ADDR_LEN]; 6300 6301 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); 6302 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); 6303 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 6304 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 6305 6306 if (tcamx & tcamy) 6307 continue; 6308 6309 tcamxy2valmask(tcamx, tcamy, addr, &mask); 6310 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" 6311 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], 6312 addr[3], addr[4], addr[5], (uintmax_t)mask, 6313 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', 6314 G_PORTMAP(cls_hi), G_PF(cls_lo), 6315 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); 6316 6317 if (cls_lo & F_REPLICATE) { 6318 struct fw_ldst_cmd ldst_cmd; 6319 6320 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 6321 ldst_cmd.op_to_addrspace = 6322 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 6323 F_FW_CMD_REQUEST | F_FW_CMD_READ | 6324 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 6325 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 6326 ldst_cmd.u.mps.fid_ctl = 6327 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 6328 V_FW_LDST_CMD_CTL(i)); 6329 6330 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 6331 "t4mps"); 6332 if (rc) 6333 break; 6334 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 6335 sizeof(ldst_cmd), &ldst_cmd); 6336 end_synchronized_op(sc, 0); 6337 6338 if (rc != 0) { 6339 sbuf_printf(sb, 6340 " ------------ error %3u ------------", rc); 6341 rc = 0; 6342 } else { 6343 sbuf_printf(sb, " %08x %08x %08x %08x", 6344 be32toh(ldst_cmd.u.mps.rplc127_96), 6345 be32toh(ldst_cmd.u.mps.rplc95_64), 6346 be32toh(ldst_cmd.u.mps.rplc63_32), 6347 be32toh(ldst_cmd.u.mps.rplc31_0)); 6348 } 6349 } else 6350 sbuf_printf(sb, "%36s", ""); 6351 6352 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), 6353 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), 6354 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); 6355 } 6356 6357 if (rc) 6358 (void) sbuf_finish(sb); 6359 else 6360 rc = sbuf_finish(sb); 6361 sbuf_delete(sb); 6362 6363 return (rc); 6364 } 6365 6366 static int 6367 sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 6368 { 6369 struct adapter *sc = arg1; 6370 struct sbuf *sb; 6371 int rc; 6372 uint16_t mtus[NMTUS]; 6373 6374 rc = sysctl_wire_old_buffer(req, 0); 6375 if (rc != 0) 6376 return (rc); 6377 6378 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6379 if (sb == NULL) 6380 return (ENOMEM); 6381 6382 t4_read_mtu_tbl(sc, mtus, NULL); 6383 6384 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 6385 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 6386 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 6387 mtus[14], mtus[15]); 6388 6389 rc = sbuf_finish(sb); 6390 sbuf_delete(sb); 6391 6392 return (rc); 6393 } 6394 6395 static int 6396 sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 6397 { 6398 struct adapter *sc = arg1; 6399 struct sbuf *sb; 6400 int rc, i; 6401 uint32_t cnt[PM_NSTATS]; 6402 uint64_t cyc[PM_NSTATS]; 6403 static const char *rx_stats[] = { 6404 "Read:", "Write bypass:", "Write mem:", "Flush:" 6405 }; 6406 static const char *tx_stats[] = { 6407 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:" 6408 }; 6409 6410 rc = sysctl_wire_old_buffer(req, 0); 6411 if (rc != 0) 6412 return (rc); 6413 6414 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6415 if (sb == NULL) 6416 return (ENOMEM); 6417 6418 t4_pmtx_get_stats(sc, cnt, cyc); 6419 sbuf_printf(sb, " Tx pcmds Tx bytes"); 6420 for (i = 0; i < ARRAY_SIZE(tx_stats); i++) 6421 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i], 6422 cyc[i]); 6423 6424 t4_pmrx_get_stats(sc, cnt, cyc); 6425 sbuf_printf(sb, "\n Rx pcmds Rx bytes"); 6426 for (i = 0; i < ARRAY_SIZE(rx_stats); i++) 6427 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i], 6428 cyc[i]); 6429 6430 rc = sbuf_finish(sb); 6431 sbuf_delete(sb); 6432 6433 return (rc); 6434 } 6435 6436 static int 6437 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 6438 { 6439 struct adapter *sc = arg1; 6440 struct sbuf *sb; 6441 int rc; 6442 struct tp_rdma_stats stats; 6443 6444 rc = sysctl_wire_old_buffer(req, 0); 6445 if (rc != 0) 6446 return (rc); 6447 6448 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6449 if (sb == NULL) 6450 return (ENOMEM); 6451 6452 t4_tp_get_rdma_stats(sc, &stats); 6453 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 6454 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 6455 6456 rc = sbuf_finish(sb); 6457 sbuf_delete(sb); 6458 6459 return (rc); 6460 } 6461 6462 static int 6463 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 6464 { 6465 struct adapter *sc = arg1; 6466 struct sbuf *sb; 6467 int rc; 6468 struct tp_tcp_stats v4, v6; 6469 6470 rc = sysctl_wire_old_buffer(req, 0); 6471 if (rc != 0) 6472 return (rc); 6473 6474 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6475 if (sb == NULL) 6476 return (ENOMEM); 6477 6478 t4_tp_get_tcp_stats(sc, &v4, &v6); 6479 sbuf_printf(sb, 6480 " IP IPv6\n"); 6481 sbuf_printf(sb, "OutRsts: %20u %20u\n", 6482 v4.tcpOutRsts, v6.tcpOutRsts); 6483 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 6484 v4.tcpInSegs, v6.tcpInSegs); 6485 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 6486 v4.tcpOutSegs, v6.tcpOutSegs); 6487 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 6488 v4.tcpRetransSegs, v6.tcpRetransSegs); 6489 6490 rc = sbuf_finish(sb); 6491 sbuf_delete(sb); 6492 6493 return (rc); 6494 } 6495 6496 static int 6497 sysctl_tids(SYSCTL_HANDLER_ARGS) 6498 { 6499 struct adapter *sc = arg1; 6500 struct sbuf *sb; 6501 int rc; 6502 struct tid_info *t = &sc->tids; 6503 6504 rc = sysctl_wire_old_buffer(req, 0); 6505 if (rc != 0) 6506 return (rc); 6507 6508 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6509 if (sb == NULL) 6510 return (ENOMEM); 6511 6512 if (t->natids) { 6513 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 6514 t->atids_in_use); 6515 } 6516 6517 if (t->ntids) { 6518 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6519 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 6520 6521 if (b) { 6522 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 6523 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6524 t->ntids - 1); 6525 } else { 6526 sbuf_printf(sb, "TID range: %u-%u", 6527 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6528 t->ntids - 1); 6529 } 6530 } else 6531 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 6532 sbuf_printf(sb, ", in use: %u\n", 6533 atomic_load_acq_int(&t->tids_in_use)); 6534 } 6535 6536 if (t->nstids) { 6537 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 6538 t->stid_base + t->nstids - 1, t->stids_in_use); 6539 } 6540 6541 if (t->nftids) { 6542 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 6543 t->ftid_base + t->nftids - 1); 6544 } 6545 6546 if (t->netids) { 6547 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base, 6548 t->etid_base + t->netids - 1); 6549 } 6550 6551 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 6552 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 6553 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 6554 6555 rc = sbuf_finish(sb); 6556 sbuf_delete(sb); 6557 6558 return (rc); 6559 } 6560 6561 static int 6562 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 6563 { 6564 struct adapter *sc = arg1; 6565 struct sbuf *sb; 6566 int rc; 6567 struct tp_err_stats stats; 6568 6569 rc = sysctl_wire_old_buffer(req, 0); 6570 if (rc != 0) 6571 return (rc); 6572 6573 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6574 if (sb == NULL) 6575 return (ENOMEM); 6576 6577 t4_tp_get_err_stats(sc, &stats); 6578 6579 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6580 "channel 3\n"); 6581 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 6582 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 6583 stats.macInErrs[3]); 6584 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 6585 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 6586 stats.hdrInErrs[3]); 6587 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 6588 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 6589 stats.tcpInErrs[3]); 6590 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 6591 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 6592 stats.tcp6InErrs[3]); 6593 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 6594 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 6595 stats.tnlCongDrops[3]); 6596 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 6597 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 6598 stats.tnlTxDrops[3]); 6599 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 6600 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 6601 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 6602 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 6603 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 6604 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 6605 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 6606 stats.ofldNoNeigh, stats.ofldCongDefer); 6607 6608 rc = sbuf_finish(sb); 6609 sbuf_delete(sb); 6610 6611 return (rc); 6612 } 6613 6614 struct field_desc { 6615 const char *name; 6616 u_int start; 6617 u_int width; 6618 }; 6619 6620 static void 6621 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) 6622 { 6623 char buf[32]; 6624 int line_size = 0; 6625 6626 while (f->name) { 6627 uint64_t mask = (1ULL << f->width) - 1; 6628 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, 6629 ((uintmax_t)v >> f->start) & mask); 6630 6631 if (line_size + len >= 79) { 6632 line_size = 8; 6633 sbuf_printf(sb, "\n "); 6634 } 6635 sbuf_printf(sb, "%s ", buf); 6636 line_size += len + 1; 6637 f++; 6638 } 6639 sbuf_printf(sb, "\n"); 6640 } 6641 6642 static struct field_desc tp_la0[] = { 6643 { "RcfOpCodeOut", 60, 4 }, 6644 { "State", 56, 4 }, 6645 { "WcfState", 52, 4 }, 6646 { "RcfOpcSrcOut", 50, 2 }, 6647 { "CRxError", 49, 1 }, 6648 { "ERxError", 48, 1 }, 6649 { "SanityFailed", 47, 1 }, 6650 { "SpuriousMsg", 46, 1 }, 6651 { "FlushInputMsg", 45, 1 }, 6652 { "FlushInputCpl", 44, 1 }, 6653 { "RssUpBit", 43, 1 }, 6654 { "RssFilterHit", 42, 1 }, 6655 { "Tid", 32, 10 }, 6656 { "InitTcb", 31, 1 }, 6657 { "LineNumber", 24, 7 }, 6658 { "Emsg", 23, 1 }, 6659 { "EdataOut", 22, 1 }, 6660 { "Cmsg", 21, 1 }, 6661 { "CdataOut", 20, 1 }, 6662 { "EreadPdu", 19, 1 }, 6663 { "CreadPdu", 18, 1 }, 6664 { "TunnelPkt", 17, 1 }, 6665 { "RcfPeerFin", 16, 1 }, 6666 { "RcfReasonOut", 12, 4 }, 6667 { "TxCchannel", 10, 2 }, 6668 { "RcfTxChannel", 8, 2 }, 6669 { "RxEchannel", 6, 2 }, 6670 { "RcfRxChannel", 5, 1 }, 6671 { "RcfDataOutSrdy", 4, 1 }, 6672 { "RxDvld", 3, 1 }, 6673 { "RxOoDvld", 2, 1 }, 6674 { "RxCongestion", 1, 1 }, 6675 { "TxCongestion", 0, 1 }, 6676 { NULL } 6677 }; 6678 6679 static struct field_desc tp_la1[] = { 6680 { "CplCmdIn", 56, 8 }, 6681 { "CplCmdOut", 48, 8 }, 6682 { "ESynOut", 47, 1 }, 6683 { "EAckOut", 46, 1 }, 6684 { "EFinOut", 45, 1 }, 6685 { "ERstOut", 44, 1 }, 6686 { "SynIn", 43, 1 }, 6687 { "AckIn", 42, 1 }, 6688 { "FinIn", 41, 1 }, 6689 { "RstIn", 40, 1 }, 6690 { "DataIn", 39, 1 }, 6691 { "DataInVld", 38, 1 }, 6692 { "PadIn", 37, 1 }, 6693 { "RxBufEmpty", 36, 1 }, 6694 { "RxDdp", 35, 1 }, 6695 { "RxFbCongestion", 34, 1 }, 6696 { "TxFbCongestion", 33, 1 }, 6697 { "TxPktSumSrdy", 32, 1 }, 6698 { "RcfUlpType", 28, 4 }, 6699 { "Eread", 27, 1 }, 6700 { "Ebypass", 26, 1 }, 6701 { "Esave", 25, 1 }, 6702 { "Static0", 24, 1 }, 6703 { "Cread", 23, 1 }, 6704 { "Cbypass", 22, 1 }, 6705 { "Csave", 21, 1 }, 6706 { "CPktOut", 20, 1 }, 6707 { "RxPagePoolFull", 18, 2 }, 6708 { "RxLpbkPkt", 17, 1 }, 6709 { "TxLpbkPkt", 16, 1 }, 6710 { "RxVfValid", 15, 1 }, 6711 { "SynLearned", 14, 1 }, 6712 { "SetDelEntry", 13, 1 }, 6713 { "SetInvEntry", 12, 1 }, 6714 { "CpcmdDvld", 11, 1 }, 6715 { "CpcmdSave", 10, 1 }, 6716 { "RxPstructsFull", 8, 2 }, 6717 { "EpcmdDvld", 7, 1 }, 6718 { "EpcmdFlush", 6, 1 }, 6719 { "EpcmdTrimPrefix", 5, 1 }, 6720 { "EpcmdTrimPostfix", 4, 1 }, 6721 { "ERssIp4Pkt", 3, 1 }, 6722 { "ERssIp6Pkt", 2, 1 }, 6723 { "ERssTcpUdpPkt", 1, 1 }, 6724 { "ERssFceFipPkt", 0, 1 }, 6725 { NULL } 6726 }; 6727 6728 static struct field_desc tp_la2[] = { 6729 { "CplCmdIn", 56, 8 }, 6730 { "MpsVfVld", 55, 1 }, 6731 { "MpsPf", 52, 3 }, 6732 { "MpsVf", 44, 8 }, 6733 { "SynIn", 43, 1 }, 6734 { "AckIn", 42, 1 }, 6735 { "FinIn", 41, 1 }, 6736 { "RstIn", 40, 1 }, 6737 { "DataIn", 39, 1 }, 6738 { "DataInVld", 38, 1 }, 6739 { "PadIn", 37, 1 }, 6740 { "RxBufEmpty", 36, 1 }, 6741 { "RxDdp", 35, 1 }, 6742 { "RxFbCongestion", 34, 1 }, 6743 { "TxFbCongestion", 33, 1 }, 6744 { "TxPktSumSrdy", 32, 1 }, 6745 { "RcfUlpType", 28, 4 }, 6746 { "Eread", 27, 1 }, 6747 { "Ebypass", 26, 1 }, 6748 { "Esave", 25, 1 }, 6749 { "Static0", 24, 1 }, 6750 { "Cread", 23, 1 }, 6751 { "Cbypass", 22, 1 }, 6752 { "Csave", 21, 1 }, 6753 { "CPktOut", 20, 1 }, 6754 { "RxPagePoolFull", 18, 2 }, 6755 { "RxLpbkPkt", 17, 1 }, 6756 { "TxLpbkPkt", 16, 1 }, 6757 { "RxVfValid", 15, 1 }, 6758 { "SynLearned", 14, 1 }, 6759 { "SetDelEntry", 13, 1 }, 6760 { "SetInvEntry", 12, 1 }, 6761 { "CpcmdDvld", 11, 1 }, 6762 { "CpcmdSave", 10, 1 }, 6763 { "RxPstructsFull", 8, 2 }, 6764 { "EpcmdDvld", 7, 1 }, 6765 { "EpcmdFlush", 6, 1 }, 6766 { "EpcmdTrimPrefix", 5, 1 }, 6767 { "EpcmdTrimPostfix", 4, 1 }, 6768 { "ERssIp4Pkt", 3, 1 }, 6769 { "ERssIp6Pkt", 2, 1 }, 6770 { "ERssTcpUdpPkt", 1, 1 }, 6771 { "ERssFceFipPkt", 0, 1 }, 6772 { NULL } 6773 }; 6774 6775 static void 6776 tp_la_show(struct sbuf *sb, uint64_t *p, int idx) 6777 { 6778 6779 field_desc_show(sb, *p, tp_la0); 6780 } 6781 6782 static void 6783 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) 6784 { 6785 6786 if (idx) 6787 sbuf_printf(sb, "\n"); 6788 field_desc_show(sb, p[0], tp_la0); 6789 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6790 field_desc_show(sb, p[1], tp_la0); 6791 } 6792 6793 static void 6794 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) 6795 { 6796 6797 if (idx) 6798 sbuf_printf(sb, "\n"); 6799 field_desc_show(sb, p[0], tp_la0); 6800 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6801 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); 6802 } 6803 6804 static int 6805 sysctl_tp_la(SYSCTL_HANDLER_ARGS) 6806 { 6807 struct adapter *sc = arg1; 6808 struct sbuf *sb; 6809 uint64_t *buf, *p; 6810 int rc; 6811 u_int i, inc; 6812 void (*show_func)(struct sbuf *, uint64_t *, int); 6813 6814 rc = sysctl_wire_old_buffer(req, 0); 6815 if (rc != 0) 6816 return (rc); 6817 6818 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6819 if (sb == NULL) 6820 return (ENOMEM); 6821 6822 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); 6823 6824 t4_tp_read_la(sc, buf, NULL); 6825 p = buf; 6826 6827 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { 6828 case 2: 6829 inc = 2; 6830 show_func = tp_la_show2; 6831 break; 6832 case 3: 6833 inc = 2; 6834 show_func = tp_la_show3; 6835 break; 6836 default: 6837 inc = 1; 6838 show_func = tp_la_show; 6839 } 6840 6841 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) 6842 (*show_func)(sb, p, i); 6843 6844 rc = sbuf_finish(sb); 6845 sbuf_delete(sb); 6846 free(buf, M_CXGBE); 6847 return (rc); 6848 } 6849 6850 static int 6851 sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 6852 { 6853 struct adapter *sc = arg1; 6854 struct sbuf *sb; 6855 int rc; 6856 u64 nrate[NCHAN], orate[NCHAN]; 6857 6858 rc = sysctl_wire_old_buffer(req, 0); 6859 if (rc != 0) 6860 return (rc); 6861 6862 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6863 if (sb == NULL) 6864 return (ENOMEM); 6865 6866 t4_get_chan_txrate(sc, nrate, orate); 6867 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6868 "channel 3\n"); 6869 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 6870 nrate[0], nrate[1], nrate[2], nrate[3]); 6871 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 6872 orate[0], orate[1], orate[2], orate[3]); 6873 6874 rc = sbuf_finish(sb); 6875 sbuf_delete(sb); 6876 6877 return (rc); 6878 } 6879 6880 static int 6881 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) 6882 { 6883 struct adapter *sc = arg1; 6884 struct sbuf *sb; 6885 uint32_t *buf, *p; 6886 int rc, i; 6887 6888 rc = sysctl_wire_old_buffer(req, 0); 6889 if (rc != 0) 6890 return (rc); 6891 6892 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6893 if (sb == NULL) 6894 return (ENOMEM); 6895 6896 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, 6897 M_ZERO | M_WAITOK); 6898 6899 t4_ulprx_read_la(sc, buf); 6900 p = buf; 6901 6902 sbuf_printf(sb, " Pcmd Type Message" 6903 " Data"); 6904 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { 6905 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", 6906 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); 6907 } 6908 6909 rc = sbuf_finish(sb); 6910 sbuf_delete(sb); 6911 free(buf, M_CXGBE); 6912 return (rc); 6913 } 6914 6915 static int 6916 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 6917 { 6918 struct adapter *sc = arg1; 6919 struct sbuf *sb; 6920 int rc, v; 6921 6922 rc = sysctl_wire_old_buffer(req, 0); 6923 if (rc != 0) 6924 return (rc); 6925 6926 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6927 if (sb == NULL) 6928 return (ENOMEM); 6929 6930 v = t4_read_reg(sc, A_SGE_STAT_CFG); 6931 if (G_STATSOURCE_T5(v) == 7) { 6932 if (G_STATMODE(v) == 0) { 6933 sbuf_printf(sb, "total %d, incomplete %d", 6934 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6935 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6936 } else if (G_STATMODE(v) == 1) { 6937 sbuf_printf(sb, "total %d, data overflow %d", 6938 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6939 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6940 } 6941 } 6942 rc = sbuf_finish(sb); 6943 sbuf_delete(sb); 6944 6945 return (rc); 6946 } 6947 #endif 6948 6949 static inline void 6950 txq_start(struct ifnet *ifp, struct sge_txq *txq) 6951 { 6952 struct buf_ring *br; 6953 struct mbuf *m; 6954 6955 TXQ_LOCK_ASSERT_OWNED(txq); 6956 6957 br = txq->br; 6958 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 6959 if (m) 6960 t4_eth_tx(ifp, txq, m); 6961 } 6962 6963 void 6964 t4_tx_callout(void *arg) 6965 { 6966 struct sge_eq *eq = arg; 6967 struct adapter *sc; 6968 6969 if (EQ_TRYLOCK(eq) == 0) 6970 goto reschedule; 6971 6972 if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) { 6973 EQ_UNLOCK(eq); 6974 reschedule: 6975 if (__predict_true(!(eq->flags && EQ_DOOMED))) 6976 callout_schedule(&eq->tx_callout, 1); 6977 return; 6978 } 6979 6980 EQ_LOCK_ASSERT_OWNED(eq); 6981 6982 if (__predict_true((eq->flags & EQ_DOOMED) == 0)) { 6983 6984 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 6985 struct sge_txq *txq = arg; 6986 struct port_info *pi = txq->ifp->if_softc; 6987 6988 sc = pi->adapter; 6989 } else { 6990 struct sge_wrq *wrq = arg; 6991 6992 sc = wrq->adapter; 6993 } 6994 6995 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task); 6996 } 6997 6998 EQ_UNLOCK(eq); 6999 } 7000 7001 void 7002 t4_tx_task(void *arg, int count) 7003 { 7004 struct sge_eq *eq = arg; 7005 7006 EQ_LOCK(eq); 7007 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 7008 struct sge_txq *txq = arg; 7009 txq_start(txq->ifp, txq); 7010 } else { 7011 struct sge_wrq *wrq = arg; 7012 t4_wrq_tx_locked(wrq->adapter, wrq, NULL); 7013 } 7014 EQ_UNLOCK(eq); 7015 } 7016 7017 static uint32_t 7018 fconf_to_mode(uint32_t fconf) 7019 { 7020 uint32_t mode; 7021 7022 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 7023 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 7024 7025 if (fconf & F_FRAGMENTATION) 7026 mode |= T4_FILTER_IP_FRAGMENT; 7027 7028 if (fconf & F_MPSHITTYPE) 7029 mode |= T4_FILTER_MPS_HIT_TYPE; 7030 7031 if (fconf & F_MACMATCH) 7032 mode |= T4_FILTER_MAC_IDX; 7033 7034 if (fconf & F_ETHERTYPE) 7035 mode |= T4_FILTER_ETH_TYPE; 7036 7037 if (fconf & F_PROTOCOL) 7038 mode |= T4_FILTER_IP_PROTO; 7039 7040 if (fconf & F_TOS) 7041 mode |= T4_FILTER_IP_TOS; 7042 7043 if (fconf & F_VLAN) 7044 mode |= T4_FILTER_VLAN; 7045 7046 if (fconf & F_VNIC_ID) 7047 mode |= T4_FILTER_VNIC; 7048 7049 if (fconf & F_PORT) 7050 mode |= T4_FILTER_PORT; 7051 7052 if (fconf & F_FCOE) 7053 mode |= T4_FILTER_FCoE; 7054 7055 return (mode); 7056 } 7057 7058 static uint32_t 7059 mode_to_fconf(uint32_t mode) 7060 { 7061 uint32_t fconf = 0; 7062 7063 if (mode & T4_FILTER_IP_FRAGMENT) 7064 fconf |= F_FRAGMENTATION; 7065 7066 if (mode & T4_FILTER_MPS_HIT_TYPE) 7067 fconf |= F_MPSHITTYPE; 7068 7069 if (mode & T4_FILTER_MAC_IDX) 7070 fconf |= F_MACMATCH; 7071 7072 if (mode & T4_FILTER_ETH_TYPE) 7073 fconf |= F_ETHERTYPE; 7074 7075 if (mode & T4_FILTER_IP_PROTO) 7076 fconf |= F_PROTOCOL; 7077 7078 if (mode & T4_FILTER_IP_TOS) 7079 fconf |= F_TOS; 7080 7081 if (mode & T4_FILTER_VLAN) 7082 fconf |= F_VLAN; 7083 7084 if (mode & T4_FILTER_VNIC) 7085 fconf |= F_VNIC_ID; 7086 7087 if (mode & T4_FILTER_PORT) 7088 fconf |= F_PORT; 7089 7090 if (mode & T4_FILTER_FCoE) 7091 fconf |= F_FCOE; 7092 7093 return (fconf); 7094 } 7095 7096 static uint32_t 7097 fspec_to_fconf(struct t4_filter_specification *fs) 7098 { 7099 uint32_t fconf = 0; 7100 7101 if (fs->val.frag || fs->mask.frag) 7102 fconf |= F_FRAGMENTATION; 7103 7104 if (fs->val.matchtype || fs->mask.matchtype) 7105 fconf |= F_MPSHITTYPE; 7106 7107 if (fs->val.macidx || fs->mask.macidx) 7108 fconf |= F_MACMATCH; 7109 7110 if (fs->val.ethtype || fs->mask.ethtype) 7111 fconf |= F_ETHERTYPE; 7112 7113 if (fs->val.proto || fs->mask.proto) 7114 fconf |= F_PROTOCOL; 7115 7116 if (fs->val.tos || fs->mask.tos) 7117 fconf |= F_TOS; 7118 7119 if (fs->val.vlan_vld || fs->mask.vlan_vld) 7120 fconf |= F_VLAN; 7121 7122 if (fs->val.vnic_vld || fs->mask.vnic_vld) 7123 fconf |= F_VNIC_ID; 7124 7125 if (fs->val.iport || fs->mask.iport) 7126 fconf |= F_PORT; 7127 7128 if (fs->val.fcoe || fs->mask.fcoe) 7129 fconf |= F_FCOE; 7130 7131 return (fconf); 7132 } 7133 7134 static int 7135 get_filter_mode(struct adapter *sc, uint32_t *mode) 7136 { 7137 int rc; 7138 uint32_t fconf; 7139 7140 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7141 "t4getfm"); 7142 if (rc) 7143 return (rc); 7144 7145 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 7146 A_TP_VLAN_PRI_MAP); 7147 7148 if (sc->params.tp.vlan_pri_map != fconf) { 7149 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 7150 device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map, 7151 fconf); 7152 sc->params.tp.vlan_pri_map = fconf; 7153 } 7154 7155 *mode = fconf_to_mode(sc->params.tp.vlan_pri_map); 7156 7157 end_synchronized_op(sc, LOCK_HELD); 7158 return (0); 7159 } 7160 7161 static int 7162 set_filter_mode(struct adapter *sc, uint32_t mode) 7163 { 7164 uint32_t fconf; 7165 int rc; 7166 7167 fconf = mode_to_fconf(mode); 7168 7169 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7170 "t4setfm"); 7171 if (rc) 7172 return (rc); 7173 7174 if (sc->tids.ftids_in_use > 0) { 7175 rc = EBUSY; 7176 goto done; 7177 } 7178 7179 #ifdef TCP_OFFLOAD 7180 if (sc->offload_map) { 7181 rc = EBUSY; 7182 goto done; 7183 } 7184 #endif 7185 7186 #ifdef notyet 7187 rc = -t4_set_filter_mode(sc, fconf); 7188 if (rc == 0) 7189 sc->filter_mode = fconf; 7190 #else 7191 rc = ENOTSUP; 7192 #endif 7193 7194 done: 7195 end_synchronized_op(sc, LOCK_HELD); 7196 return (rc); 7197 } 7198 7199 static inline uint64_t 7200 get_filter_hits(struct adapter *sc, uint32_t fid) 7201 { 7202 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 7203 uint64_t hits; 7204 7205 memwin_info(sc, 0, &mw_base, NULL); 7206 off = position_memwin(sc, 0, 7207 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 7208 if (is_t4(sc)) { 7209 hits = t4_read_reg64(sc, mw_base + off + 16); 7210 hits = be64toh(hits); 7211 } else { 7212 hits = t4_read_reg(sc, mw_base + off + 24); 7213 hits = be32toh(hits); 7214 } 7215 7216 return (hits); 7217 } 7218 7219 static int 7220 get_filter(struct adapter *sc, struct t4_filter *t) 7221 { 7222 int i, rc, nfilters = sc->tids.nftids; 7223 struct filter_entry *f; 7224 7225 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7226 "t4getf"); 7227 if (rc) 7228 return (rc); 7229 7230 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 7231 t->idx >= nfilters) { 7232 t->idx = 0xffffffff; 7233 goto done; 7234 } 7235 7236 f = &sc->tids.ftid_tab[t->idx]; 7237 for (i = t->idx; i < nfilters; i++, f++) { 7238 if (f->valid) { 7239 t->idx = i; 7240 t->l2tidx = f->l2t ? f->l2t->idx : 0; 7241 t->smtidx = f->smtidx; 7242 if (f->fs.hitcnts) 7243 t->hits = get_filter_hits(sc, t->idx); 7244 else 7245 t->hits = UINT64_MAX; 7246 t->fs = f->fs; 7247 7248 goto done; 7249 } 7250 } 7251 7252 t->idx = 0xffffffff; 7253 done: 7254 end_synchronized_op(sc, LOCK_HELD); 7255 return (0); 7256 } 7257 7258 static int 7259 set_filter(struct adapter *sc, struct t4_filter *t) 7260 { 7261 unsigned int nfilters, nports; 7262 struct filter_entry *f; 7263 int i, rc; 7264 7265 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 7266 if (rc) 7267 return (rc); 7268 7269 nfilters = sc->tids.nftids; 7270 nports = sc->params.nports; 7271 7272 if (nfilters == 0) { 7273 rc = ENOTSUP; 7274 goto done; 7275 } 7276 7277 if (!(sc->flags & FULL_INIT_DONE)) { 7278 rc = EAGAIN; 7279 goto done; 7280 } 7281 7282 if (t->idx >= nfilters) { 7283 rc = EINVAL; 7284 goto done; 7285 } 7286 7287 /* Validate against the global filter mode */ 7288 if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) != 7289 sc->params.tp.vlan_pri_map) { 7290 rc = E2BIG; 7291 goto done; 7292 } 7293 7294 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 7295 rc = EINVAL; 7296 goto done; 7297 } 7298 7299 if (t->fs.val.iport >= nports) { 7300 rc = EINVAL; 7301 goto done; 7302 } 7303 7304 /* Can't specify an iq if not steering to it */ 7305 if (!t->fs.dirsteer && t->fs.iq) { 7306 rc = EINVAL; 7307 goto done; 7308 } 7309 7310 /* IPv6 filter idx must be 4 aligned */ 7311 if (t->fs.type == 1 && 7312 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 7313 rc = EINVAL; 7314 goto done; 7315 } 7316 7317 if (sc->tids.ftid_tab == NULL) { 7318 KASSERT(sc->tids.ftids_in_use == 0, 7319 ("%s: no memory allocated but filters_in_use > 0", 7320 __func__)); 7321 7322 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 7323 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 7324 if (sc->tids.ftid_tab == NULL) { 7325 rc = ENOMEM; 7326 goto done; 7327 } 7328 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 7329 } 7330 7331 for (i = 0; i < 4; i++) { 7332 f = &sc->tids.ftid_tab[t->idx + i]; 7333 7334 if (f->pending || f->valid) { 7335 rc = EBUSY; 7336 goto done; 7337 } 7338 if (f->locked) { 7339 rc = EPERM; 7340 goto done; 7341 } 7342 7343 if (t->fs.type == 0) 7344 break; 7345 } 7346 7347 f = &sc->tids.ftid_tab[t->idx]; 7348 f->fs = t->fs; 7349 7350 rc = set_filter_wr(sc, t->idx); 7351 done: 7352 end_synchronized_op(sc, 0); 7353 7354 if (rc == 0) { 7355 mtx_lock(&sc->tids.ftid_lock); 7356 for (;;) { 7357 if (f->pending == 0) { 7358 rc = f->valid ? 0 : EIO; 7359 break; 7360 } 7361 7362 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7363 PCATCH, "t4setfw", 0)) { 7364 rc = EINPROGRESS; 7365 break; 7366 } 7367 } 7368 mtx_unlock(&sc->tids.ftid_lock); 7369 } 7370 return (rc); 7371 } 7372 7373 static int 7374 del_filter(struct adapter *sc, struct t4_filter *t) 7375 { 7376 unsigned int nfilters; 7377 struct filter_entry *f; 7378 int rc; 7379 7380 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 7381 if (rc) 7382 return (rc); 7383 7384 nfilters = sc->tids.nftids; 7385 7386 if (nfilters == 0) { 7387 rc = ENOTSUP; 7388 goto done; 7389 } 7390 7391 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 7392 t->idx >= nfilters) { 7393 rc = EINVAL; 7394 goto done; 7395 } 7396 7397 if (!(sc->flags & FULL_INIT_DONE)) { 7398 rc = EAGAIN; 7399 goto done; 7400 } 7401 7402 f = &sc->tids.ftid_tab[t->idx]; 7403 7404 if (f->pending) { 7405 rc = EBUSY; 7406 goto done; 7407 } 7408 if (f->locked) { 7409 rc = EPERM; 7410 goto done; 7411 } 7412 7413 if (f->valid) { 7414 t->fs = f->fs; /* extra info for the caller */ 7415 rc = del_filter_wr(sc, t->idx); 7416 } 7417 7418 done: 7419 end_synchronized_op(sc, 0); 7420 7421 if (rc == 0) { 7422 mtx_lock(&sc->tids.ftid_lock); 7423 for (;;) { 7424 if (f->pending == 0) { 7425 rc = f->valid ? EIO : 0; 7426 break; 7427 } 7428 7429 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7430 PCATCH, "t4delfw", 0)) { 7431 rc = EINPROGRESS; 7432 break; 7433 } 7434 } 7435 mtx_unlock(&sc->tids.ftid_lock); 7436 } 7437 7438 return (rc); 7439 } 7440 7441 static void 7442 clear_filter(struct filter_entry *f) 7443 { 7444 if (f->l2t) 7445 t4_l2t_release(f->l2t); 7446 7447 bzero(f, sizeof (*f)); 7448 } 7449 7450 static int 7451 set_filter_wr(struct adapter *sc, int fidx) 7452 { 7453 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7454 struct wrqe *wr; 7455 struct fw_filter_wr *fwr; 7456 unsigned int ftid; 7457 7458 ASSERT_SYNCHRONIZED_OP(sc); 7459 7460 if (f->fs.newdmac || f->fs.newvlan) { 7461 /* This filter needs an L2T entry; allocate one. */ 7462 f->l2t = t4_l2t_alloc_switching(sc->l2t); 7463 if (f->l2t == NULL) 7464 return (EAGAIN); 7465 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 7466 f->fs.dmac)) { 7467 t4_l2t_release(f->l2t); 7468 f->l2t = NULL; 7469 return (ENOMEM); 7470 } 7471 } 7472 7473 ftid = sc->tids.ftid_base + fidx; 7474 7475 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 7476 if (wr == NULL) 7477 return (ENOMEM); 7478 7479 fwr = wrtod(wr); 7480 bzero(fwr, sizeof (*fwr)); 7481 7482 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 7483 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 7484 fwr->tid_to_iq = 7485 htobe32(V_FW_FILTER_WR_TID(ftid) | 7486 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 7487 V_FW_FILTER_WR_NOREPLY(0) | 7488 V_FW_FILTER_WR_IQ(f->fs.iq)); 7489 fwr->del_filter_to_l2tix = 7490 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 7491 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 7492 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 7493 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 7494 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 7495 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 7496 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 7497 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 7498 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 7499 f->fs.newvlan == VLAN_REWRITE) | 7500 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 7501 f->fs.newvlan == VLAN_REWRITE) | 7502 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 7503 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 7504 V_FW_FILTER_WR_PRIO(f->fs.prio) | 7505 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 7506 fwr->ethtype = htobe16(f->fs.val.ethtype); 7507 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 7508 fwr->frag_to_ovlan_vldm = 7509 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 7510 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 7511 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 7512 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 7513 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 7514 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 7515 fwr->smac_sel = 0; 7516 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 7517 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 7518 fwr->maci_to_matchtypem = 7519 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 7520 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 7521 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 7522 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 7523 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 7524 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 7525 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 7526 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 7527 fwr->ptcl = f->fs.val.proto; 7528 fwr->ptclm = f->fs.mask.proto; 7529 fwr->ttyp = f->fs.val.tos; 7530 fwr->ttypm = f->fs.mask.tos; 7531 fwr->ivlan = htobe16(f->fs.val.vlan); 7532 fwr->ivlanm = htobe16(f->fs.mask.vlan); 7533 fwr->ovlan = htobe16(f->fs.val.vnic); 7534 fwr->ovlanm = htobe16(f->fs.mask.vnic); 7535 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 7536 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 7537 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 7538 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 7539 fwr->lp = htobe16(f->fs.val.dport); 7540 fwr->lpm = htobe16(f->fs.mask.dport); 7541 fwr->fp = htobe16(f->fs.val.sport); 7542 fwr->fpm = htobe16(f->fs.mask.sport); 7543 if (f->fs.newsmac) 7544 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 7545 7546 f->pending = 1; 7547 sc->tids.ftids_in_use++; 7548 7549 t4_wrq_tx(sc, wr); 7550 return (0); 7551 } 7552 7553 static int 7554 del_filter_wr(struct adapter *sc, int fidx) 7555 { 7556 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7557 struct wrqe *wr; 7558 struct fw_filter_wr *fwr; 7559 unsigned int ftid; 7560 7561 ftid = sc->tids.ftid_base + fidx; 7562 7563 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 7564 if (wr == NULL) 7565 return (ENOMEM); 7566 fwr = wrtod(wr); 7567 bzero(fwr, sizeof (*fwr)); 7568 7569 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 7570 7571 f->pending = 1; 7572 t4_wrq_tx(sc, wr); 7573 return (0); 7574 } 7575 7576 int 7577 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 7578 { 7579 struct adapter *sc = iq->adapter; 7580 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 7581 unsigned int idx = GET_TID(rpl); 7582 unsigned int rc; 7583 struct filter_entry *f; 7584 7585 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 7586 rss->opcode)); 7587 7588 if (is_ftid(sc, idx)) { 7589 7590 idx -= sc->tids.ftid_base; 7591 f = &sc->tids.ftid_tab[idx]; 7592 rc = G_COOKIE(rpl->cookie); 7593 7594 mtx_lock(&sc->tids.ftid_lock); 7595 if (rc == FW_FILTER_WR_FLT_ADDED) { 7596 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 7597 __func__, idx)); 7598 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 7599 f->pending = 0; /* asynchronous setup completed */ 7600 f->valid = 1; 7601 } else { 7602 if (rc != FW_FILTER_WR_FLT_DELETED) { 7603 /* Add or delete failed, display an error */ 7604 log(LOG_ERR, 7605 "filter %u setup failed with error %u\n", 7606 idx, rc); 7607 } 7608 7609 clear_filter(f); 7610 sc->tids.ftids_in_use--; 7611 } 7612 wakeup(&sc->tids.ftid_tab); 7613 mtx_unlock(&sc->tids.ftid_lock); 7614 } 7615 7616 return (0); 7617 } 7618 7619 static int 7620 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 7621 { 7622 int rc; 7623 7624 if (cntxt->cid > M_CTXTQID) 7625 return (EINVAL); 7626 7627 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 7628 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 7629 return (EINVAL); 7630 7631 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 7632 if (rc) 7633 return (rc); 7634 7635 if (sc->flags & FW_OK) { 7636 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 7637 &cntxt->data[0]); 7638 if (rc == 0) 7639 goto done; 7640 } 7641 7642 /* 7643 * Read via firmware failed or wasn't even attempted. Read directly via 7644 * the backdoor. 7645 */ 7646 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 7647 done: 7648 end_synchronized_op(sc, 0); 7649 return (rc); 7650 } 7651 7652 static int 7653 load_fw(struct adapter *sc, struct t4_data *fw) 7654 { 7655 int rc; 7656 uint8_t *fw_data; 7657 7658 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 7659 if (rc) 7660 return (rc); 7661 7662 if (sc->flags & FULL_INIT_DONE) { 7663 rc = EBUSY; 7664 goto done; 7665 } 7666 7667 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 7668 if (fw_data == NULL) { 7669 rc = ENOMEM; 7670 goto done; 7671 } 7672 7673 rc = copyin(fw->data, fw_data, fw->len); 7674 if (rc == 0) 7675 rc = -t4_load_fw(sc, fw_data, fw->len); 7676 7677 free(fw_data, M_CXGBE); 7678 done: 7679 end_synchronized_op(sc, 0); 7680 return (rc); 7681 } 7682 7683 static int 7684 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 7685 { 7686 uint32_t addr, off, remaining, i, n; 7687 uint32_t *buf, *b; 7688 uint32_t mw_base, mw_aperture; 7689 int rc; 7690 uint8_t *dst; 7691 7692 rc = validate_mem_range(sc, mr->addr, mr->len); 7693 if (rc != 0) 7694 return (rc); 7695 7696 memwin_info(sc, win, &mw_base, &mw_aperture); 7697 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 7698 addr = mr->addr; 7699 remaining = mr->len; 7700 dst = (void *)mr->data; 7701 7702 while (remaining) { 7703 off = position_memwin(sc, win, addr); 7704 7705 /* number of bytes that we'll copy in the inner loop */ 7706 n = min(remaining, mw_aperture - off); 7707 for (i = 0; i < n; i += 4) 7708 *b++ = t4_read_reg(sc, mw_base + off + i); 7709 7710 rc = copyout(buf, dst, n); 7711 if (rc != 0) 7712 break; 7713 7714 b = buf; 7715 dst += n; 7716 remaining -= n; 7717 addr += n; 7718 } 7719 7720 free(buf, M_CXGBE); 7721 return (rc); 7722 } 7723 7724 static int 7725 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 7726 { 7727 int rc; 7728 7729 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 7730 return (EINVAL); 7731 7732 if (i2cd->len > sizeof(i2cd->data)) 7733 return (EFBIG); 7734 7735 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 7736 if (rc) 7737 return (rc); 7738 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 7739 i2cd->offset, i2cd->len, &i2cd->data[0]); 7740 end_synchronized_op(sc, 0); 7741 7742 return (rc); 7743 } 7744 7745 static int 7746 in_range(int val, int lo, int hi) 7747 { 7748 7749 return (val < 0 || (val <= hi && val >= lo)); 7750 } 7751 7752 static int 7753 set_sched_class(struct adapter *sc, struct t4_sched_params *p) 7754 { 7755 int fw_subcmd, fw_type, rc; 7756 7757 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc"); 7758 if (rc) 7759 return (rc); 7760 7761 if (!(sc->flags & FULL_INIT_DONE)) { 7762 rc = EAGAIN; 7763 goto done; 7764 } 7765 7766 /* 7767 * Translate the cxgbetool parameters into T4 firmware parameters. (The 7768 * sub-command and type are in common locations.) 7769 */ 7770 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG) 7771 fw_subcmd = FW_SCHED_SC_CONFIG; 7772 else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS) 7773 fw_subcmd = FW_SCHED_SC_PARAMS; 7774 else { 7775 rc = EINVAL; 7776 goto done; 7777 } 7778 if (p->type == SCHED_CLASS_TYPE_PACKET) 7779 fw_type = FW_SCHED_TYPE_PKTSCHED; 7780 else { 7781 rc = EINVAL; 7782 goto done; 7783 } 7784 7785 if (fw_subcmd == FW_SCHED_SC_CONFIG) { 7786 /* Vet our parameters ..*/ 7787 if (p->u.config.minmax < 0) { 7788 rc = EINVAL; 7789 goto done; 7790 } 7791 7792 /* And pass the request to the firmware ...*/ 7793 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1); 7794 goto done; 7795 } 7796 7797 if (fw_subcmd == FW_SCHED_SC_PARAMS) { 7798 int fw_level; 7799 int fw_mode; 7800 int fw_rateunit; 7801 int fw_ratemode; 7802 7803 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL) 7804 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL; 7805 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) 7806 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR; 7807 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) 7808 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL; 7809 else { 7810 rc = EINVAL; 7811 goto done; 7812 } 7813 7814 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS) 7815 fw_mode = FW_SCHED_PARAMS_MODE_CLASS; 7816 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW) 7817 fw_mode = FW_SCHED_PARAMS_MODE_FLOW; 7818 else { 7819 rc = EINVAL; 7820 goto done; 7821 } 7822 7823 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS) 7824 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 7825 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS) 7826 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE; 7827 else { 7828 rc = EINVAL; 7829 goto done; 7830 } 7831 7832 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL) 7833 fw_ratemode = FW_SCHED_PARAMS_RATE_REL; 7834 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS) 7835 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS; 7836 else { 7837 rc = EINVAL; 7838 goto done; 7839 } 7840 7841 /* Vet our parameters ... */ 7842 if (!in_range(p->u.params.channel, 0, 3) || 7843 !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) || 7844 !in_range(p->u.params.minrate, 0, 10000000) || 7845 !in_range(p->u.params.maxrate, 0, 10000000) || 7846 !in_range(p->u.params.weight, 0, 100)) { 7847 rc = ERANGE; 7848 goto done; 7849 } 7850 7851 /* 7852 * Translate any unset parameters into the firmware's 7853 * nomenclature and/or fail the call if the parameters 7854 * are required ... 7855 */ 7856 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 || 7857 p->u.params.channel < 0 || p->u.params.cl < 0) { 7858 rc = EINVAL; 7859 goto done; 7860 } 7861 if (p->u.params.minrate < 0) 7862 p->u.params.minrate = 0; 7863 if (p->u.params.maxrate < 0) { 7864 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7865 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7866 rc = EINVAL; 7867 goto done; 7868 } else 7869 p->u.params.maxrate = 0; 7870 } 7871 if (p->u.params.weight < 0) { 7872 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) { 7873 rc = EINVAL; 7874 goto done; 7875 } else 7876 p->u.params.weight = 0; 7877 } 7878 if (p->u.params.pktsize < 0) { 7879 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7880 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7881 rc = EINVAL; 7882 goto done; 7883 } else 7884 p->u.params.pktsize = 0; 7885 } 7886 7887 /* See what the firmware thinks of the request ... */ 7888 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode, 7889 fw_rateunit, fw_ratemode, p->u.params.channel, 7890 p->u.params.cl, p->u.params.minrate, p->u.params.maxrate, 7891 p->u.params.weight, p->u.params.pktsize, 1); 7892 goto done; 7893 } 7894 7895 rc = EINVAL; 7896 done: 7897 end_synchronized_op(sc, 0); 7898 return (rc); 7899 } 7900 7901 static int 7902 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p) 7903 { 7904 struct port_info *pi = NULL; 7905 struct sge_txq *txq; 7906 uint32_t fw_mnem, fw_queue, fw_class; 7907 int i, rc; 7908 7909 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq"); 7910 if (rc) 7911 return (rc); 7912 7913 if (!(sc->flags & FULL_INIT_DONE)) { 7914 rc = EAGAIN; 7915 goto done; 7916 } 7917 7918 if (p->port >= sc->params.nports) { 7919 rc = EINVAL; 7920 goto done; 7921 } 7922 7923 pi = sc->port[p->port]; 7924 if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) { 7925 rc = EINVAL; 7926 goto done; 7927 } 7928 7929 /* 7930 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX 7931 * Scheduling Class in this case). 7932 */ 7933 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 7934 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH)); 7935 fw_class = p->cl < 0 ? 0xffffffff : p->cl; 7936 7937 /* 7938 * If op.queue is non-negative, then we're only changing the scheduling 7939 * on a single specified TX queue. 7940 */ 7941 if (p->queue >= 0) { 7942 txq = &sc->sge.txq[pi->first_txq + p->queue]; 7943 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7944 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7945 &fw_class); 7946 goto done; 7947 } 7948 7949 /* 7950 * Change the scheduling on all the TX queues for the 7951 * interface. 7952 */ 7953 for_each_txq(pi, i, txq) { 7954 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7955 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7956 &fw_class); 7957 if (rc) 7958 goto done; 7959 } 7960 7961 rc = 0; 7962 done: 7963 end_synchronized_op(sc, 0); 7964 return (rc); 7965 } 7966 7967 int 7968 t4_os_find_pci_capability(struct adapter *sc, int cap) 7969 { 7970 int i; 7971 7972 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 7973 } 7974 7975 int 7976 t4_os_pci_save_state(struct adapter *sc) 7977 { 7978 device_t dev; 7979 struct pci_devinfo *dinfo; 7980 7981 dev = sc->dev; 7982 dinfo = device_get_ivars(dev); 7983 7984 pci_cfg_save(dev, dinfo, 0); 7985 return (0); 7986 } 7987 7988 int 7989 t4_os_pci_restore_state(struct adapter *sc) 7990 { 7991 device_t dev; 7992 struct pci_devinfo *dinfo; 7993 7994 dev = sc->dev; 7995 dinfo = device_get_ivars(dev); 7996 7997 pci_cfg_restore(dev, dinfo); 7998 return (0); 7999 } 8000 8001 void 8002 t4_os_portmod_changed(const struct adapter *sc, int idx) 8003 { 8004 struct port_info *pi = sc->port[idx]; 8005 static const char *mod_str[] = { 8006 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 8007 }; 8008 8009 build_medialist(pi, &pi->media); 8010 #ifdef DEV_NETMAP 8011 build_medialist(pi, &pi->nm_media); 8012 #endif 8013 8014 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 8015 if_printf(pi->ifp, "transceiver unplugged.\n"); 8016 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 8017 if_printf(pi->ifp, "unknown transceiver inserted.\n"); 8018 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 8019 if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 8020 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 8021 if_printf(pi->ifp, "%s transceiver inserted.\n", 8022 mod_str[pi->mod_type]); 8023 } else { 8024 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 8025 pi->mod_type); 8026 } 8027 } 8028 8029 void 8030 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason) 8031 { 8032 struct port_info *pi = sc->port[idx]; 8033 struct ifnet *ifp = pi->ifp; 8034 8035 if (link_stat) { 8036 pi->linkdnrc = -1; 8037 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 8038 if_link_state_change(ifp, LINK_STATE_UP); 8039 } else { 8040 if (reason >= 0) 8041 pi->linkdnrc = reason; 8042 if_link_state_change(ifp, LINK_STATE_DOWN); 8043 } 8044 } 8045 8046 void 8047 t4_iterate(void (*func)(struct adapter *, void *), void *arg) 8048 { 8049 struct adapter *sc; 8050 8051 sx_slock(&t4_list_lock); 8052 SLIST_FOREACH(sc, &t4_list, link) { 8053 /* 8054 * func should not make any assumptions about what state sc is 8055 * in - the only guarantee is that sc->sc_lock is a valid lock. 8056 */ 8057 func(sc, arg); 8058 } 8059 sx_sunlock(&t4_list_lock); 8060 } 8061 8062 static int 8063 t4_open(struct cdev *dev, int flags, int type, struct thread *td) 8064 { 8065 return (0); 8066 } 8067 8068 static int 8069 t4_close(struct cdev *dev, int flags, int type, struct thread *td) 8070 { 8071 return (0); 8072 } 8073 8074 static int 8075 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 8076 struct thread *td) 8077 { 8078 int rc; 8079 struct adapter *sc = dev->si_drv1; 8080 8081 rc = priv_check(td, PRIV_DRIVER); 8082 if (rc != 0) 8083 return (rc); 8084 8085 switch (cmd) { 8086 case CHELSIO_T4_GETREG: { 8087 struct t4_reg *edata = (struct t4_reg *)data; 8088 8089 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8090 return (EFAULT); 8091 8092 if (edata->size == 4) 8093 edata->val = t4_read_reg(sc, edata->addr); 8094 else if (edata->size == 8) 8095 edata->val = t4_read_reg64(sc, edata->addr); 8096 else 8097 return (EINVAL); 8098 8099 break; 8100 } 8101 case CHELSIO_T4_SETREG: { 8102 struct t4_reg *edata = (struct t4_reg *)data; 8103 8104 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8105 return (EFAULT); 8106 8107 if (edata->size == 4) { 8108 if (edata->val & 0xffffffff00000000) 8109 return (EINVAL); 8110 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 8111 } else if (edata->size == 8) 8112 t4_write_reg64(sc, edata->addr, edata->val); 8113 else 8114 return (EINVAL); 8115 break; 8116 } 8117 case CHELSIO_T4_REGDUMP: { 8118 struct t4_regdump *regs = (struct t4_regdump *)data; 8119 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 8120 uint8_t *buf; 8121 8122 if (regs->len < reglen) { 8123 regs->len = reglen; /* hint to the caller */ 8124 return (ENOBUFS); 8125 } 8126 8127 regs->len = reglen; 8128 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 8129 t4_get_regs(sc, regs, buf); 8130 rc = copyout(buf, regs->data, reglen); 8131 free(buf, M_CXGBE); 8132 break; 8133 } 8134 case CHELSIO_T4_GET_FILTER_MODE: 8135 rc = get_filter_mode(sc, (uint32_t *)data); 8136 break; 8137 case CHELSIO_T4_SET_FILTER_MODE: 8138 rc = set_filter_mode(sc, *(uint32_t *)data); 8139 break; 8140 case CHELSIO_T4_GET_FILTER: 8141 rc = get_filter(sc, (struct t4_filter *)data); 8142 break; 8143 case CHELSIO_T4_SET_FILTER: 8144 rc = set_filter(sc, (struct t4_filter *)data); 8145 break; 8146 case CHELSIO_T4_DEL_FILTER: 8147 rc = del_filter(sc, (struct t4_filter *)data); 8148 break; 8149 case CHELSIO_T4_GET_SGE_CONTEXT: 8150 rc = get_sge_context(sc, (struct t4_sge_context *)data); 8151 break; 8152 case CHELSIO_T4_LOAD_FW: 8153 rc = load_fw(sc, (struct t4_data *)data); 8154 break; 8155 case CHELSIO_T4_GET_MEM: 8156 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 8157 break; 8158 case CHELSIO_T4_GET_I2C: 8159 rc = read_i2c(sc, (struct t4_i2c_data *)data); 8160 break; 8161 case CHELSIO_T4_CLEAR_STATS: { 8162 int i; 8163 u_int port_id = *(uint32_t *)data; 8164 struct port_info *pi; 8165 8166 if (port_id >= sc->params.nports) 8167 return (EINVAL); 8168 pi = sc->port[port_id]; 8169 8170 /* MAC stats */ 8171 t4_clr_port_stats(sc, pi->tx_chan); 8172 8173 if (pi->flags & PORT_INIT_DONE) { 8174 struct sge_rxq *rxq; 8175 struct sge_txq *txq; 8176 struct sge_wrq *wrq; 8177 8178 for_each_rxq(pi, i, rxq) { 8179 #if defined(INET) || defined(INET6) 8180 rxq->lro.lro_queued = 0; 8181 rxq->lro.lro_flushed = 0; 8182 #endif 8183 rxq->rxcsum = 0; 8184 rxq->vlan_extraction = 0; 8185 } 8186 8187 for_each_txq(pi, i, txq) { 8188 txq->txcsum = 0; 8189 txq->tso_wrs = 0; 8190 txq->vlan_insertion = 0; 8191 txq->imm_wrs = 0; 8192 txq->sgl_wrs = 0; 8193 txq->txpkt_wrs = 0; 8194 txq->txpkts_wrs = 0; 8195 txq->txpkts_pkts = 0; 8196 txq->br->br_drops = 0; 8197 txq->no_dmamap = 0; 8198 txq->no_desc = 0; 8199 } 8200 8201 #ifdef TCP_OFFLOAD 8202 /* nothing to clear for each ofld_rxq */ 8203 8204 for_each_ofld_txq(pi, i, wrq) { 8205 wrq->tx_wrs = 0; 8206 wrq->no_desc = 0; 8207 } 8208 #endif 8209 wrq = &sc->sge.ctrlq[pi->port_id]; 8210 wrq->tx_wrs = 0; 8211 wrq->no_desc = 0; 8212 } 8213 break; 8214 } 8215 case CHELSIO_T4_SCHED_CLASS: 8216 rc = set_sched_class(sc, (struct t4_sched_params *)data); 8217 break; 8218 case CHELSIO_T4_SCHED_QUEUE: 8219 rc = set_sched_queue(sc, (struct t4_sched_queue *)data); 8220 break; 8221 case CHELSIO_T4_GET_TRACER: 8222 rc = t4_get_tracer(sc, (struct t4_tracer *)data); 8223 break; 8224 case CHELSIO_T4_SET_TRACER: 8225 rc = t4_set_tracer(sc, (struct t4_tracer *)data); 8226 break; 8227 default: 8228 rc = EINVAL; 8229 } 8230 8231 return (rc); 8232 } 8233 8234 #ifdef TCP_OFFLOAD 8235 void 8236 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask, 8237 const unsigned int *pgsz_order) 8238 { 8239 struct port_info *pi = ifp->if_softc; 8240 struct adapter *sc = pi->adapter; 8241 8242 t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask); 8243 t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) | 8244 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) | 8245 V_HPZ3(pgsz_order[3])); 8246 } 8247 8248 static int 8249 toe_capability(struct port_info *pi, int enable) 8250 { 8251 int rc; 8252 struct adapter *sc = pi->adapter; 8253 8254 ASSERT_SYNCHRONIZED_OP(sc); 8255 8256 if (!is_offload(sc)) 8257 return (ENODEV); 8258 8259 if (enable) { 8260 if (!(sc->flags & FULL_INIT_DONE)) { 8261 rc = cxgbe_init_synchronized(pi); 8262 if (rc) 8263 return (rc); 8264 } 8265 8266 if (isset(&sc->offload_map, pi->port_id)) 8267 return (0); 8268 8269 if (!(sc->flags & TOM_INIT_DONE)) { 8270 rc = t4_activate_uld(sc, ULD_TOM); 8271 if (rc == EAGAIN) { 8272 log(LOG_WARNING, 8273 "You must kldload t4_tom.ko before trying " 8274 "to enable TOE on a cxgbe interface.\n"); 8275 } 8276 if (rc != 0) 8277 return (rc); 8278 KASSERT(sc->tom_softc != NULL, 8279 ("%s: TOM activated but softc NULL", __func__)); 8280 KASSERT(sc->flags & TOM_INIT_DONE, 8281 ("%s: TOM activated but flag not set", __func__)); 8282 } 8283 8284 setbit(&sc->offload_map, pi->port_id); 8285 } else { 8286 if (!isset(&sc->offload_map, pi->port_id)) 8287 return (0); 8288 8289 KASSERT(sc->flags & TOM_INIT_DONE, 8290 ("%s: TOM never initialized?", __func__)); 8291 clrbit(&sc->offload_map, pi->port_id); 8292 } 8293 8294 return (0); 8295 } 8296 8297 /* 8298 * Add an upper layer driver to the global list. 8299 */ 8300 int 8301 t4_register_uld(struct uld_info *ui) 8302 { 8303 int rc = 0; 8304 struct uld_info *u; 8305 8306 sx_xlock(&t4_uld_list_lock); 8307 SLIST_FOREACH(u, &t4_uld_list, link) { 8308 if (u->uld_id == ui->uld_id) { 8309 rc = EEXIST; 8310 goto done; 8311 } 8312 } 8313 8314 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 8315 ui->refcount = 0; 8316 done: 8317 sx_xunlock(&t4_uld_list_lock); 8318 return (rc); 8319 } 8320 8321 int 8322 t4_unregister_uld(struct uld_info *ui) 8323 { 8324 int rc = EINVAL; 8325 struct uld_info *u; 8326 8327 sx_xlock(&t4_uld_list_lock); 8328 8329 SLIST_FOREACH(u, &t4_uld_list, link) { 8330 if (u == ui) { 8331 if (ui->refcount > 0) { 8332 rc = EBUSY; 8333 goto done; 8334 } 8335 8336 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 8337 rc = 0; 8338 goto done; 8339 } 8340 } 8341 done: 8342 sx_xunlock(&t4_uld_list_lock); 8343 return (rc); 8344 } 8345 8346 int 8347 t4_activate_uld(struct adapter *sc, int id) 8348 { 8349 int rc = EAGAIN; 8350 struct uld_info *ui; 8351 8352 ASSERT_SYNCHRONIZED_OP(sc); 8353 8354 sx_slock(&t4_uld_list_lock); 8355 8356 SLIST_FOREACH(ui, &t4_uld_list, link) { 8357 if (ui->uld_id == id) { 8358 if (!(sc->flags & FULL_INIT_DONE)) { 8359 rc = adapter_full_init(sc); 8360 if (rc != 0) 8361 goto done; 8362 } 8363 8364 rc = ui->activate(sc); 8365 if (rc == 0) 8366 ui->refcount++; 8367 goto done; 8368 } 8369 } 8370 done: 8371 sx_sunlock(&t4_uld_list_lock); 8372 8373 return (rc); 8374 } 8375 8376 int 8377 t4_deactivate_uld(struct adapter *sc, int id) 8378 { 8379 int rc = EINVAL; 8380 struct uld_info *ui; 8381 8382 ASSERT_SYNCHRONIZED_OP(sc); 8383 8384 sx_slock(&t4_uld_list_lock); 8385 8386 SLIST_FOREACH(ui, &t4_uld_list, link) { 8387 if (ui->uld_id == id) { 8388 rc = ui->deactivate(sc); 8389 if (rc == 0) 8390 ui->refcount--; 8391 goto done; 8392 } 8393 } 8394 done: 8395 sx_sunlock(&t4_uld_list_lock); 8396 8397 return (rc); 8398 } 8399 #endif 8400 8401 /* 8402 * Come up with reasonable defaults for some of the tunables, provided they're 8403 * not set by the user (in which case we'll use the values as is). 8404 */ 8405 static void 8406 tweak_tunables(void) 8407 { 8408 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 8409 8410 if (t4_ntxq10g < 1) 8411 t4_ntxq10g = min(nc, NTXQ_10G); 8412 8413 if (t4_ntxq1g < 1) 8414 t4_ntxq1g = min(nc, NTXQ_1G); 8415 8416 if (t4_nrxq10g < 1) 8417 t4_nrxq10g = min(nc, NRXQ_10G); 8418 8419 if (t4_nrxq1g < 1) 8420 t4_nrxq1g = min(nc, NRXQ_1G); 8421 8422 #ifdef TCP_OFFLOAD 8423 if (t4_nofldtxq10g < 1) 8424 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 8425 8426 if (t4_nofldtxq1g < 1) 8427 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 8428 8429 if (t4_nofldrxq10g < 1) 8430 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 8431 8432 if (t4_nofldrxq1g < 1) 8433 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 8434 8435 if (t4_toecaps_allowed == -1) 8436 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 8437 #else 8438 if (t4_toecaps_allowed == -1) 8439 t4_toecaps_allowed = 0; 8440 #endif 8441 8442 #ifdef DEV_NETMAP 8443 if (t4_nnmtxq10g < 1) 8444 t4_nnmtxq10g = min(nc, NNMTXQ_10G); 8445 8446 if (t4_nnmtxq1g < 1) 8447 t4_nnmtxq1g = min(nc, NNMTXQ_1G); 8448 8449 if (t4_nnmrxq10g < 1) 8450 t4_nnmrxq10g = min(nc, NNMRXQ_10G); 8451 8452 if (t4_nnmrxq1g < 1) 8453 t4_nnmrxq1g = min(nc, NNMRXQ_1G); 8454 #endif 8455 8456 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 8457 t4_tmr_idx_10g = TMR_IDX_10G; 8458 8459 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 8460 t4_pktc_idx_10g = PKTC_IDX_10G; 8461 8462 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 8463 t4_tmr_idx_1g = TMR_IDX_1G; 8464 8465 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 8466 t4_pktc_idx_1g = PKTC_IDX_1G; 8467 8468 if (t4_qsize_txq < 128) 8469 t4_qsize_txq = 128; 8470 8471 if (t4_qsize_rxq < 128) 8472 t4_qsize_rxq = 128; 8473 while (t4_qsize_rxq & 7) 8474 t4_qsize_rxq++; 8475 8476 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 8477 } 8478 8479 static struct sx mlu; /* mod load unload */ 8480 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload"); 8481 8482 static int 8483 mod_event(module_t mod, int cmd, void *arg) 8484 { 8485 int rc = 0; 8486 static int loaded = 0; 8487 8488 switch (cmd) { 8489 case MOD_LOAD: 8490 sx_xlock(&mlu); 8491 if (loaded++ == 0) { 8492 t4_sge_modload(); 8493 sx_init(&t4_list_lock, "T4/T5 adapters"); 8494 SLIST_INIT(&t4_list); 8495 #ifdef TCP_OFFLOAD 8496 sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); 8497 SLIST_INIT(&t4_uld_list); 8498 #endif 8499 t4_tracer_modload(); 8500 tweak_tunables(); 8501 } 8502 sx_xunlock(&mlu); 8503 break; 8504 8505 case MOD_UNLOAD: 8506 sx_xlock(&mlu); 8507 if (--loaded == 0) { 8508 int tries; 8509 8510 sx_slock(&t4_list_lock); 8511 if (!SLIST_EMPTY(&t4_list)) { 8512 rc = EBUSY; 8513 sx_sunlock(&t4_list_lock); 8514 goto done_unload; 8515 } 8516 #ifdef TCP_OFFLOAD 8517 sx_slock(&t4_uld_list_lock); 8518 if (!SLIST_EMPTY(&t4_uld_list)) { 8519 rc = EBUSY; 8520 sx_sunlock(&t4_uld_list_lock); 8521 sx_sunlock(&t4_list_lock); 8522 goto done_unload; 8523 } 8524 #endif 8525 tries = 0; 8526 while (tries++ < 5 && t4_sge_extfree_refs() != 0) { 8527 uprintf("%ju clusters with custom free routine " 8528 "still is use.\n", t4_sge_extfree_refs()); 8529 pause("t4unload", 2 * hz); 8530 } 8531 #ifdef TCP_OFFLOAD 8532 sx_sunlock(&t4_uld_list_lock); 8533 #endif 8534 sx_sunlock(&t4_list_lock); 8535 8536 if (t4_sge_extfree_refs() == 0) { 8537 t4_tracer_modunload(); 8538 #ifdef TCP_OFFLOAD 8539 sx_destroy(&t4_uld_list_lock); 8540 #endif 8541 sx_destroy(&t4_list_lock); 8542 t4_sge_modunload(); 8543 loaded = 0; 8544 } else { 8545 rc = EBUSY; 8546 loaded++; /* undo earlier decrement */ 8547 } 8548 } 8549 done_unload: 8550 sx_xunlock(&mlu); 8551 break; 8552 } 8553 8554 return (rc); 8555 } 8556 8557 static devclass_t t4_devclass, t5_devclass; 8558 static devclass_t cxgbe_devclass, cxl_devclass; 8559 8560 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 8561 MODULE_VERSION(t4nex, 1); 8562 MODULE_DEPEND(t4nex, firmware, 1, 1, 1); 8563 8564 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 8565 MODULE_VERSION(t5nex, 1); 8566 MODULE_DEPEND(t5nex, firmware, 1, 1, 1); 8567 8568 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 8569 MODULE_VERSION(cxgbe, 1); 8570 8571 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 8572 MODULE_VERSION(cxl, 1); 8573