1 /*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_inet.h" 32 #include "opt_inet6.h" 33 34 #include <sys/param.h> 35 #include <sys/conf.h> 36 #include <sys/priv.h> 37 #include <sys/kernel.h> 38 #include <sys/bus.h> 39 #include <sys/module.h> 40 #include <sys/malloc.h> 41 #include <sys/queue.h> 42 #include <sys/taskqueue.h> 43 #include <sys/pciio.h> 44 #include <dev/pci/pcireg.h> 45 #include <dev/pci/pcivar.h> 46 #include <dev/pci/pci_private.h> 47 #include <sys/firmware.h> 48 #include <sys/sbuf.h> 49 #include <sys/smp.h> 50 #include <sys/socket.h> 51 #include <sys/sockio.h> 52 #include <sys/sysctl.h> 53 #include <net/ethernet.h> 54 #include <net/if.h> 55 #include <net/if_types.h> 56 #include <net/if_dl.h> 57 #include <net/if_vlan_var.h> 58 #if defined(__i386__) || defined(__amd64__) 59 #include <vm/vm.h> 60 #include <vm/pmap.h> 61 #endif 62 63 #include "common/common.h" 64 #include "common/t4_msg.h" 65 #include "common/t4_regs.h" 66 #include "common/t4_regs_values.h" 67 #include "t4_ioctl.h" 68 #include "t4_l2t.h" 69 70 /* T4 bus driver interface */ 71 static int t4_probe(device_t); 72 static int t4_attach(device_t); 73 static int t4_detach(device_t); 74 static device_method_t t4_methods[] = { 75 DEVMETHOD(device_probe, t4_probe), 76 DEVMETHOD(device_attach, t4_attach), 77 DEVMETHOD(device_detach, t4_detach), 78 79 DEVMETHOD_END 80 }; 81 static driver_t t4_driver = { 82 "t4nex", 83 t4_methods, 84 sizeof(struct adapter) 85 }; 86 87 88 /* T4 port (cxgbe) interface */ 89 static int cxgbe_probe(device_t); 90 static int cxgbe_attach(device_t); 91 static int cxgbe_detach(device_t); 92 static device_method_t cxgbe_methods[] = { 93 DEVMETHOD(device_probe, cxgbe_probe), 94 DEVMETHOD(device_attach, cxgbe_attach), 95 DEVMETHOD(device_detach, cxgbe_detach), 96 { 0, 0 } 97 }; 98 static driver_t cxgbe_driver = { 99 "cxgbe", 100 cxgbe_methods, 101 sizeof(struct port_info) 102 }; 103 104 static d_ioctl_t t4_ioctl; 105 static d_open_t t4_open; 106 static d_close_t t4_close; 107 108 static struct cdevsw t4_cdevsw = { 109 .d_version = D_VERSION, 110 .d_flags = 0, 111 .d_open = t4_open, 112 .d_close = t4_close, 113 .d_ioctl = t4_ioctl, 114 .d_name = "t4nex", 115 }; 116 117 /* T5 bus driver interface */ 118 static int t5_probe(device_t); 119 static device_method_t t5_methods[] = { 120 DEVMETHOD(device_probe, t5_probe), 121 DEVMETHOD(device_attach, t4_attach), 122 DEVMETHOD(device_detach, t4_detach), 123 124 DEVMETHOD_END 125 }; 126 static driver_t t5_driver = { 127 "t5nex", 128 t5_methods, 129 sizeof(struct adapter) 130 }; 131 132 133 /* T5 port (cxl) interface */ 134 static driver_t cxl_driver = { 135 "cxl", 136 cxgbe_methods, 137 sizeof(struct port_info) 138 }; 139 140 static struct cdevsw t5_cdevsw = { 141 .d_version = D_VERSION, 142 .d_flags = 0, 143 .d_open = t4_open, 144 .d_close = t4_close, 145 .d_ioctl = t4_ioctl, 146 .d_name = "t5nex", 147 }; 148 149 /* ifnet + media interface */ 150 static void cxgbe_init(void *); 151 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 152 static int cxgbe_transmit(struct ifnet *, struct mbuf *); 153 static void cxgbe_qflush(struct ifnet *); 154 static uint64_t cxgbe_get_counter(struct ifnet *, ift_counter); 155 static int cxgbe_media_change(struct ifnet *); 156 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 157 158 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 159 160 /* 161 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 162 * then ADAPTER_LOCK, then t4_uld_list_lock. 163 */ 164 static struct sx t4_list_lock; 165 SLIST_HEAD(, adapter) t4_list; 166 #ifdef TCP_OFFLOAD 167 static struct sx t4_uld_list_lock; 168 SLIST_HEAD(, uld_info) t4_uld_list; 169 #endif 170 171 /* 172 * Tunables. See tweak_tunables() too. 173 * 174 * Each tunable is set to a default value here if it's known at compile-time. 175 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 176 * provide a reasonable default when the driver is loaded. 177 * 178 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 179 * T5 are under hw.cxl. 180 */ 181 182 /* 183 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 184 */ 185 #define NTXQ_10G 16 186 static int t4_ntxq10g = -1; 187 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 188 189 #define NRXQ_10G 8 190 static int t4_nrxq10g = -1; 191 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 192 193 #define NTXQ_1G 4 194 static int t4_ntxq1g = -1; 195 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 196 197 #define NRXQ_1G 2 198 static int t4_nrxq1g = -1; 199 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 200 201 static int t4_rsrv_noflowq = 0; 202 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq); 203 204 #ifdef TCP_OFFLOAD 205 #define NOFLDTXQ_10G 8 206 static int t4_nofldtxq10g = -1; 207 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 208 209 #define NOFLDRXQ_10G 2 210 static int t4_nofldrxq10g = -1; 211 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 212 213 #define NOFLDTXQ_1G 2 214 static int t4_nofldtxq1g = -1; 215 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 216 217 #define NOFLDRXQ_1G 1 218 static int t4_nofldrxq1g = -1; 219 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 220 #endif 221 222 #ifdef DEV_NETMAP 223 #define NNMTXQ_10G 2 224 static int t4_nnmtxq10g = -1; 225 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g); 226 227 #define NNMRXQ_10G 2 228 static int t4_nnmrxq10g = -1; 229 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g); 230 231 #define NNMTXQ_1G 1 232 static int t4_nnmtxq1g = -1; 233 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g); 234 235 #define NNMRXQ_1G 1 236 static int t4_nnmrxq1g = -1; 237 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g); 238 #endif 239 240 /* 241 * Holdoff parameters for 10G and 1G ports. 242 */ 243 #define TMR_IDX_10G 1 244 static int t4_tmr_idx_10g = TMR_IDX_10G; 245 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 246 247 #define PKTC_IDX_10G (-1) 248 static int t4_pktc_idx_10g = PKTC_IDX_10G; 249 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 250 251 #define TMR_IDX_1G 1 252 static int t4_tmr_idx_1g = TMR_IDX_1G; 253 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 254 255 #define PKTC_IDX_1G (-1) 256 static int t4_pktc_idx_1g = PKTC_IDX_1G; 257 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 258 259 /* 260 * Size (# of entries) of each tx and rx queue. 261 */ 262 static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 263 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 264 265 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 266 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 267 268 /* 269 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 270 */ 271 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 272 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 273 274 /* 275 * Configuration file. 276 */ 277 #define DEFAULT_CF "default" 278 #define FLASH_CF "flash" 279 #define UWIRE_CF "uwire" 280 #define FPGA_CF "fpga" 281 static char t4_cfg_file[32] = DEFAULT_CF; 282 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 283 284 /* 285 * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively). 286 * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them. 287 * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water 288 * mark or when signalled to do so, 0 to never emit PAUSE. 289 */ 290 static int t4_pause_settings = PAUSE_TX | PAUSE_RX; 291 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings); 292 293 /* 294 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 295 * encouraged respectively). 296 */ 297 static unsigned int t4_fw_install = 1; 298 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 299 300 /* 301 * ASIC features that will be used. Disable the ones you don't want so that the 302 * chip resources aren't wasted on features that will not be used. 303 */ 304 static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 305 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 306 307 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 308 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 309 310 static int t4_toecaps_allowed = -1; 311 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 312 313 static int t4_rdmacaps_allowed = 0; 314 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 315 316 static int t4_iscsicaps_allowed = 0; 317 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 318 319 static int t4_fcoecaps_allowed = 0; 320 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 321 322 static int t5_write_combine = 0; 323 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 324 325 struct intrs_and_queues { 326 uint16_t intr_type; /* INTx, MSI, or MSI-X */ 327 uint16_t nirq; /* Total # of vectors */ 328 uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */ 329 uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */ 330 uint16_t ntxq10g; /* # of NIC txq's for each 10G port */ 331 uint16_t nrxq10g; /* # of NIC rxq's for each 10G port */ 332 uint16_t ntxq1g; /* # of NIC txq's for each 1G port */ 333 uint16_t nrxq1g; /* # of NIC rxq's for each 1G port */ 334 uint16_t rsrv_noflowq; /* Flag whether to reserve queue 0 */ 335 #ifdef TCP_OFFLOAD 336 uint16_t nofldtxq10g; /* # of TOE txq's for each 10G port */ 337 uint16_t nofldrxq10g; /* # of TOE rxq's for each 10G port */ 338 uint16_t nofldtxq1g; /* # of TOE txq's for each 1G port */ 339 uint16_t nofldrxq1g; /* # of TOE rxq's for each 1G port */ 340 #endif 341 #ifdef DEV_NETMAP 342 uint16_t nnmtxq10g; /* # of netmap txq's for each 10G port */ 343 uint16_t nnmrxq10g; /* # of netmap rxq's for each 10G port */ 344 uint16_t nnmtxq1g; /* # of netmap txq's for each 1G port */ 345 uint16_t nnmrxq1g; /* # of netmap rxq's for each 1G port */ 346 #endif 347 }; 348 349 struct filter_entry { 350 uint32_t valid:1; /* filter allocated and valid */ 351 uint32_t locked:1; /* filter is administratively locked */ 352 uint32_t pending:1; /* filter action is pending firmware reply */ 353 uint32_t smtidx:8; /* Source MAC Table index for smac */ 354 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 355 356 struct t4_filter_specification fs; 357 }; 358 359 static int map_bars_0_and_4(struct adapter *); 360 static int map_bar_2(struct adapter *); 361 static void setup_memwin(struct adapter *); 362 static int validate_mem_range(struct adapter *, uint32_t, int); 363 static int fwmtype_to_hwmtype(int); 364 static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 365 uint32_t *); 366 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 367 static uint32_t position_memwin(struct adapter *, int, uint32_t); 368 static int cfg_itype_and_nqueues(struct adapter *, int, int, 369 struct intrs_and_queues *); 370 static int prep_firmware(struct adapter *); 371 static int partition_resources(struct adapter *, const struct firmware *, 372 const char *); 373 static int get_params__pre_init(struct adapter *); 374 static int get_params__post_init(struct adapter *); 375 static int set_params__post_init(struct adapter *); 376 static void t4_set_desc(struct adapter *); 377 static void build_medialist(struct port_info *, struct ifmedia *); 378 static int cxgbe_init_synchronized(struct port_info *); 379 static int cxgbe_uninit_synchronized(struct port_info *); 380 static int setup_intr_handlers(struct adapter *); 381 static void quiesce_eq(struct adapter *, struct sge_eq *); 382 static void quiesce_iq(struct adapter *, struct sge_iq *); 383 static void quiesce_fl(struct adapter *, struct sge_fl *); 384 static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 385 driver_intr_t *, void *, char *); 386 static int t4_free_irq(struct adapter *, struct irq *); 387 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 388 unsigned int); 389 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 390 static void cxgbe_refresh_stats(struct adapter *, struct port_info *); 391 static void cxgbe_tick(void *); 392 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 393 static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 394 struct mbuf *); 395 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 396 static int fw_msg_not_handled(struct adapter *, const __be64 *); 397 static int t4_sysctls(struct adapter *); 398 static int cxgbe_sysctls(struct port_info *); 399 static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 400 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 401 static int sysctl_btphy(SYSCTL_HANDLER_ARGS); 402 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); 403 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 404 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 405 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 406 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 407 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS); 408 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 409 static int sysctl_temperature(SYSCTL_HANDLER_ARGS); 410 #ifdef SBUF_DRAIN 411 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 412 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 413 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 414 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); 415 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); 416 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 417 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 418 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 419 static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 420 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 421 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 422 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 423 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); 424 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 425 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); 426 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 427 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 428 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 429 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 430 static int sysctl_tids(SYSCTL_HANDLER_ARGS); 431 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 432 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 433 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 434 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 435 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 436 #endif 437 static inline void txq_start(struct ifnet *, struct sge_txq *); 438 static uint32_t fconf_to_mode(uint32_t); 439 static uint32_t mode_to_fconf(uint32_t); 440 static uint32_t fspec_to_fconf(struct t4_filter_specification *); 441 static int get_filter_mode(struct adapter *, uint32_t *); 442 static int set_filter_mode(struct adapter *, uint32_t); 443 static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 444 static int get_filter(struct adapter *, struct t4_filter *); 445 static int set_filter(struct adapter *, struct t4_filter *); 446 static int del_filter(struct adapter *, struct t4_filter *); 447 static void clear_filter(struct filter_entry *); 448 static int set_filter_wr(struct adapter *, int); 449 static int del_filter_wr(struct adapter *, int); 450 static int get_sge_context(struct adapter *, struct t4_sge_context *); 451 static int load_fw(struct adapter *, struct t4_data *); 452 static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 453 static int read_i2c(struct adapter *, struct t4_i2c_data *); 454 static int set_sched_class(struct adapter *, struct t4_sched_params *); 455 static int set_sched_queue(struct adapter *, struct t4_sched_queue *); 456 #ifdef TCP_OFFLOAD 457 static int toe_capability(struct port_info *, int); 458 #endif 459 static int mod_event(module_t, int, void *); 460 461 struct { 462 uint16_t device; 463 char *desc; 464 } t4_pciids[] = { 465 {0xa000, "Chelsio Terminator 4 FPGA"}, 466 {0x4400, "Chelsio T440-dbg"}, 467 {0x4401, "Chelsio T420-CR"}, 468 {0x4402, "Chelsio T422-CR"}, 469 {0x4403, "Chelsio T440-CR"}, 470 {0x4404, "Chelsio T420-BCH"}, 471 {0x4405, "Chelsio T440-BCH"}, 472 {0x4406, "Chelsio T440-CH"}, 473 {0x4407, "Chelsio T420-SO"}, 474 {0x4408, "Chelsio T420-CX"}, 475 {0x4409, "Chelsio T420-BT"}, 476 {0x440a, "Chelsio T404-BT"}, 477 {0x440e, "Chelsio T440-LP-CR"}, 478 }, t5_pciids[] = { 479 {0xb000, "Chelsio Terminator 5 FPGA"}, 480 {0x5400, "Chelsio T580-dbg"}, 481 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ 482 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ 483 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ 484 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ 485 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ 486 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ 487 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ 488 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ 489 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 490 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ 491 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ 492 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ 493 {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */ 494 #ifdef notyet 495 {0x5404, "Chelsio T520-BCH"}, 496 {0x5405, "Chelsio T540-BCH"}, 497 {0x5406, "Chelsio T540-CH"}, 498 {0x5408, "Chelsio T520-CX"}, 499 {0x540b, "Chelsio B520-SR"}, 500 {0x540c, "Chelsio B504-BT"}, 501 {0x540f, "Chelsio Amsterdam"}, 502 {0x5413, "Chelsio T580-CHR"}, 503 #endif 504 }; 505 506 #ifdef TCP_OFFLOAD 507 /* 508 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 509 * exactly the same for both rxq and ofld_rxq. 510 */ 511 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 512 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 513 #endif 514 515 /* No easy way to include t4_msg.h before adapter.h so we check this way */ 516 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 517 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 518 519 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); 520 521 static int 522 t4_probe(device_t dev) 523 { 524 int i; 525 uint16_t v = pci_get_vendor(dev); 526 uint16_t d = pci_get_device(dev); 527 uint8_t f = pci_get_function(dev); 528 529 if (v != PCI_VENDOR_ID_CHELSIO) 530 return (ENXIO); 531 532 /* Attach only to PF0 of the FPGA */ 533 if (d == 0xa000 && f != 0) 534 return (ENXIO); 535 536 for (i = 0; i < nitems(t4_pciids); i++) { 537 if (d == t4_pciids[i].device) { 538 device_set_desc(dev, t4_pciids[i].desc); 539 return (BUS_PROBE_DEFAULT); 540 } 541 } 542 543 return (ENXIO); 544 } 545 546 static int 547 t5_probe(device_t dev) 548 { 549 int i; 550 uint16_t v = pci_get_vendor(dev); 551 uint16_t d = pci_get_device(dev); 552 uint8_t f = pci_get_function(dev); 553 554 if (v != PCI_VENDOR_ID_CHELSIO) 555 return (ENXIO); 556 557 /* Attach only to PF0 of the FPGA */ 558 if (d == 0xb000 && f != 0) 559 return (ENXIO); 560 561 for (i = 0; i < nitems(t5_pciids); i++) { 562 if (d == t5_pciids[i].device) { 563 device_set_desc(dev, t5_pciids[i].desc); 564 return (BUS_PROBE_DEFAULT); 565 } 566 } 567 568 return (ENXIO); 569 } 570 571 static int 572 t4_attach(device_t dev) 573 { 574 struct adapter *sc; 575 int rc = 0, i, n10g, n1g, rqidx, tqidx; 576 struct intrs_and_queues iaq; 577 struct sge *s; 578 #ifdef TCP_OFFLOAD 579 int ofld_rqidx, ofld_tqidx; 580 #endif 581 #ifdef DEV_NETMAP 582 int nm_rqidx, nm_tqidx; 583 #endif 584 const char *pcie_ts; 585 586 sc = device_get_softc(dev); 587 sc->dev = dev; 588 589 pci_enable_busmaster(dev); 590 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 591 uint32_t v; 592 593 pci_set_max_read_req(dev, 4096); 594 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 595 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 596 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 597 598 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5); 599 } 600 601 sc->traceq = -1; 602 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); 603 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", 604 device_get_nameunit(dev)); 605 606 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 607 device_get_nameunit(dev)); 608 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 609 sx_xlock(&t4_list_lock); 610 SLIST_INSERT_HEAD(&t4_list, sc, link); 611 sx_xunlock(&t4_list_lock); 612 613 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 614 TAILQ_INIT(&sc->sfl); 615 callout_init(&sc->sfl_callout, CALLOUT_MPSAFE); 616 617 mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF); 618 619 rc = map_bars_0_and_4(sc); 620 if (rc != 0) 621 goto done; /* error message displayed already */ 622 623 /* 624 * This is the real PF# to which we're attaching. Works from within PCI 625 * passthrough environments too, where pci_get_function() could return a 626 * different PF# depending on the passthrough configuration. We need to 627 * use the real PF# in all our communication with the firmware. 628 */ 629 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 630 sc->mbox = sc->pf; 631 632 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 633 sc->an_handler = an_not_handled; 634 for (i = 0; i < nitems(sc->cpl_handler); i++) 635 sc->cpl_handler[i] = cpl_not_handled; 636 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 637 sc->fw_msg_handler[i] = fw_msg_not_handled; 638 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 639 t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt); 640 t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt); 641 t4_init_sge_cpl_handlers(sc); 642 643 /* Prepare the adapter for operation */ 644 rc = -t4_prep_adapter(sc); 645 if (rc != 0) { 646 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 647 goto done; 648 } 649 650 /* 651 * Do this really early, with the memory windows set up even before the 652 * character device. The userland tool's register i/o and mem read 653 * will work even in "recovery mode". 654 */ 655 setup_memwin(sc); 656 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 657 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 658 device_get_nameunit(dev)); 659 if (sc->cdev == NULL) 660 device_printf(dev, "failed to create nexus char device.\n"); 661 else 662 sc->cdev->si_drv1 = sc; 663 664 /* Go no further if recovery mode has been requested. */ 665 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 666 device_printf(dev, "recovery mode.\n"); 667 goto done; 668 } 669 670 /* Prepare the firmware for operation */ 671 rc = prep_firmware(sc); 672 if (rc != 0) 673 goto done; /* error message displayed already */ 674 675 rc = get_params__post_init(sc); 676 if (rc != 0) 677 goto done; /* error message displayed already */ 678 679 rc = set_params__post_init(sc); 680 if (rc != 0) 681 goto done; /* error message displayed already */ 682 683 rc = map_bar_2(sc); 684 if (rc != 0) 685 goto done; /* error message displayed already */ 686 687 rc = t4_create_dma_tag(sc); 688 if (rc != 0) 689 goto done; /* error message displayed already */ 690 691 /* 692 * First pass over all the ports - allocate VIs and initialize some 693 * basic parameters like mac address, port type, etc. We also figure 694 * out whether a port is 10G or 1G and use that information when 695 * calculating how many interrupts to attempt to allocate. 696 */ 697 n10g = n1g = 0; 698 for_each_port(sc, i) { 699 struct port_info *pi; 700 701 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 702 sc->port[i] = pi; 703 704 /* These must be set before t4_port_init */ 705 pi->adapter = sc; 706 pi->port_id = i; 707 708 /* Allocate the vi and initialize parameters like mac addr */ 709 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 710 if (rc != 0) { 711 device_printf(dev, "unable to initialize port %d: %d\n", 712 i, rc); 713 free(pi, M_CXGBE); 714 sc->port[i] = NULL; 715 goto done; 716 } 717 718 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX); 719 pi->link_cfg.requested_fc |= t4_pause_settings; 720 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX); 721 pi->link_cfg.fc |= t4_pause_settings; 722 723 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 724 if (rc != 0) { 725 device_printf(dev, "port %d l1cfg failed: %d\n", i, rc); 726 free(pi, M_CXGBE); 727 sc->port[i] = NULL; 728 goto done; 729 } 730 731 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 732 device_get_nameunit(dev), i); 733 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 734 sc->chan_map[pi->tx_chan] = i; 735 736 if (is_10G_port(pi) || is_40G_port(pi)) { 737 n10g++; 738 pi->tmr_idx = t4_tmr_idx_10g; 739 pi->pktc_idx = t4_pktc_idx_10g; 740 } else { 741 n1g++; 742 pi->tmr_idx = t4_tmr_idx_1g; 743 pi->pktc_idx = t4_pktc_idx_1g; 744 } 745 746 pi->xact_addr_filt = -1; 747 pi->linkdnrc = -1; 748 749 pi->qsize_rxq = t4_qsize_rxq; 750 pi->qsize_txq = t4_qsize_txq; 751 752 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 753 if (pi->dev == NULL) { 754 device_printf(dev, 755 "failed to add device for port %d.\n", i); 756 rc = ENXIO; 757 goto done; 758 } 759 device_set_softc(pi->dev, pi); 760 } 761 762 /* 763 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 764 */ 765 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 766 if (rc != 0) 767 goto done; /* error message displayed already */ 768 769 sc->intr_type = iaq.intr_type; 770 sc->intr_count = iaq.nirq; 771 772 s = &sc->sge; 773 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 774 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 775 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 776 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 777 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 778 #ifdef TCP_OFFLOAD 779 if (is_offload(sc)) { 780 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 781 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 782 s->neq += s->nofldtxq + s->nofldrxq; 783 s->niq += s->nofldrxq; 784 785 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 786 M_CXGBE, M_ZERO | M_WAITOK); 787 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 788 M_CXGBE, M_ZERO | M_WAITOK); 789 } 790 #endif 791 #ifdef DEV_NETMAP 792 s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g; 793 s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g; 794 s->neq += s->nnmtxq + s->nnmrxq; 795 s->niq += s->nnmrxq; 796 797 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq), 798 M_CXGBE, M_ZERO | M_WAITOK); 799 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq), 800 M_CXGBE, M_ZERO | M_WAITOK); 801 #endif 802 803 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 804 M_ZERO | M_WAITOK); 805 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 806 M_ZERO | M_WAITOK); 807 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 808 M_ZERO | M_WAITOK); 809 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 810 M_ZERO | M_WAITOK); 811 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 812 M_ZERO | M_WAITOK); 813 814 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 815 M_ZERO | M_WAITOK); 816 817 t4_init_l2t(sc, M_WAITOK); 818 819 /* 820 * Second pass over the ports. This time we know the number of rx and 821 * tx queues that each port should get. 822 */ 823 rqidx = tqidx = 0; 824 #ifdef TCP_OFFLOAD 825 ofld_rqidx = ofld_tqidx = 0; 826 #endif 827 #ifdef DEV_NETMAP 828 nm_rqidx = nm_tqidx = 0; 829 #endif 830 for_each_port(sc, i) { 831 struct port_info *pi = sc->port[i]; 832 833 if (pi == NULL) 834 continue; 835 836 pi->first_rxq = rqidx; 837 pi->first_txq = tqidx; 838 if (is_10G_port(pi) || is_40G_port(pi)) { 839 pi->flags |= iaq.intr_flags_10g; 840 pi->nrxq = iaq.nrxq10g; 841 pi->ntxq = iaq.ntxq10g; 842 } else { 843 pi->flags |= iaq.intr_flags_1g; 844 pi->nrxq = iaq.nrxq1g; 845 pi->ntxq = iaq.ntxq1g; 846 } 847 848 if (pi->ntxq > 1) 849 pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0; 850 else 851 pi->rsrv_noflowq = 0; 852 853 rqidx += pi->nrxq; 854 tqidx += pi->ntxq; 855 #ifdef TCP_OFFLOAD 856 if (is_offload(sc)) { 857 pi->first_ofld_rxq = ofld_rqidx; 858 pi->first_ofld_txq = ofld_tqidx; 859 if (is_10G_port(pi) || is_40G_port(pi)) { 860 pi->nofldrxq = iaq.nofldrxq10g; 861 pi->nofldtxq = iaq.nofldtxq10g; 862 } else { 863 pi->nofldrxq = iaq.nofldrxq1g; 864 pi->nofldtxq = iaq.nofldtxq1g; 865 } 866 ofld_rqidx += pi->nofldrxq; 867 ofld_tqidx += pi->nofldtxq; 868 } 869 #endif 870 #ifdef DEV_NETMAP 871 pi->first_nm_rxq = nm_rqidx; 872 pi->first_nm_txq = nm_tqidx; 873 if (is_10G_port(pi) || is_40G_port(pi)) { 874 pi->nnmrxq = iaq.nnmrxq10g; 875 pi->nnmtxq = iaq.nnmtxq10g; 876 } else { 877 pi->nnmrxq = iaq.nnmrxq1g; 878 pi->nnmtxq = iaq.nnmtxq1g; 879 } 880 nm_rqidx += pi->nnmrxq; 881 nm_tqidx += pi->nnmtxq; 882 #endif 883 } 884 885 rc = setup_intr_handlers(sc); 886 if (rc != 0) { 887 device_printf(dev, 888 "failed to setup interrupt handlers: %d\n", rc); 889 goto done; 890 } 891 892 rc = bus_generic_attach(dev); 893 if (rc != 0) { 894 device_printf(dev, 895 "failed to attach all child ports: %d\n", rc); 896 goto done; 897 } 898 899 switch (sc->params.pci.speed) { 900 case 0x1: 901 pcie_ts = "2.5"; 902 break; 903 case 0x2: 904 pcie_ts = "5.0"; 905 break; 906 case 0x3: 907 pcie_ts = "8.0"; 908 break; 909 default: 910 pcie_ts = "??"; 911 break; 912 } 913 device_printf(dev, 914 "PCIe x%d (%s GTS/s) (%d), %d ports, %d %s interrupt%s, %d eq, %d iq\n", 915 sc->params.pci.width, pcie_ts, sc->params.pci.speed, 916 sc->params.nports, sc->intr_count, 917 sc->intr_type == INTR_MSIX ? "MSI-X" : 918 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 919 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 920 921 t4_set_desc(sc); 922 923 done: 924 if (rc != 0 && sc->cdev) { 925 /* cdev was created and so cxgbetool works; recover that way. */ 926 device_printf(dev, 927 "error during attach, adapter is now in recovery mode.\n"); 928 rc = 0; 929 } 930 931 if (rc != 0) 932 t4_detach(dev); 933 else 934 t4_sysctls(sc); 935 936 return (rc); 937 } 938 939 /* 940 * Idempotent 941 */ 942 static int 943 t4_detach(device_t dev) 944 { 945 struct adapter *sc; 946 struct port_info *pi; 947 int i, rc; 948 949 sc = device_get_softc(dev); 950 951 if (sc->flags & FULL_INIT_DONE) 952 t4_intr_disable(sc); 953 954 if (sc->cdev) { 955 destroy_dev(sc->cdev); 956 sc->cdev = NULL; 957 } 958 959 rc = bus_generic_detach(dev); 960 if (rc) { 961 device_printf(dev, 962 "failed to detach child devices: %d\n", rc); 963 return (rc); 964 } 965 966 for (i = 0; i < sc->intr_count; i++) 967 t4_free_irq(sc, &sc->irq[i]); 968 969 for (i = 0; i < MAX_NPORTS; i++) { 970 pi = sc->port[i]; 971 if (pi) { 972 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid); 973 if (pi->dev) 974 device_delete_child(dev, pi->dev); 975 976 mtx_destroy(&pi->pi_lock); 977 free(pi, M_CXGBE); 978 } 979 } 980 981 if (sc->flags & FULL_INIT_DONE) 982 adapter_full_uninit(sc); 983 984 if (sc->flags & FW_OK) 985 t4_fw_bye(sc, sc->mbox); 986 987 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 988 pci_release_msi(dev); 989 990 if (sc->regs_res) 991 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 992 sc->regs_res); 993 994 if (sc->udbs_res) 995 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 996 sc->udbs_res); 997 998 if (sc->msix_res) 999 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 1000 sc->msix_res); 1001 1002 if (sc->l2t) 1003 t4_free_l2t(sc->l2t); 1004 1005 #ifdef TCP_OFFLOAD 1006 free(sc->sge.ofld_rxq, M_CXGBE); 1007 free(sc->sge.ofld_txq, M_CXGBE); 1008 #endif 1009 #ifdef DEV_NETMAP 1010 free(sc->sge.nm_rxq, M_CXGBE); 1011 free(sc->sge.nm_txq, M_CXGBE); 1012 #endif 1013 free(sc->irq, M_CXGBE); 1014 free(sc->sge.rxq, M_CXGBE); 1015 free(sc->sge.txq, M_CXGBE); 1016 free(sc->sge.ctrlq, M_CXGBE); 1017 free(sc->sge.iqmap, M_CXGBE); 1018 free(sc->sge.eqmap, M_CXGBE); 1019 free(sc->tids.ftid_tab, M_CXGBE); 1020 t4_destroy_dma_tag(sc); 1021 if (mtx_initialized(&sc->sc_lock)) { 1022 sx_xlock(&t4_list_lock); 1023 SLIST_REMOVE(&t4_list, sc, adapter, link); 1024 sx_xunlock(&t4_list_lock); 1025 mtx_destroy(&sc->sc_lock); 1026 } 1027 1028 if (mtx_initialized(&sc->tids.ftid_lock)) 1029 mtx_destroy(&sc->tids.ftid_lock); 1030 if (mtx_initialized(&sc->sfl_lock)) 1031 mtx_destroy(&sc->sfl_lock); 1032 if (mtx_initialized(&sc->ifp_lock)) 1033 mtx_destroy(&sc->ifp_lock); 1034 if (mtx_initialized(&sc->regwin_lock)) 1035 mtx_destroy(&sc->regwin_lock); 1036 1037 bzero(sc, sizeof(*sc)); 1038 1039 return (0); 1040 } 1041 1042 static int 1043 cxgbe_probe(device_t dev) 1044 { 1045 char buf[128]; 1046 struct port_info *pi = device_get_softc(dev); 1047 1048 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 1049 device_set_desc_copy(dev, buf); 1050 1051 return (BUS_PROBE_DEFAULT); 1052 } 1053 1054 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 1055 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 1056 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS) 1057 #define T4_CAP_ENABLE (T4_CAP) 1058 1059 static int 1060 cxgbe_attach(device_t dev) 1061 { 1062 struct port_info *pi = device_get_softc(dev); 1063 struct ifnet *ifp; 1064 char *s; 1065 int n, o; 1066 1067 /* Allocate an ifnet and set it up */ 1068 ifp = if_alloc(IFT_ETHER); 1069 if (ifp == NULL) { 1070 device_printf(dev, "Cannot allocate ifnet\n"); 1071 return (ENOMEM); 1072 } 1073 pi->ifp = ifp; 1074 ifp->if_softc = pi; 1075 1076 callout_init(&pi->tick, CALLOUT_MPSAFE); 1077 1078 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1079 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1080 1081 ifp->if_init = cxgbe_init; 1082 ifp->if_ioctl = cxgbe_ioctl; 1083 ifp->if_transmit = cxgbe_transmit; 1084 ifp->if_qflush = cxgbe_qflush; 1085 ifp->if_get_counter = cxgbe_get_counter; 1086 1087 ifp->if_capabilities = T4_CAP; 1088 #ifdef TCP_OFFLOAD 1089 if (is_offload(pi->adapter)) 1090 ifp->if_capabilities |= IFCAP_TOE; 1091 #endif 1092 ifp->if_capenable = T4_CAP_ENABLE; 1093 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1094 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1095 1096 ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1097 ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS; 1098 ifp->if_hw_tsomaxsegsize = 65536; 1099 1100 /* Initialize ifmedia for this port */ 1101 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 1102 cxgbe_media_status); 1103 build_medialist(pi, &pi->media); 1104 1105 pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 1106 EVENTHANDLER_PRI_ANY); 1107 1108 ether_ifattach(ifp, pi->hw_addr); 1109 1110 n = 128; 1111 s = malloc(n, M_CXGBE, M_WAITOK); 1112 o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq); 1113 MPASS(n > o); 1114 #ifdef TCP_OFFLOAD 1115 if (is_offload(pi->adapter)) { 1116 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)", 1117 pi->nofldtxq, pi->nofldrxq); 1118 MPASS(n > o); 1119 } 1120 #endif 1121 #ifdef DEV_NETMAP 1122 o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq, 1123 pi->nnmrxq); 1124 MPASS(n > o); 1125 #endif 1126 device_printf(dev, "%s\n", s); 1127 free(s, M_CXGBE); 1128 1129 #ifdef DEV_NETMAP 1130 /* nm_media handled here to keep implementation private to this file */ 1131 ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change, 1132 cxgbe_media_status); 1133 build_medialist(pi, &pi->nm_media); 1134 create_netmap_ifnet(pi); /* logs errors it something fails */ 1135 #endif 1136 cxgbe_sysctls(pi); 1137 1138 return (0); 1139 } 1140 1141 static int 1142 cxgbe_detach(device_t dev) 1143 { 1144 struct port_info *pi = device_get_softc(dev); 1145 struct adapter *sc = pi->adapter; 1146 struct ifnet *ifp = pi->ifp; 1147 1148 /* Tell if_ioctl and if_init that the port is going away */ 1149 ADAPTER_LOCK(sc); 1150 SET_DOOMED(pi); 1151 wakeup(&sc->flags); 1152 while (IS_BUSY(sc)) 1153 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 1154 SET_BUSY(sc); 1155 #ifdef INVARIANTS 1156 sc->last_op = "t4detach"; 1157 sc->last_op_thr = curthread; 1158 #endif 1159 ADAPTER_UNLOCK(sc); 1160 1161 if (pi->flags & HAS_TRACEQ) { 1162 sc->traceq = -1; /* cloner should not create ifnet */ 1163 t4_tracer_port_detach(sc); 1164 } 1165 1166 if (pi->vlan_c) 1167 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c); 1168 1169 PORT_LOCK(pi); 1170 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1171 callout_stop(&pi->tick); 1172 PORT_UNLOCK(pi); 1173 callout_drain(&pi->tick); 1174 1175 /* Let detach proceed even if these fail. */ 1176 cxgbe_uninit_synchronized(pi); 1177 port_full_uninit(pi); 1178 1179 ifmedia_removeall(&pi->media); 1180 ether_ifdetach(pi->ifp); 1181 if_free(pi->ifp); 1182 1183 #ifdef DEV_NETMAP 1184 /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */ 1185 destroy_netmap_ifnet(pi); 1186 #endif 1187 1188 ADAPTER_LOCK(sc); 1189 CLR_BUSY(sc); 1190 wakeup(&sc->flags); 1191 ADAPTER_UNLOCK(sc); 1192 1193 return (0); 1194 } 1195 1196 static void 1197 cxgbe_init(void *arg) 1198 { 1199 struct port_info *pi = arg; 1200 struct adapter *sc = pi->adapter; 1201 1202 if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0) 1203 return; 1204 cxgbe_init_synchronized(pi); 1205 end_synchronized_op(sc, 0); 1206 } 1207 1208 static int 1209 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1210 { 1211 int rc = 0, mtu, flags, can_sleep; 1212 struct port_info *pi = ifp->if_softc; 1213 struct adapter *sc = pi->adapter; 1214 struct ifreq *ifr = (struct ifreq *)data; 1215 uint32_t mask; 1216 1217 switch (cmd) { 1218 case SIOCSIFMTU: 1219 mtu = ifr->ifr_mtu; 1220 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1221 return (EINVAL); 1222 1223 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu"); 1224 if (rc) 1225 return (rc); 1226 ifp->if_mtu = mtu; 1227 if (pi->flags & PORT_INIT_DONE) { 1228 t4_update_fl_bufsize(ifp); 1229 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1230 rc = update_mac_settings(ifp, XGMAC_MTU); 1231 } 1232 end_synchronized_op(sc, 0); 1233 break; 1234 1235 case SIOCSIFFLAGS: 1236 can_sleep = 0; 1237 redo_sifflags: 1238 rc = begin_synchronized_op(sc, pi, 1239 can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg"); 1240 if (rc) 1241 return (rc); 1242 1243 if (ifp->if_flags & IFF_UP) { 1244 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1245 flags = pi->if_flags; 1246 if ((ifp->if_flags ^ flags) & 1247 (IFF_PROMISC | IFF_ALLMULTI)) { 1248 if (can_sleep == 1) { 1249 end_synchronized_op(sc, 0); 1250 can_sleep = 0; 1251 goto redo_sifflags; 1252 } 1253 rc = update_mac_settings(ifp, 1254 XGMAC_PROMISC | XGMAC_ALLMULTI); 1255 } 1256 } else { 1257 if (can_sleep == 0) { 1258 end_synchronized_op(sc, LOCK_HELD); 1259 can_sleep = 1; 1260 goto redo_sifflags; 1261 } 1262 rc = cxgbe_init_synchronized(pi); 1263 } 1264 pi->if_flags = ifp->if_flags; 1265 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1266 if (can_sleep == 0) { 1267 end_synchronized_op(sc, LOCK_HELD); 1268 can_sleep = 1; 1269 goto redo_sifflags; 1270 } 1271 rc = cxgbe_uninit_synchronized(pi); 1272 } 1273 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD); 1274 break; 1275 1276 case SIOCADDMULTI: 1277 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1278 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi"); 1279 if (rc) 1280 return (rc); 1281 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1282 rc = update_mac_settings(ifp, XGMAC_MCADDRS); 1283 end_synchronized_op(sc, LOCK_HELD); 1284 break; 1285 1286 case SIOCSIFCAP: 1287 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap"); 1288 if (rc) 1289 return (rc); 1290 1291 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1292 if (mask & IFCAP_TXCSUM) { 1293 ifp->if_capenable ^= IFCAP_TXCSUM; 1294 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1295 1296 if (IFCAP_TSO4 & ifp->if_capenable && 1297 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1298 ifp->if_capenable &= ~IFCAP_TSO4; 1299 if_printf(ifp, 1300 "tso4 disabled due to -txcsum.\n"); 1301 } 1302 } 1303 if (mask & IFCAP_TXCSUM_IPV6) { 1304 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1305 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1306 1307 if (IFCAP_TSO6 & ifp->if_capenable && 1308 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1309 ifp->if_capenable &= ~IFCAP_TSO6; 1310 if_printf(ifp, 1311 "tso6 disabled due to -txcsum6.\n"); 1312 } 1313 } 1314 if (mask & IFCAP_RXCSUM) 1315 ifp->if_capenable ^= IFCAP_RXCSUM; 1316 if (mask & IFCAP_RXCSUM_IPV6) 1317 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1318 1319 /* 1320 * Note that we leave CSUM_TSO alone (it is always set). The 1321 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1322 * sending a TSO request our way, so it's sufficient to toggle 1323 * IFCAP_TSOx only. 1324 */ 1325 if (mask & IFCAP_TSO4) { 1326 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1327 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1328 if_printf(ifp, "enable txcsum first.\n"); 1329 rc = EAGAIN; 1330 goto fail; 1331 } 1332 ifp->if_capenable ^= IFCAP_TSO4; 1333 } 1334 if (mask & IFCAP_TSO6) { 1335 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1336 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1337 if_printf(ifp, "enable txcsum6 first.\n"); 1338 rc = EAGAIN; 1339 goto fail; 1340 } 1341 ifp->if_capenable ^= IFCAP_TSO6; 1342 } 1343 if (mask & IFCAP_LRO) { 1344 #if defined(INET) || defined(INET6) 1345 int i; 1346 struct sge_rxq *rxq; 1347 1348 ifp->if_capenable ^= IFCAP_LRO; 1349 for_each_rxq(pi, i, rxq) { 1350 if (ifp->if_capenable & IFCAP_LRO) 1351 rxq->iq.flags |= IQ_LRO_ENABLED; 1352 else 1353 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1354 } 1355 #endif 1356 } 1357 #ifdef TCP_OFFLOAD 1358 if (mask & IFCAP_TOE) { 1359 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1360 1361 rc = toe_capability(pi, enable); 1362 if (rc != 0) 1363 goto fail; 1364 1365 ifp->if_capenable ^= mask; 1366 } 1367 #endif 1368 if (mask & IFCAP_VLAN_HWTAGGING) { 1369 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1370 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1371 rc = update_mac_settings(ifp, XGMAC_VLANEX); 1372 } 1373 if (mask & IFCAP_VLAN_MTU) { 1374 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1375 1376 /* Need to find out how to disable auto-mtu-inflation */ 1377 } 1378 if (mask & IFCAP_VLAN_HWTSO) 1379 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1380 if (mask & IFCAP_VLAN_HWCSUM) 1381 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1382 1383 #ifdef VLAN_CAPABILITIES 1384 VLAN_CAPABILITIES(ifp); 1385 #endif 1386 fail: 1387 end_synchronized_op(sc, 0); 1388 break; 1389 1390 case SIOCSIFMEDIA: 1391 case SIOCGIFMEDIA: 1392 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1393 break; 1394 1395 case SIOCGI2C: { 1396 struct ifi2creq i2c; 1397 1398 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); 1399 if (rc != 0) 1400 break; 1401 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) { 1402 rc = EPERM; 1403 break; 1404 } 1405 if (i2c.len > sizeof(i2c.data)) { 1406 rc = EINVAL; 1407 break; 1408 } 1409 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4i2c"); 1410 if (rc) 1411 return (rc); 1412 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr, 1413 i2c.offset, i2c.len, &i2c.data[0]); 1414 end_synchronized_op(sc, 0); 1415 if (rc == 0) 1416 rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); 1417 break; 1418 } 1419 1420 default: 1421 rc = ether_ioctl(ifp, cmd, data); 1422 } 1423 1424 return (rc); 1425 } 1426 1427 static int 1428 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1429 { 1430 struct port_info *pi = ifp->if_softc; 1431 struct adapter *sc = pi->adapter; 1432 struct sge_txq *txq = &sc->sge.txq[pi->first_txq]; 1433 struct buf_ring *br; 1434 int rc; 1435 1436 M_ASSERTPKTHDR(m); 1437 1438 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1439 m_freem(m); 1440 return (ENETDOWN); 1441 } 1442 1443 /* check if flowid is set */ 1444 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 1445 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) 1446 + pi->rsrv_noflowq); 1447 br = txq->br; 1448 1449 if (TXQ_TRYLOCK(txq) == 0) { 1450 struct sge_eq *eq = &txq->eq; 1451 1452 /* 1453 * It is possible that t4_eth_tx finishes up and releases the 1454 * lock between the TRYLOCK above and the drbr_enqueue here. We 1455 * need to make sure that this mbuf doesn't just sit there in 1456 * the drbr. 1457 */ 1458 1459 rc = drbr_enqueue(ifp, br, m); 1460 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 && 1461 !(eq->flags & EQ_DOOMED)) 1462 callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq); 1463 return (rc); 1464 } 1465 1466 /* 1467 * txq->m is the mbuf that is held up due to a temporary shortage of 1468 * resources and it should be put on the wire first. Then what's in 1469 * drbr and finally the mbuf that was just passed in to us. 1470 * 1471 * Return code should indicate the fate of the mbuf that was passed in 1472 * this time. 1473 */ 1474 1475 TXQ_LOCK_ASSERT_OWNED(txq); 1476 if (drbr_needs_enqueue(ifp, br) || txq->m) { 1477 1478 /* Queued for transmission. */ 1479 1480 rc = drbr_enqueue(ifp, br, m); 1481 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 1482 (void) t4_eth_tx(ifp, txq, m); 1483 TXQ_UNLOCK(txq); 1484 return (rc); 1485 } 1486 1487 /* Direct transmission. */ 1488 rc = t4_eth_tx(ifp, txq, m); 1489 if (rc != 0 && txq->m) 1490 rc = 0; /* held, will be transmitted soon (hopefully) */ 1491 1492 TXQ_UNLOCK(txq); 1493 return (rc); 1494 } 1495 1496 static void 1497 cxgbe_qflush(struct ifnet *ifp) 1498 { 1499 struct port_info *pi = ifp->if_softc; 1500 struct sge_txq *txq; 1501 int i; 1502 struct mbuf *m; 1503 1504 /* queues do not exist if !PORT_INIT_DONE. */ 1505 if (pi->flags & PORT_INIT_DONE) { 1506 for_each_txq(pi, i, txq) { 1507 TXQ_LOCK(txq); 1508 m_freem(txq->m); 1509 txq->m = NULL; 1510 while ((m = buf_ring_dequeue_sc(txq->br)) != NULL) 1511 m_freem(m); 1512 TXQ_UNLOCK(txq); 1513 } 1514 } 1515 if_qflush(ifp); 1516 } 1517 1518 static uint64_t 1519 cxgbe_get_counter(struct ifnet *ifp, ift_counter c) 1520 { 1521 struct port_info *pi = ifp->if_softc; 1522 struct adapter *sc = pi->adapter; 1523 struct port_stats *s = &pi->stats; 1524 1525 cxgbe_refresh_stats(sc, pi); 1526 1527 switch (c) { 1528 case IFCOUNTER_IPACKETS: 1529 return (s->rx_frames - s->rx_pause); 1530 1531 case IFCOUNTER_IERRORS: 1532 return (s->rx_jabber + s->rx_runt + s->rx_too_long + 1533 s->rx_fcs_err + s->rx_len_err); 1534 1535 case IFCOUNTER_OPACKETS: 1536 return (s->tx_frames - s->tx_pause); 1537 1538 case IFCOUNTER_OERRORS: 1539 return (s->tx_error_frames); 1540 1541 case IFCOUNTER_IBYTES: 1542 return (s->rx_octets - s->rx_pause * 64); 1543 1544 case IFCOUNTER_OBYTES: 1545 return (s->tx_octets - s->tx_pause * 64); 1546 1547 case IFCOUNTER_IMCASTS: 1548 return (s->rx_mcast_frames - s->rx_pause); 1549 1550 case IFCOUNTER_OMCASTS: 1551 return (s->tx_mcast_frames - s->tx_pause); 1552 1553 case IFCOUNTER_IQDROPS: 1554 return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 1555 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 1556 s->rx_trunc3 + pi->tnl_cong_drops); 1557 1558 case IFCOUNTER_OQDROPS: { 1559 uint64_t drops; 1560 1561 drops = s->tx_drop; 1562 if (pi->flags & PORT_INIT_DONE) { 1563 int i; 1564 struct sge_txq *txq; 1565 1566 for_each_txq(pi, i, txq) 1567 drops += txq->br->br_drops; 1568 } 1569 1570 return (drops); 1571 1572 } 1573 1574 default: 1575 return (if_get_counter_default(ifp, c)); 1576 } 1577 } 1578 1579 static int 1580 cxgbe_media_change(struct ifnet *ifp) 1581 { 1582 struct port_info *pi = ifp->if_softc; 1583 1584 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1585 1586 return (EOPNOTSUPP); 1587 } 1588 1589 static void 1590 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1591 { 1592 struct port_info *pi = ifp->if_softc; 1593 struct ifmedia *media = NULL; 1594 struct ifmedia_entry *cur; 1595 int speed = pi->link_cfg.speed; 1596 #ifdef INVARIANTS 1597 int data = (pi->port_type << 8) | pi->mod_type; 1598 #endif 1599 1600 if (ifp == pi->ifp) 1601 media = &pi->media; 1602 #ifdef DEV_NETMAP 1603 else if (ifp == pi->nm_ifp) 1604 media = &pi->nm_media; 1605 #endif 1606 MPASS(media != NULL); 1607 1608 cur = media->ifm_cur; 1609 MPASS(cur->ifm_data == data); 1610 1611 ifmr->ifm_status = IFM_AVALID; 1612 if (!pi->link_cfg.link_ok) 1613 return; 1614 1615 ifmr->ifm_status |= IFM_ACTIVE; 1616 1617 /* active and current will differ iff current media is autoselect. */ 1618 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1619 return; 1620 1621 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1622 if (speed == SPEED_10000) 1623 ifmr->ifm_active |= IFM_10G_T; 1624 else if (speed == SPEED_1000) 1625 ifmr->ifm_active |= IFM_1000_T; 1626 else if (speed == SPEED_100) 1627 ifmr->ifm_active |= IFM_100_TX; 1628 else if (speed == SPEED_10) 1629 ifmr->ifm_active |= IFM_10_T; 1630 else 1631 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1632 speed)); 1633 } 1634 1635 void 1636 t4_fatal_err(struct adapter *sc) 1637 { 1638 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1639 t4_intr_disable(sc); 1640 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1641 device_get_nameunit(sc->dev)); 1642 } 1643 1644 static int 1645 map_bars_0_and_4(struct adapter *sc) 1646 { 1647 sc->regs_rid = PCIR_BAR(0); 1648 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1649 &sc->regs_rid, RF_ACTIVE); 1650 if (sc->regs_res == NULL) { 1651 device_printf(sc->dev, "cannot map registers.\n"); 1652 return (ENXIO); 1653 } 1654 sc->bt = rman_get_bustag(sc->regs_res); 1655 sc->bh = rman_get_bushandle(sc->regs_res); 1656 sc->mmio_len = rman_get_size(sc->regs_res); 1657 setbit(&sc->doorbells, DOORBELL_KDB); 1658 1659 sc->msix_rid = PCIR_BAR(4); 1660 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1661 &sc->msix_rid, RF_ACTIVE); 1662 if (sc->msix_res == NULL) { 1663 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1664 return (ENXIO); 1665 } 1666 1667 return (0); 1668 } 1669 1670 static int 1671 map_bar_2(struct adapter *sc) 1672 { 1673 1674 /* 1675 * T4: only iWARP driver uses the userspace doorbells. There is no need 1676 * to map it if RDMA is disabled. 1677 */ 1678 if (is_t4(sc) && sc->rdmacaps == 0) 1679 return (0); 1680 1681 sc->udbs_rid = PCIR_BAR(2); 1682 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1683 &sc->udbs_rid, RF_ACTIVE); 1684 if (sc->udbs_res == NULL) { 1685 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1686 return (ENXIO); 1687 } 1688 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1689 1690 if (is_t5(sc)) { 1691 setbit(&sc->doorbells, DOORBELL_UDB); 1692 #if defined(__i386__) || defined(__amd64__) 1693 if (t5_write_combine) { 1694 int rc; 1695 1696 /* 1697 * Enable write combining on BAR2. This is the 1698 * userspace doorbell BAR and is split into 128B 1699 * (UDBS_SEG_SIZE) doorbell regions, each associated 1700 * with an egress queue. The first 64B has the doorbell 1701 * and the second 64B can be used to submit a tx work 1702 * request with an implicit doorbell. 1703 */ 1704 1705 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1706 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1707 if (rc == 0) { 1708 clrbit(&sc->doorbells, DOORBELL_UDB); 1709 setbit(&sc->doorbells, DOORBELL_WCWR); 1710 setbit(&sc->doorbells, DOORBELL_UDBWC); 1711 } else { 1712 device_printf(sc->dev, 1713 "couldn't enable write combining: %d\n", 1714 rc); 1715 } 1716 1717 t4_write_reg(sc, A_SGE_STAT_CFG, 1718 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1719 } 1720 #endif 1721 } 1722 1723 return (0); 1724 } 1725 1726 static const struct memwin t4_memwin[] = { 1727 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1728 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1729 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1730 }; 1731 1732 static const struct memwin t5_memwin[] = { 1733 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1734 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1735 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1736 }; 1737 1738 static void 1739 setup_memwin(struct adapter *sc) 1740 { 1741 const struct memwin *mw; 1742 int i, n; 1743 uint32_t bar0; 1744 1745 if (is_t4(sc)) { 1746 /* 1747 * Read low 32b of bar0 indirectly via the hardware backdoor 1748 * mechanism. Works from within PCI passthrough environments 1749 * too, where rman_get_start() can return a different value. We 1750 * need to program the T4 memory window decoders with the actual 1751 * addresses that will be coming across the PCIe link. 1752 */ 1753 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1754 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1755 1756 mw = &t4_memwin[0]; 1757 n = nitems(t4_memwin); 1758 } else { 1759 /* T5 uses the relative offset inside the PCIe BAR */ 1760 bar0 = 0; 1761 1762 mw = &t5_memwin[0]; 1763 n = nitems(t5_memwin); 1764 } 1765 1766 for (i = 0; i < n; i++, mw++) { 1767 t4_write_reg(sc, 1768 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 1769 (mw->base + bar0) | V_BIR(0) | 1770 V_WINDOW(ilog2(mw->aperture) - 10)); 1771 } 1772 1773 /* flush */ 1774 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1775 } 1776 1777 /* 1778 * Verify that the memory range specified by the addr/len pair is valid and lies 1779 * entirely within a single region (EDCx or MCx). 1780 */ 1781 static int 1782 validate_mem_range(struct adapter *sc, uint32_t addr, int len) 1783 { 1784 uint32_t em, addr_len, maddr, mlen; 1785 1786 /* Memory can only be accessed in naturally aligned 4 byte units */ 1787 if (addr & 3 || len & 3 || len == 0) 1788 return (EINVAL); 1789 1790 /* Enabled memories */ 1791 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1792 if (em & F_EDRAM0_ENABLE) { 1793 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1794 maddr = G_EDRAM0_BASE(addr_len) << 20; 1795 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1796 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1797 addr + len <= maddr + mlen) 1798 return (0); 1799 } 1800 if (em & F_EDRAM1_ENABLE) { 1801 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1802 maddr = G_EDRAM1_BASE(addr_len) << 20; 1803 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1804 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1805 addr + len <= maddr + mlen) 1806 return (0); 1807 } 1808 if (em & F_EXT_MEM_ENABLE) { 1809 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1810 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1811 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1812 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1813 addr + len <= maddr + mlen) 1814 return (0); 1815 } 1816 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 1817 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1818 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1819 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1820 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1821 addr + len <= maddr + mlen) 1822 return (0); 1823 } 1824 1825 return (EFAULT); 1826 } 1827 1828 static int 1829 fwmtype_to_hwmtype(int mtype) 1830 { 1831 1832 switch (mtype) { 1833 case FW_MEMTYPE_EDC0: 1834 return (MEM_EDC0); 1835 case FW_MEMTYPE_EDC1: 1836 return (MEM_EDC1); 1837 case FW_MEMTYPE_EXTMEM: 1838 return (MEM_MC0); 1839 case FW_MEMTYPE_EXTMEM1: 1840 return (MEM_MC1); 1841 default: 1842 panic("%s: cannot translate fw mtype %d.", __func__, mtype); 1843 } 1844 } 1845 1846 /* 1847 * Verify that the memory range specified by the memtype/offset/len pair is 1848 * valid and lies entirely within the memtype specified. The global address of 1849 * the start of the range is returned in addr. 1850 */ 1851 static int 1852 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 1853 uint32_t *addr) 1854 { 1855 uint32_t em, addr_len, maddr, mlen; 1856 1857 /* Memory can only be accessed in naturally aligned 4 byte units */ 1858 if (off & 3 || len & 3 || len == 0) 1859 return (EINVAL); 1860 1861 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1862 switch (fwmtype_to_hwmtype(mtype)) { 1863 case MEM_EDC0: 1864 if (!(em & F_EDRAM0_ENABLE)) 1865 return (EINVAL); 1866 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1867 maddr = G_EDRAM0_BASE(addr_len) << 20; 1868 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1869 break; 1870 case MEM_EDC1: 1871 if (!(em & F_EDRAM1_ENABLE)) 1872 return (EINVAL); 1873 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1874 maddr = G_EDRAM1_BASE(addr_len) << 20; 1875 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1876 break; 1877 case MEM_MC: 1878 if (!(em & F_EXT_MEM_ENABLE)) 1879 return (EINVAL); 1880 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1881 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1882 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1883 break; 1884 case MEM_MC1: 1885 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 1886 return (EINVAL); 1887 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1888 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1889 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1890 break; 1891 default: 1892 return (EINVAL); 1893 } 1894 1895 if (mlen > 0 && off < mlen && off + len <= mlen) { 1896 *addr = maddr + off; /* global address */ 1897 return (0); 1898 } 1899 1900 return (EFAULT); 1901 } 1902 1903 static void 1904 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 1905 { 1906 const struct memwin *mw; 1907 1908 if (is_t4(sc)) { 1909 KASSERT(win >= 0 && win < nitems(t4_memwin), 1910 ("%s: incorrect memwin# (%d)", __func__, win)); 1911 mw = &t4_memwin[win]; 1912 } else { 1913 KASSERT(win >= 0 && win < nitems(t5_memwin), 1914 ("%s: incorrect memwin# (%d)", __func__, win)); 1915 mw = &t5_memwin[win]; 1916 } 1917 1918 if (base != NULL) 1919 *base = mw->base; 1920 if (aperture != NULL) 1921 *aperture = mw->aperture; 1922 } 1923 1924 /* 1925 * Positions the memory window such that it can be used to access the specified 1926 * address in the chip's address space. The return value is the offset of addr 1927 * from the start of the window. 1928 */ 1929 static uint32_t 1930 position_memwin(struct adapter *sc, int n, uint32_t addr) 1931 { 1932 uint32_t start, pf; 1933 uint32_t reg; 1934 1935 KASSERT(n >= 0 && n <= 3, 1936 ("%s: invalid window %d.", __func__, n)); 1937 KASSERT((addr & 3) == 0, 1938 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 1939 1940 if (is_t4(sc)) { 1941 pf = 0; 1942 start = addr & ~0xf; /* start must be 16B aligned */ 1943 } else { 1944 pf = V_PFNUM(sc->pf); 1945 start = addr & ~0x7f; /* start must be 128B aligned */ 1946 } 1947 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 1948 1949 t4_write_reg(sc, reg, start | pf); 1950 t4_read_reg(sc, reg); 1951 1952 return (addr - start); 1953 } 1954 1955 static int 1956 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1957 struct intrs_and_queues *iaq) 1958 { 1959 int rc, itype, navail, nrxq10g, nrxq1g, n; 1960 int nofldrxq10g = 0, nofldrxq1g = 0; 1961 int nnmrxq10g = 0, nnmrxq1g = 0; 1962 1963 bzero(iaq, sizeof(*iaq)); 1964 1965 iaq->ntxq10g = t4_ntxq10g; 1966 iaq->ntxq1g = t4_ntxq1g; 1967 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1968 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1969 iaq->rsrv_noflowq = t4_rsrv_noflowq; 1970 #ifdef TCP_OFFLOAD 1971 if (is_offload(sc)) { 1972 iaq->nofldtxq10g = t4_nofldtxq10g; 1973 iaq->nofldtxq1g = t4_nofldtxq1g; 1974 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1975 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1976 } 1977 #endif 1978 #ifdef DEV_NETMAP 1979 iaq->nnmtxq10g = t4_nnmtxq10g; 1980 iaq->nnmtxq1g = t4_nnmtxq1g; 1981 iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g; 1982 iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g; 1983 #endif 1984 1985 for (itype = INTR_MSIX; itype; itype >>= 1) { 1986 1987 if ((itype & t4_intr_types) == 0) 1988 continue; /* not allowed */ 1989 1990 if (itype == INTR_MSIX) 1991 navail = pci_msix_count(sc->dev); 1992 else if (itype == INTR_MSI) 1993 navail = pci_msi_count(sc->dev); 1994 else 1995 navail = 1; 1996 restart: 1997 if (navail == 0) 1998 continue; 1999 2000 iaq->intr_type = itype; 2001 iaq->intr_flags_10g = 0; 2002 iaq->intr_flags_1g = 0; 2003 2004 /* 2005 * Best option: an interrupt vector for errors, one for the 2006 * firmware event queue, and one for every rxq (NIC, TOE, and 2007 * netmap). 2008 */ 2009 iaq->nirq = T4_EXTRA_INTR; 2010 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g); 2011 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g); 2012 if (iaq->nirq <= navail && 2013 (itype != INTR_MSI || powerof2(iaq->nirq))) { 2014 iaq->intr_flags_10g = INTR_ALL; 2015 iaq->intr_flags_1g = INTR_ALL; 2016 goto allocate; 2017 } 2018 2019 /* 2020 * Second best option: a vector for errors, one for the firmware 2021 * event queue, and vectors for either all the NIC rx queues or 2022 * all the TOE rx queues. The queues that don't get vectors 2023 * will forward their interrupts to those that do. 2024 * 2025 * Note: netmap rx queues cannot be created early and so they 2026 * can't be setup to receive forwarded interrupts for others. 2027 */ 2028 iaq->nirq = T4_EXTRA_INTR; 2029 if (nrxq10g >= nofldrxq10g) { 2030 iaq->intr_flags_10g = INTR_RXQ; 2031 iaq->nirq += n10g * nrxq10g; 2032 #ifdef DEV_NETMAP 2033 iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g); 2034 #endif 2035 } else { 2036 iaq->intr_flags_10g = INTR_OFLD_RXQ; 2037 iaq->nirq += n10g * nofldrxq10g; 2038 #ifdef DEV_NETMAP 2039 iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g); 2040 #endif 2041 } 2042 if (nrxq1g >= nofldrxq1g) { 2043 iaq->intr_flags_1g = INTR_RXQ; 2044 iaq->nirq += n1g * nrxq1g; 2045 #ifdef DEV_NETMAP 2046 iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g); 2047 #endif 2048 } else { 2049 iaq->intr_flags_1g = INTR_OFLD_RXQ; 2050 iaq->nirq += n1g * nofldrxq1g; 2051 #ifdef DEV_NETMAP 2052 iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g); 2053 #endif 2054 } 2055 if (iaq->nirq <= navail && 2056 (itype != INTR_MSI || powerof2(iaq->nirq))) 2057 goto allocate; 2058 2059 /* 2060 * Next best option: an interrupt vector for errors, one for the 2061 * firmware event queue, and at least one per port. At this 2062 * point we know we'll have to downsize nrxq and/or nofldrxq 2063 * and/or nnmrxq to fit what's available to us. 2064 */ 2065 iaq->nirq = T4_EXTRA_INTR; 2066 iaq->nirq += n10g + n1g; 2067 if (iaq->nirq <= navail) { 2068 int leftover = navail - iaq->nirq; 2069 2070 if (n10g > 0) { 2071 int target = max(nrxq10g, nofldrxq10g); 2072 2073 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ? 2074 INTR_RXQ : INTR_OFLD_RXQ; 2075 2076 n = 1; 2077 while (n < target && leftover >= n10g) { 2078 leftover -= n10g; 2079 iaq->nirq += n10g; 2080 n++; 2081 } 2082 iaq->nrxq10g = min(n, nrxq10g); 2083 #ifdef TCP_OFFLOAD 2084 iaq->nofldrxq10g = min(n, nofldrxq10g); 2085 #endif 2086 #ifdef DEV_NETMAP 2087 iaq->nnmrxq10g = min(n, nnmrxq10g); 2088 #endif 2089 } 2090 2091 if (n1g > 0) { 2092 int target = max(nrxq1g, nofldrxq1g); 2093 2094 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ? 2095 INTR_RXQ : INTR_OFLD_RXQ; 2096 2097 n = 1; 2098 while (n < target && leftover >= n1g) { 2099 leftover -= n1g; 2100 iaq->nirq += n1g; 2101 n++; 2102 } 2103 iaq->nrxq1g = min(n, nrxq1g); 2104 #ifdef TCP_OFFLOAD 2105 iaq->nofldrxq1g = min(n, nofldrxq1g); 2106 #endif 2107 #ifdef DEV_NETMAP 2108 iaq->nnmrxq1g = min(n, nnmrxq1g); 2109 #endif 2110 } 2111 2112 if (itype != INTR_MSI || powerof2(iaq->nirq)) 2113 goto allocate; 2114 } 2115 2116 /* 2117 * Least desirable option: one interrupt vector for everything. 2118 */ 2119 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 2120 iaq->intr_flags_10g = iaq->intr_flags_1g = 0; 2121 #ifdef TCP_OFFLOAD 2122 if (is_offload(sc)) 2123 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 2124 #endif 2125 #ifdef DEV_NETMAP 2126 iaq->nnmrxq10g = iaq->nnmrxq1g = 1; 2127 #endif 2128 2129 allocate: 2130 navail = iaq->nirq; 2131 rc = 0; 2132 if (itype == INTR_MSIX) 2133 rc = pci_alloc_msix(sc->dev, &navail); 2134 else if (itype == INTR_MSI) 2135 rc = pci_alloc_msi(sc->dev, &navail); 2136 2137 if (rc == 0) { 2138 if (navail == iaq->nirq) 2139 return (0); 2140 2141 /* 2142 * Didn't get the number requested. Use whatever number 2143 * the kernel is willing to allocate (it's in navail). 2144 */ 2145 device_printf(sc->dev, "fewer vectors than requested, " 2146 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 2147 itype, iaq->nirq, navail); 2148 pci_release_msi(sc->dev); 2149 goto restart; 2150 } 2151 2152 device_printf(sc->dev, 2153 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 2154 itype, rc, iaq->nirq, navail); 2155 } 2156 2157 device_printf(sc->dev, 2158 "failed to find a usable interrupt type. " 2159 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 2160 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 2161 2162 return (ENXIO); 2163 } 2164 2165 #define FW_VERSION(chip) ( \ 2166 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ 2167 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ 2168 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ 2169 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) 2170 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) 2171 2172 struct fw_info { 2173 uint8_t chip; 2174 char *kld_name; 2175 char *fw_mod_name; 2176 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 2177 } fw_info[] = { 2178 { 2179 .chip = CHELSIO_T4, 2180 .kld_name = "t4fw_cfg", 2181 .fw_mod_name = "t4fw", 2182 .fw_hdr = { 2183 .chip = FW_HDR_CHIP_T4, 2184 .fw_ver = htobe32_const(FW_VERSION(T4)), 2185 .intfver_nic = FW_INTFVER(T4, NIC), 2186 .intfver_vnic = FW_INTFVER(T4, VNIC), 2187 .intfver_ofld = FW_INTFVER(T4, OFLD), 2188 .intfver_ri = FW_INTFVER(T4, RI), 2189 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 2190 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 2191 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 2192 .intfver_fcoe = FW_INTFVER(T4, FCOE), 2193 }, 2194 }, { 2195 .chip = CHELSIO_T5, 2196 .kld_name = "t5fw_cfg", 2197 .fw_mod_name = "t5fw", 2198 .fw_hdr = { 2199 .chip = FW_HDR_CHIP_T5, 2200 .fw_ver = htobe32_const(FW_VERSION(T5)), 2201 .intfver_nic = FW_INTFVER(T5, NIC), 2202 .intfver_vnic = FW_INTFVER(T5, VNIC), 2203 .intfver_ofld = FW_INTFVER(T5, OFLD), 2204 .intfver_ri = FW_INTFVER(T5, RI), 2205 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 2206 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 2207 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 2208 .intfver_fcoe = FW_INTFVER(T5, FCOE), 2209 }, 2210 } 2211 }; 2212 2213 static struct fw_info * 2214 find_fw_info(int chip) 2215 { 2216 int i; 2217 2218 for (i = 0; i < nitems(fw_info); i++) { 2219 if (fw_info[i].chip == chip) 2220 return (&fw_info[i]); 2221 } 2222 return (NULL); 2223 } 2224 2225 /* 2226 * Is the given firmware API compatible with the one the driver was compiled 2227 * with? 2228 */ 2229 static int 2230 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 2231 { 2232 2233 /* short circuit if it's the exact same firmware version */ 2234 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 2235 return (1); 2236 2237 /* 2238 * XXX: Is this too conservative? Perhaps I should limit this to the 2239 * features that are supported in the driver. 2240 */ 2241 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 2242 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 2243 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 2244 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 2245 return (1); 2246 #undef SAME_INTF 2247 2248 return (0); 2249 } 2250 2251 /* 2252 * The firmware in the KLD is usable, but should it be installed? This routine 2253 * explains itself in detail if it indicates the KLD firmware should be 2254 * installed. 2255 */ 2256 static int 2257 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 2258 { 2259 const char *reason; 2260 2261 if (!card_fw_usable) { 2262 reason = "incompatible or unusable"; 2263 goto install; 2264 } 2265 2266 if (k > c) { 2267 reason = "older than the version bundled with this driver"; 2268 goto install; 2269 } 2270 2271 if (t4_fw_install == 2 && k != c) { 2272 reason = "different than the version bundled with this driver"; 2273 goto install; 2274 } 2275 2276 return (0); 2277 2278 install: 2279 if (t4_fw_install == 0) { 2280 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2281 "but the driver is prohibited from installing a different " 2282 "firmware on the card.\n", 2283 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2284 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 2285 2286 return (0); 2287 } 2288 2289 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2290 "installing firmware %u.%u.%u.%u on card.\n", 2291 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2292 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 2293 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2294 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2295 2296 return (1); 2297 } 2298 /* 2299 * Establish contact with the firmware and determine if we are the master driver 2300 * or not, and whether we are responsible for chip initialization. 2301 */ 2302 static int 2303 prep_firmware(struct adapter *sc) 2304 { 2305 const struct firmware *fw = NULL, *default_cfg; 2306 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 2307 enum dev_state state; 2308 struct fw_info *fw_info; 2309 struct fw_hdr *card_fw; /* fw on the card */ 2310 const struct fw_hdr *kld_fw; /* fw in the KLD */ 2311 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 2312 against */ 2313 2314 /* Contact firmware. */ 2315 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 2316 if (rc < 0 || state == DEV_STATE_ERR) { 2317 rc = -rc; 2318 device_printf(sc->dev, 2319 "failed to connect to the firmware: %d, %d.\n", rc, state); 2320 return (rc); 2321 } 2322 pf = rc; 2323 if (pf == sc->mbox) 2324 sc->flags |= MASTER_PF; 2325 else if (state == DEV_STATE_UNINIT) { 2326 /* 2327 * We didn't get to be the master so we definitely won't be 2328 * configuring the chip. It's a bug if someone else hasn't 2329 * configured it already. 2330 */ 2331 device_printf(sc->dev, "couldn't be master(%d), " 2332 "device not already initialized either(%d).\n", rc, state); 2333 return (EDOOFUS); 2334 } 2335 2336 /* This is the firmware whose headers the driver was compiled against */ 2337 fw_info = find_fw_info(chip_id(sc)); 2338 if (fw_info == NULL) { 2339 device_printf(sc->dev, 2340 "unable to look up firmware information for chip %d.\n", 2341 chip_id(sc)); 2342 return (EINVAL); 2343 } 2344 drv_fw = &fw_info->fw_hdr; 2345 2346 /* 2347 * The firmware KLD contains many modules. The KLD name is also the 2348 * name of the module that contains the default config file. 2349 */ 2350 default_cfg = firmware_get(fw_info->kld_name); 2351 2352 /* Read the header of the firmware on the card */ 2353 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 2354 rc = -t4_read_flash(sc, FLASH_FW_START, 2355 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 2356 if (rc == 0) 2357 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 2358 else { 2359 device_printf(sc->dev, 2360 "Unable to read card's firmware header: %d\n", rc); 2361 card_fw_usable = 0; 2362 } 2363 2364 /* This is the firmware in the KLD */ 2365 fw = firmware_get(fw_info->fw_mod_name); 2366 if (fw != NULL) { 2367 kld_fw = (const void *)fw->data; 2368 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 2369 } else { 2370 kld_fw = NULL; 2371 kld_fw_usable = 0; 2372 } 2373 2374 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2375 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) { 2376 /* 2377 * Common case: the firmware on the card is an exact match and 2378 * the KLD is an exact match too, or the KLD is 2379 * absent/incompatible. Note that t4_fw_install = 2 is ignored 2380 * here -- use cxgbetool loadfw if you want to reinstall the 2381 * same firmware as the one on the card. 2382 */ 2383 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2384 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2385 be32toh(card_fw->fw_ver))) { 2386 2387 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 2388 if (rc != 0) { 2389 device_printf(sc->dev, 2390 "failed to install firmware: %d\n", rc); 2391 goto done; 2392 } 2393 2394 /* Installed successfully, update the cached header too. */ 2395 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2396 card_fw_usable = 1; 2397 need_fw_reset = 0; /* already reset as part of load_fw */ 2398 } 2399 2400 if (!card_fw_usable) { 2401 uint32_t d, c, k; 2402 2403 d = ntohl(drv_fw->fw_ver); 2404 c = ntohl(card_fw->fw_ver); 2405 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2406 2407 device_printf(sc->dev, "Cannot find a usable firmware: " 2408 "fw_install %d, chip state %d, " 2409 "driver compiled with %d.%d.%d.%d, " 2410 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2411 t4_fw_install, state, 2412 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2413 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2414 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2415 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2416 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2417 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2418 rc = EINVAL; 2419 goto done; 2420 } 2421 2422 /* We're using whatever's on the card and it's known to be good. */ 2423 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2424 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2425 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2426 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2427 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2428 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2429 t4_get_tp_version(sc, &sc->params.tp_vers); 2430 2431 /* Reset device */ 2432 if (need_fw_reset && 2433 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2434 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2435 if (rc != ETIMEDOUT && rc != EIO) 2436 t4_fw_bye(sc, sc->mbox); 2437 goto done; 2438 } 2439 sc->flags |= FW_OK; 2440 2441 rc = get_params__pre_init(sc); 2442 if (rc != 0) 2443 goto done; /* error message displayed already */ 2444 2445 /* Partition adapter resources as specified in the config file. */ 2446 if (state == DEV_STATE_UNINIT) { 2447 2448 KASSERT(sc->flags & MASTER_PF, 2449 ("%s: trying to change chip settings when not master.", 2450 __func__)); 2451 2452 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2453 if (rc != 0) 2454 goto done; /* error message displayed already */ 2455 2456 t4_tweak_chip_settings(sc); 2457 2458 /* get basic stuff going */ 2459 rc = -t4_fw_initialize(sc, sc->mbox); 2460 if (rc != 0) { 2461 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2462 goto done; 2463 } 2464 } else { 2465 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2466 sc->cfcsum = 0; 2467 } 2468 2469 done: 2470 free(card_fw, M_CXGBE); 2471 if (fw != NULL) 2472 firmware_put(fw, FIRMWARE_UNLOAD); 2473 if (default_cfg != NULL) 2474 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2475 2476 return (rc); 2477 } 2478 2479 #define FW_PARAM_DEV(param) \ 2480 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2481 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2482 #define FW_PARAM_PFVF(param) \ 2483 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2484 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2485 2486 /* 2487 * Partition chip resources for use between various PFs, VFs, etc. 2488 */ 2489 static int 2490 partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2491 const char *name_prefix) 2492 { 2493 const struct firmware *cfg = NULL; 2494 int rc = 0; 2495 struct fw_caps_config_cmd caps; 2496 uint32_t mtype, moff, finicsum, cfcsum; 2497 2498 /* 2499 * Figure out what configuration file to use. Pick the default config 2500 * file for the card if the user hasn't specified one explicitly. 2501 */ 2502 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2503 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2504 /* Card specific overrides go here. */ 2505 if (pci_get_device(sc->dev) == 0x440a) 2506 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2507 if (is_fpga(sc)) 2508 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2509 } 2510 2511 /* 2512 * We need to load another module if the profile is anything except 2513 * "default" or "flash". 2514 */ 2515 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2516 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2517 char s[32]; 2518 2519 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2520 cfg = firmware_get(s); 2521 if (cfg == NULL) { 2522 if (default_cfg != NULL) { 2523 device_printf(sc->dev, 2524 "unable to load module \"%s\" for " 2525 "configuration profile \"%s\", will use " 2526 "the default config file instead.\n", 2527 s, sc->cfg_file); 2528 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2529 "%s", DEFAULT_CF); 2530 } else { 2531 device_printf(sc->dev, 2532 "unable to load module \"%s\" for " 2533 "configuration profile \"%s\", will use " 2534 "the config file on the card's flash " 2535 "instead.\n", s, sc->cfg_file); 2536 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2537 "%s", FLASH_CF); 2538 } 2539 } 2540 } 2541 2542 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2543 default_cfg == NULL) { 2544 device_printf(sc->dev, 2545 "default config file not available, will use the config " 2546 "file on the card's flash instead.\n"); 2547 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2548 } 2549 2550 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2551 u_int cflen, i, n; 2552 const uint32_t *cfdata; 2553 uint32_t param, val, addr, off, mw_base, mw_aperture; 2554 2555 KASSERT(cfg != NULL || default_cfg != NULL, 2556 ("%s: no config to upload", __func__)); 2557 2558 /* 2559 * Ask the firmware where it wants us to upload the config file. 2560 */ 2561 param = FW_PARAM_DEV(CF); 2562 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2563 if (rc != 0) { 2564 /* No support for config file? Shouldn't happen. */ 2565 device_printf(sc->dev, 2566 "failed to query config file location: %d.\n", rc); 2567 goto done; 2568 } 2569 mtype = G_FW_PARAMS_PARAM_Y(val); 2570 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2571 2572 /* 2573 * XXX: sheer laziness. We deliberately added 4 bytes of 2574 * useless stuffing/comments at the end of the config file so 2575 * it's ok to simply throw away the last remaining bytes when 2576 * the config file is not an exact multiple of 4. This also 2577 * helps with the validate_mt_off_len check. 2578 */ 2579 if (cfg != NULL) { 2580 cflen = cfg->datasize & ~3; 2581 cfdata = cfg->data; 2582 } else { 2583 cflen = default_cfg->datasize & ~3; 2584 cfdata = default_cfg->data; 2585 } 2586 2587 if (cflen > FLASH_CFG_MAX_SIZE) { 2588 device_printf(sc->dev, 2589 "config file too long (%d, max allowed is %d). " 2590 "Will try to use the config on the card, if any.\n", 2591 cflen, FLASH_CFG_MAX_SIZE); 2592 goto use_config_on_flash; 2593 } 2594 2595 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2596 if (rc != 0) { 2597 device_printf(sc->dev, 2598 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2599 "Will try to use the config on the card, if any.\n", 2600 __func__, mtype, moff, cflen, rc); 2601 goto use_config_on_flash; 2602 } 2603 2604 memwin_info(sc, 2, &mw_base, &mw_aperture); 2605 while (cflen) { 2606 off = position_memwin(sc, 2, addr); 2607 n = min(cflen, mw_aperture - off); 2608 for (i = 0; i < n; i += 4) 2609 t4_write_reg(sc, mw_base + off + i, *cfdata++); 2610 cflen -= n; 2611 addr += n; 2612 } 2613 } else { 2614 use_config_on_flash: 2615 mtype = FW_MEMTYPE_FLASH; 2616 moff = t4_flash_cfg_addr(sc); 2617 } 2618 2619 bzero(&caps, sizeof(caps)); 2620 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2621 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2622 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2623 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2624 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2625 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2626 if (rc != 0) { 2627 device_printf(sc->dev, 2628 "failed to pre-process config file: %d " 2629 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2630 goto done; 2631 } 2632 2633 finicsum = be32toh(caps.finicsum); 2634 cfcsum = be32toh(caps.cfcsum); 2635 if (finicsum != cfcsum) { 2636 device_printf(sc->dev, 2637 "WARNING: config file checksum mismatch: %08x %08x\n", 2638 finicsum, cfcsum); 2639 } 2640 sc->cfcsum = cfcsum; 2641 2642 #define LIMIT_CAPS(x) do { \ 2643 caps.x &= htobe16(t4_##x##_allowed); \ 2644 } while (0) 2645 2646 /* 2647 * Let the firmware know what features will (not) be used so it can tune 2648 * things accordingly. 2649 */ 2650 LIMIT_CAPS(linkcaps); 2651 LIMIT_CAPS(niccaps); 2652 LIMIT_CAPS(toecaps); 2653 LIMIT_CAPS(rdmacaps); 2654 LIMIT_CAPS(iscsicaps); 2655 LIMIT_CAPS(fcoecaps); 2656 #undef LIMIT_CAPS 2657 2658 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2659 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2660 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2661 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2662 if (rc != 0) { 2663 device_printf(sc->dev, 2664 "failed to process config file: %d.\n", rc); 2665 } 2666 done: 2667 if (cfg != NULL) 2668 firmware_put(cfg, FIRMWARE_UNLOAD); 2669 return (rc); 2670 } 2671 2672 /* 2673 * Retrieve parameters that are needed (or nice to have) very early. 2674 */ 2675 static int 2676 get_params__pre_init(struct adapter *sc) 2677 { 2678 int rc; 2679 uint32_t param[2], val[2]; 2680 struct fw_devlog_cmd cmd; 2681 struct devlog_params *dlog = &sc->params.devlog; 2682 2683 param[0] = FW_PARAM_DEV(PORTVEC); 2684 param[1] = FW_PARAM_DEV(CCLK); 2685 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2686 if (rc != 0) { 2687 device_printf(sc->dev, 2688 "failed to query parameters (pre_init): %d.\n", rc); 2689 return (rc); 2690 } 2691 2692 sc->params.portvec = val[0]; 2693 sc->params.nports = bitcount32(val[0]); 2694 sc->params.vpd.cclk = val[1]; 2695 2696 /* Read device log parameters. */ 2697 bzero(&cmd, sizeof(cmd)); 2698 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2699 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2700 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2701 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2702 if (rc != 0) { 2703 device_printf(sc->dev, 2704 "failed to get devlog parameters: %d.\n", rc); 2705 bzero(dlog, sizeof (*dlog)); 2706 rc = 0; /* devlog isn't critical for device operation */ 2707 } else { 2708 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2709 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2710 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2711 dlog->size = be32toh(cmd.memsize_devlog); 2712 } 2713 2714 return (rc); 2715 } 2716 2717 /* 2718 * Retrieve various parameters that are of interest to the driver. The device 2719 * has been initialized by the firmware at this point. 2720 */ 2721 static int 2722 get_params__post_init(struct adapter *sc) 2723 { 2724 int rc; 2725 uint32_t param[7], val[7]; 2726 struct fw_caps_config_cmd caps; 2727 2728 param[0] = FW_PARAM_PFVF(IQFLINT_START); 2729 param[1] = FW_PARAM_PFVF(EQ_START); 2730 param[2] = FW_PARAM_PFVF(FILTER_START); 2731 param[3] = FW_PARAM_PFVF(FILTER_END); 2732 param[4] = FW_PARAM_PFVF(L2T_START); 2733 param[5] = FW_PARAM_PFVF(L2T_END); 2734 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2735 if (rc != 0) { 2736 device_printf(sc->dev, 2737 "failed to query parameters (post_init): %d.\n", rc); 2738 return (rc); 2739 } 2740 2741 sc->sge.iq_start = val[0]; 2742 sc->sge.eq_start = val[1]; 2743 sc->tids.ftid_base = val[2]; 2744 sc->tids.nftids = val[3] - val[2] + 1; 2745 sc->params.ftid_min = val[2]; 2746 sc->params.ftid_max = val[3]; 2747 sc->vres.l2t.start = val[4]; 2748 sc->vres.l2t.size = val[5] - val[4] + 1; 2749 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2750 ("%s: L2 table size (%u) larger than expected (%u)", 2751 __func__, sc->vres.l2t.size, L2T_SIZE)); 2752 2753 /* get capabilites */ 2754 bzero(&caps, sizeof(caps)); 2755 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2756 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2757 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2758 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2759 if (rc != 0) { 2760 device_printf(sc->dev, 2761 "failed to get card capabilities: %d.\n", rc); 2762 return (rc); 2763 } 2764 2765 #define READ_CAPS(x) do { \ 2766 sc->x = htobe16(caps.x); \ 2767 } while (0) 2768 READ_CAPS(linkcaps); 2769 READ_CAPS(niccaps); 2770 READ_CAPS(toecaps); 2771 READ_CAPS(rdmacaps); 2772 READ_CAPS(iscsicaps); 2773 READ_CAPS(fcoecaps); 2774 2775 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { 2776 param[0] = FW_PARAM_PFVF(ETHOFLD_START); 2777 param[1] = FW_PARAM_PFVF(ETHOFLD_END); 2778 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2779 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); 2780 if (rc != 0) { 2781 device_printf(sc->dev, 2782 "failed to query NIC parameters: %d.\n", rc); 2783 return (rc); 2784 } 2785 sc->tids.etid_base = val[0]; 2786 sc->params.etid_min = val[0]; 2787 sc->tids.netids = val[1] - val[0] + 1; 2788 sc->params.netids = sc->tids.netids; 2789 sc->params.eo_wr_cred = val[2]; 2790 sc->params.ethoffload = 1; 2791 } 2792 2793 if (sc->toecaps) { 2794 /* query offload-related parameters */ 2795 param[0] = FW_PARAM_DEV(NTID); 2796 param[1] = FW_PARAM_PFVF(SERVER_START); 2797 param[2] = FW_PARAM_PFVF(SERVER_END); 2798 param[3] = FW_PARAM_PFVF(TDDP_START); 2799 param[4] = FW_PARAM_PFVF(TDDP_END); 2800 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2801 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2802 if (rc != 0) { 2803 device_printf(sc->dev, 2804 "failed to query TOE parameters: %d.\n", rc); 2805 return (rc); 2806 } 2807 sc->tids.ntids = val[0]; 2808 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 2809 sc->tids.stid_base = val[1]; 2810 sc->tids.nstids = val[2] - val[1] + 1; 2811 sc->vres.ddp.start = val[3]; 2812 sc->vres.ddp.size = val[4] - val[3] + 1; 2813 sc->params.ofldq_wr_cred = val[5]; 2814 sc->params.offload = 1; 2815 } 2816 if (sc->rdmacaps) { 2817 param[0] = FW_PARAM_PFVF(STAG_START); 2818 param[1] = FW_PARAM_PFVF(STAG_END); 2819 param[2] = FW_PARAM_PFVF(RQ_START); 2820 param[3] = FW_PARAM_PFVF(RQ_END); 2821 param[4] = FW_PARAM_PFVF(PBL_START); 2822 param[5] = FW_PARAM_PFVF(PBL_END); 2823 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2824 if (rc != 0) { 2825 device_printf(sc->dev, 2826 "failed to query RDMA parameters(1): %d.\n", rc); 2827 return (rc); 2828 } 2829 sc->vres.stag.start = val[0]; 2830 sc->vres.stag.size = val[1] - val[0] + 1; 2831 sc->vres.rq.start = val[2]; 2832 sc->vres.rq.size = val[3] - val[2] + 1; 2833 sc->vres.pbl.start = val[4]; 2834 sc->vres.pbl.size = val[5] - val[4] + 1; 2835 2836 param[0] = FW_PARAM_PFVF(SQRQ_START); 2837 param[1] = FW_PARAM_PFVF(SQRQ_END); 2838 param[2] = FW_PARAM_PFVF(CQ_START); 2839 param[3] = FW_PARAM_PFVF(CQ_END); 2840 param[4] = FW_PARAM_PFVF(OCQ_START); 2841 param[5] = FW_PARAM_PFVF(OCQ_END); 2842 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2843 if (rc != 0) { 2844 device_printf(sc->dev, 2845 "failed to query RDMA parameters(2): %d.\n", rc); 2846 return (rc); 2847 } 2848 sc->vres.qp.start = val[0]; 2849 sc->vres.qp.size = val[1] - val[0] + 1; 2850 sc->vres.cq.start = val[2]; 2851 sc->vres.cq.size = val[3] - val[2] + 1; 2852 sc->vres.ocq.start = val[4]; 2853 sc->vres.ocq.size = val[5] - val[4] + 1; 2854 } 2855 if (sc->iscsicaps) { 2856 param[0] = FW_PARAM_PFVF(ISCSI_START); 2857 param[1] = FW_PARAM_PFVF(ISCSI_END); 2858 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2859 if (rc != 0) { 2860 device_printf(sc->dev, 2861 "failed to query iSCSI parameters: %d.\n", rc); 2862 return (rc); 2863 } 2864 sc->vres.iscsi.start = val[0]; 2865 sc->vres.iscsi.size = val[1] - val[0] + 1; 2866 } 2867 2868 /* 2869 * We've got the params we wanted to query via the firmware. Now grab 2870 * some others directly from the chip. 2871 */ 2872 rc = t4_read_chip_settings(sc); 2873 2874 return (rc); 2875 } 2876 2877 static int 2878 set_params__post_init(struct adapter *sc) 2879 { 2880 uint32_t param, val; 2881 2882 /* ask for encapsulated CPLs */ 2883 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 2884 val = 1; 2885 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2886 2887 return (0); 2888 } 2889 2890 #undef FW_PARAM_PFVF 2891 #undef FW_PARAM_DEV 2892 2893 static void 2894 t4_set_desc(struct adapter *sc) 2895 { 2896 char buf[128]; 2897 struct adapter_params *p = &sc->params; 2898 2899 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, " 2900 "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "", 2901 chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec); 2902 2903 device_set_desc_copy(sc->dev, buf); 2904 } 2905 2906 static void 2907 build_medialist(struct port_info *pi, struct ifmedia *media) 2908 { 2909 int data, m; 2910 2911 PORT_LOCK(pi); 2912 2913 ifmedia_removeall(media); 2914 2915 m = IFM_ETHER | IFM_FDX; 2916 data = (pi->port_type << 8) | pi->mod_type; 2917 2918 switch(pi->port_type) { 2919 case FW_PORT_TYPE_BT_XFI: 2920 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2921 break; 2922 2923 case FW_PORT_TYPE_BT_XAUI: 2924 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2925 /* fall through */ 2926 2927 case FW_PORT_TYPE_BT_SGMII: 2928 ifmedia_add(media, m | IFM_1000_T, data, NULL); 2929 ifmedia_add(media, m | IFM_100_TX, data, NULL); 2930 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL); 2931 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2932 break; 2933 2934 case FW_PORT_TYPE_CX4: 2935 ifmedia_add(media, m | IFM_10G_CX4, data, NULL); 2936 ifmedia_set(media, m | IFM_10G_CX4); 2937 break; 2938 2939 case FW_PORT_TYPE_QSFP_10G: 2940 case FW_PORT_TYPE_SFP: 2941 case FW_PORT_TYPE_FIBER_XFI: 2942 case FW_PORT_TYPE_FIBER_XAUI: 2943 switch (pi->mod_type) { 2944 2945 case FW_PORT_MOD_TYPE_LR: 2946 ifmedia_add(media, m | IFM_10G_LR, data, NULL); 2947 ifmedia_set(media, m | IFM_10G_LR); 2948 break; 2949 2950 case FW_PORT_MOD_TYPE_SR: 2951 ifmedia_add(media, m | IFM_10G_SR, data, NULL); 2952 ifmedia_set(media, m | IFM_10G_SR); 2953 break; 2954 2955 case FW_PORT_MOD_TYPE_LRM: 2956 ifmedia_add(media, m | IFM_10G_LRM, data, NULL); 2957 ifmedia_set(media, m | IFM_10G_LRM); 2958 break; 2959 2960 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2961 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2962 ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL); 2963 ifmedia_set(media, m | IFM_10G_TWINAX); 2964 break; 2965 2966 case FW_PORT_MOD_TYPE_NONE: 2967 m &= ~IFM_FDX; 2968 ifmedia_add(media, m | IFM_NONE, data, NULL); 2969 ifmedia_set(media, m | IFM_NONE); 2970 break; 2971 2972 case FW_PORT_MOD_TYPE_NA: 2973 case FW_PORT_MOD_TYPE_ER: 2974 default: 2975 device_printf(pi->dev, 2976 "unknown port_type (%d), mod_type (%d)\n", 2977 pi->port_type, pi->mod_type); 2978 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2979 ifmedia_set(media, m | IFM_UNKNOWN); 2980 break; 2981 } 2982 break; 2983 2984 case FW_PORT_TYPE_QSFP: 2985 switch (pi->mod_type) { 2986 2987 case FW_PORT_MOD_TYPE_LR: 2988 ifmedia_add(media, m | IFM_40G_LR4, data, NULL); 2989 ifmedia_set(media, m | IFM_40G_LR4); 2990 break; 2991 2992 case FW_PORT_MOD_TYPE_SR: 2993 ifmedia_add(media, m | IFM_40G_SR4, data, NULL); 2994 ifmedia_set(media, m | IFM_40G_SR4); 2995 break; 2996 2997 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2998 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2999 ifmedia_add(media, m | IFM_40G_CR4, data, NULL); 3000 ifmedia_set(media, m | IFM_40G_CR4); 3001 break; 3002 3003 case FW_PORT_MOD_TYPE_NONE: 3004 m &= ~IFM_FDX; 3005 ifmedia_add(media, m | IFM_NONE, data, NULL); 3006 ifmedia_set(media, m | IFM_NONE); 3007 break; 3008 3009 default: 3010 device_printf(pi->dev, 3011 "unknown port_type (%d), mod_type (%d)\n", 3012 pi->port_type, pi->mod_type); 3013 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 3014 ifmedia_set(media, m | IFM_UNKNOWN); 3015 break; 3016 } 3017 break; 3018 3019 default: 3020 device_printf(pi->dev, 3021 "unknown port_type (%d), mod_type (%d)\n", pi->port_type, 3022 pi->mod_type); 3023 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 3024 ifmedia_set(media, m | IFM_UNKNOWN); 3025 break; 3026 } 3027 3028 PORT_UNLOCK(pi); 3029 } 3030 3031 #define FW_MAC_EXACT_CHUNK 7 3032 3033 /* 3034 * Program the port's XGMAC based on parameters in ifnet. The caller also 3035 * indicates which parameters should be programmed (the rest are left alone). 3036 */ 3037 int 3038 update_mac_settings(struct ifnet *ifp, int flags) 3039 { 3040 int rc = 0; 3041 struct port_info *pi = ifp->if_softc; 3042 struct adapter *sc = pi->adapter; 3043 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 3044 uint16_t viid = 0xffff; 3045 int16_t *xact_addr_filt = NULL; 3046 3047 ASSERT_SYNCHRONIZED_OP(sc); 3048 KASSERT(flags, ("%s: not told what to update.", __func__)); 3049 3050 if (ifp == pi->ifp) { 3051 viid = pi->viid; 3052 xact_addr_filt = &pi->xact_addr_filt; 3053 } 3054 #ifdef DEV_NETMAP 3055 else if (ifp == pi->nm_ifp) { 3056 viid = pi->nm_viid; 3057 xact_addr_filt = &pi->nm_xact_addr_filt; 3058 } 3059 #endif 3060 if (flags & XGMAC_MTU) 3061 mtu = ifp->if_mtu; 3062 3063 if (flags & XGMAC_PROMISC) 3064 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 3065 3066 if (flags & XGMAC_ALLMULTI) 3067 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 3068 3069 if (flags & XGMAC_VLANEX) 3070 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 3071 3072 if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) { 3073 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti, 3074 1, vlanex, false); 3075 if (rc) { 3076 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, 3077 rc); 3078 return (rc); 3079 } 3080 } 3081 3082 if (flags & XGMAC_UCADDR) { 3083 uint8_t ucaddr[ETHER_ADDR_LEN]; 3084 3085 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 3086 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr, 3087 true, true); 3088 if (rc < 0) { 3089 rc = -rc; 3090 if_printf(ifp, "change_mac failed: %d\n", rc); 3091 return (rc); 3092 } else { 3093 *xact_addr_filt = rc; 3094 rc = 0; 3095 } 3096 } 3097 3098 if (flags & XGMAC_MCADDRS) { 3099 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 3100 int del = 1; 3101 uint64_t hash = 0; 3102 struct ifmultiaddr *ifma; 3103 int i = 0, j; 3104 3105 if_maddr_rlock(ifp); 3106 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3107 if (ifma->ifma_addr->sa_family != AF_LINK) 3108 continue; 3109 mcaddr[i] = 3110 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 3111 MPASS(ETHER_IS_MULTICAST(mcaddr[i])); 3112 i++; 3113 3114 if (i == FW_MAC_EXACT_CHUNK) { 3115 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, 3116 i, mcaddr, NULL, &hash, 0); 3117 if (rc < 0) { 3118 rc = -rc; 3119 for (j = 0; j < i; j++) { 3120 if_printf(ifp, 3121 "failed to add mc address" 3122 " %02x:%02x:%02x:" 3123 "%02x:%02x:%02x rc=%d\n", 3124 mcaddr[j][0], mcaddr[j][1], 3125 mcaddr[j][2], mcaddr[j][3], 3126 mcaddr[j][4], mcaddr[j][5], 3127 rc); 3128 } 3129 goto mcfail; 3130 } 3131 del = 0; 3132 i = 0; 3133 } 3134 } 3135 if (i > 0) { 3136 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i, 3137 mcaddr, NULL, &hash, 0); 3138 if (rc < 0) { 3139 rc = -rc; 3140 for (j = 0; j < i; j++) { 3141 if_printf(ifp, 3142 "failed to add mc address" 3143 " %02x:%02x:%02x:" 3144 "%02x:%02x:%02x rc=%d\n", 3145 mcaddr[j][0], mcaddr[j][1], 3146 mcaddr[j][2], mcaddr[j][3], 3147 mcaddr[j][4], mcaddr[j][5], 3148 rc); 3149 } 3150 goto mcfail; 3151 } 3152 } 3153 3154 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0); 3155 if (rc != 0) 3156 if_printf(ifp, "failed to set mc address hash: %d", rc); 3157 mcfail: 3158 if_maddr_runlock(ifp); 3159 } 3160 3161 return (rc); 3162 } 3163 3164 int 3165 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags, 3166 char *wmesg) 3167 { 3168 int rc, pri; 3169 3170 #ifdef WITNESS 3171 /* the caller thinks it's ok to sleep, but is it really? */ 3172 if (flags & SLEEP_OK) 3173 pause("t4slptst", 1); 3174 #endif 3175 3176 if (INTR_OK) 3177 pri = PCATCH; 3178 else 3179 pri = 0; 3180 3181 ADAPTER_LOCK(sc); 3182 for (;;) { 3183 3184 if (pi && IS_DOOMED(pi)) { 3185 rc = ENXIO; 3186 goto done; 3187 } 3188 3189 if (!IS_BUSY(sc)) { 3190 rc = 0; 3191 break; 3192 } 3193 3194 if (!(flags & SLEEP_OK)) { 3195 rc = EBUSY; 3196 goto done; 3197 } 3198 3199 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 3200 rc = EINTR; 3201 goto done; 3202 } 3203 } 3204 3205 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 3206 SET_BUSY(sc); 3207 #ifdef INVARIANTS 3208 sc->last_op = wmesg; 3209 sc->last_op_thr = curthread; 3210 #endif 3211 3212 done: 3213 if (!(flags & HOLD_LOCK) || rc) 3214 ADAPTER_UNLOCK(sc); 3215 3216 return (rc); 3217 } 3218 3219 void 3220 end_synchronized_op(struct adapter *sc, int flags) 3221 { 3222 3223 if (flags & LOCK_HELD) 3224 ADAPTER_LOCK_ASSERT_OWNED(sc); 3225 else 3226 ADAPTER_LOCK(sc); 3227 3228 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 3229 CLR_BUSY(sc); 3230 wakeup(&sc->flags); 3231 ADAPTER_UNLOCK(sc); 3232 } 3233 3234 static int 3235 cxgbe_init_synchronized(struct port_info *pi) 3236 { 3237 struct adapter *sc = pi->adapter; 3238 struct ifnet *ifp = pi->ifp; 3239 int rc = 0; 3240 3241 ASSERT_SYNCHRONIZED_OP(sc); 3242 3243 if (isset(&sc->open_device_map, pi->port_id)) { 3244 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 3245 ("mismatch between open_device_map and if_drv_flags")); 3246 return (0); /* already running */ 3247 } 3248 3249 if (!(sc->flags & FULL_INIT_DONE) && 3250 ((rc = adapter_full_init(sc)) != 0)) 3251 return (rc); /* error message displayed already */ 3252 3253 if (!(pi->flags & PORT_INIT_DONE) && 3254 ((rc = port_full_init(pi)) != 0)) 3255 return (rc); /* error message displayed already */ 3256 3257 rc = update_mac_settings(ifp, XGMAC_ALL); 3258 if (rc) 3259 goto done; /* error message displayed already */ 3260 3261 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 3262 if (rc != 0) { 3263 if_printf(ifp, "enable_vi failed: %d\n", rc); 3264 goto done; 3265 } 3266 3267 /* 3268 * The first iq of the first port to come up is used for tracing. 3269 */ 3270 if (sc->traceq < 0) { 3271 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id; 3272 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : 3273 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | 3274 V_QUEUENUMBER(sc->traceq)); 3275 pi->flags |= HAS_TRACEQ; 3276 } 3277 3278 /* all ok */ 3279 setbit(&sc->open_device_map, pi->port_id); 3280 PORT_LOCK(pi); 3281 ifp->if_drv_flags |= IFF_DRV_RUNNING; 3282 PORT_UNLOCK(pi); 3283 3284 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 3285 done: 3286 if (rc != 0) 3287 cxgbe_uninit_synchronized(pi); 3288 3289 return (rc); 3290 } 3291 3292 /* 3293 * Idempotent. 3294 */ 3295 static int 3296 cxgbe_uninit_synchronized(struct port_info *pi) 3297 { 3298 struct adapter *sc = pi->adapter; 3299 struct ifnet *ifp = pi->ifp; 3300 int rc; 3301 3302 ASSERT_SYNCHRONIZED_OP(sc); 3303 3304 /* 3305 * Disable the VI so that all its data in either direction is discarded 3306 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 3307 * tick) intact as the TP can deliver negative advice or data that it's 3308 * holding in its RAM (for an offloaded connection) even after the VI is 3309 * disabled. 3310 */ 3311 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 3312 if (rc) { 3313 if_printf(ifp, "disable_vi failed: %d\n", rc); 3314 return (rc); 3315 } 3316 3317 clrbit(&sc->open_device_map, pi->port_id); 3318 PORT_LOCK(pi); 3319 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3320 PORT_UNLOCK(pi); 3321 3322 pi->link_cfg.link_ok = 0; 3323 pi->link_cfg.speed = 0; 3324 pi->linkdnrc = -1; 3325 t4_os_link_changed(sc, pi->port_id, 0, -1); 3326 3327 return (0); 3328 } 3329 3330 /* 3331 * It is ok for this function to fail midway and return right away. t4_detach 3332 * will walk the entire sc->irq list and clean up whatever is valid. 3333 */ 3334 static int 3335 setup_intr_handlers(struct adapter *sc) 3336 { 3337 int rc, rid, p, q; 3338 char s[8]; 3339 struct irq *irq; 3340 struct port_info *pi; 3341 struct sge_rxq *rxq; 3342 #ifdef TCP_OFFLOAD 3343 struct sge_ofld_rxq *ofld_rxq; 3344 #endif 3345 #ifdef DEV_NETMAP 3346 struct sge_nm_rxq *nm_rxq; 3347 #endif 3348 3349 /* 3350 * Setup interrupts. 3351 */ 3352 irq = &sc->irq[0]; 3353 rid = sc->intr_type == INTR_INTX ? 0 : 1; 3354 if (sc->intr_count == 1) 3355 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all")); 3356 3357 /* Multiple interrupts. */ 3358 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 3359 ("%s: too few intr.", __func__)); 3360 3361 /* The first one is always error intr */ 3362 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 3363 if (rc != 0) 3364 return (rc); 3365 irq++; 3366 rid++; 3367 3368 /* The second one is always the firmware event queue */ 3369 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt"); 3370 if (rc != 0) 3371 return (rc); 3372 irq++; 3373 rid++; 3374 3375 for_each_port(sc, p) { 3376 pi = sc->port[p]; 3377 3378 if (pi->flags & INTR_RXQ) { 3379 for_each_rxq(pi, q, rxq) { 3380 snprintf(s, sizeof(s), "%d.%d", p, q); 3381 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq, 3382 s); 3383 if (rc != 0) 3384 return (rc); 3385 irq++; 3386 rid++; 3387 } 3388 } 3389 #ifdef TCP_OFFLOAD 3390 if (pi->flags & INTR_OFLD_RXQ) { 3391 for_each_ofld_rxq(pi, q, ofld_rxq) { 3392 snprintf(s, sizeof(s), "%d,%d", p, q); 3393 rc = t4_alloc_irq(sc, irq, rid, t4_intr, 3394 ofld_rxq, s); 3395 if (rc != 0) 3396 return (rc); 3397 irq++; 3398 rid++; 3399 } 3400 } 3401 #endif 3402 #ifdef DEV_NETMAP 3403 if (pi->flags & INTR_NM_RXQ) { 3404 for_each_nm_rxq(pi, q, nm_rxq) { 3405 snprintf(s, sizeof(s), "%d-%d", p, q); 3406 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr, 3407 nm_rxq, s); 3408 if (rc != 0) 3409 return (rc); 3410 irq++; 3411 rid++; 3412 } 3413 } 3414 #endif 3415 } 3416 MPASS(irq == &sc->irq[sc->intr_count]); 3417 3418 return (0); 3419 } 3420 3421 int 3422 adapter_full_init(struct adapter *sc) 3423 { 3424 int rc, i; 3425 3426 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3427 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 3428 ("%s: FULL_INIT_DONE already", __func__)); 3429 3430 /* 3431 * queues that belong to the adapter (not any particular port). 3432 */ 3433 rc = t4_setup_adapter_queues(sc); 3434 if (rc != 0) 3435 goto done; 3436 3437 for (i = 0; i < nitems(sc->tq); i++) { 3438 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 3439 taskqueue_thread_enqueue, &sc->tq[i]); 3440 if (sc->tq[i] == NULL) { 3441 device_printf(sc->dev, 3442 "failed to allocate task queue %d\n", i); 3443 rc = ENOMEM; 3444 goto done; 3445 } 3446 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3447 device_get_nameunit(sc->dev), i); 3448 } 3449 3450 t4_intr_enable(sc); 3451 sc->flags |= FULL_INIT_DONE; 3452 done: 3453 if (rc != 0) 3454 adapter_full_uninit(sc); 3455 3456 return (rc); 3457 } 3458 3459 int 3460 adapter_full_uninit(struct adapter *sc) 3461 { 3462 int i; 3463 3464 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3465 3466 t4_teardown_adapter_queues(sc); 3467 3468 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3469 taskqueue_free(sc->tq[i]); 3470 sc->tq[i] = NULL; 3471 } 3472 3473 sc->flags &= ~FULL_INIT_DONE; 3474 3475 return (0); 3476 } 3477 3478 int 3479 port_full_init(struct port_info *pi) 3480 { 3481 struct adapter *sc = pi->adapter; 3482 struct ifnet *ifp = pi->ifp; 3483 uint16_t *rss; 3484 struct sge_rxq *rxq; 3485 int rc, i, j; 3486 3487 ASSERT_SYNCHRONIZED_OP(sc); 3488 KASSERT((pi->flags & PORT_INIT_DONE) == 0, 3489 ("%s: PORT_INIT_DONE already", __func__)); 3490 3491 sysctl_ctx_init(&pi->ctx); 3492 pi->flags |= PORT_SYSCTL_CTX; 3493 3494 /* 3495 * Allocate tx/rx/fl queues for this port. 3496 */ 3497 rc = t4_setup_port_queues(pi); 3498 if (rc != 0) 3499 goto done; /* error message displayed already */ 3500 3501 /* 3502 * Setup RSS for this port. Save a copy of the RSS table for later use. 3503 */ 3504 rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); 3505 for (i = 0; i < pi->rss_size;) { 3506 for_each_rxq(pi, j, rxq) { 3507 rss[i++] = rxq->iq.abs_id; 3508 if (i == pi->rss_size) 3509 break; 3510 } 3511 } 3512 3513 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss, 3514 pi->rss_size); 3515 if (rc != 0) { 3516 if_printf(ifp, "rss_config failed: %d\n", rc); 3517 goto done; 3518 } 3519 3520 pi->rss = rss; 3521 pi->flags |= PORT_INIT_DONE; 3522 done: 3523 if (rc != 0) 3524 port_full_uninit(pi); 3525 3526 return (rc); 3527 } 3528 3529 /* 3530 * Idempotent. 3531 */ 3532 int 3533 port_full_uninit(struct port_info *pi) 3534 { 3535 struct adapter *sc = pi->adapter; 3536 int i; 3537 struct sge_rxq *rxq; 3538 struct sge_txq *txq; 3539 #ifdef TCP_OFFLOAD 3540 struct sge_ofld_rxq *ofld_rxq; 3541 struct sge_wrq *ofld_txq; 3542 #endif 3543 3544 if (pi->flags & PORT_INIT_DONE) { 3545 3546 /* Need to quiesce queues. XXX: ctrl queues? */ 3547 3548 for_each_txq(pi, i, txq) { 3549 quiesce_eq(sc, &txq->eq); 3550 } 3551 3552 #ifdef TCP_OFFLOAD 3553 for_each_ofld_txq(pi, i, ofld_txq) { 3554 quiesce_eq(sc, &ofld_txq->eq); 3555 } 3556 #endif 3557 3558 for_each_rxq(pi, i, rxq) { 3559 quiesce_iq(sc, &rxq->iq); 3560 quiesce_fl(sc, &rxq->fl); 3561 } 3562 3563 #ifdef TCP_OFFLOAD 3564 for_each_ofld_rxq(pi, i, ofld_rxq) { 3565 quiesce_iq(sc, &ofld_rxq->iq); 3566 quiesce_fl(sc, &ofld_rxq->fl); 3567 } 3568 #endif 3569 free(pi->rss, M_CXGBE); 3570 } 3571 3572 t4_teardown_port_queues(pi); 3573 pi->flags &= ~PORT_INIT_DONE; 3574 3575 return (0); 3576 } 3577 3578 static void 3579 quiesce_eq(struct adapter *sc, struct sge_eq *eq) 3580 { 3581 EQ_LOCK(eq); 3582 eq->flags |= EQ_DOOMED; 3583 3584 /* 3585 * Wait for the response to a credit flush if one's 3586 * pending. 3587 */ 3588 while (eq->flags & EQ_CRFLUSHED) 3589 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0); 3590 EQ_UNLOCK(eq); 3591 3592 callout_drain(&eq->tx_callout); /* XXX: iffy */ 3593 pause("callout", 10); /* Still iffy */ 3594 3595 taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task); 3596 } 3597 3598 static void 3599 quiesce_iq(struct adapter *sc, struct sge_iq *iq) 3600 { 3601 (void) sc; /* unused */ 3602 3603 /* Synchronize with the interrupt handler */ 3604 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 3605 pause("iqfree", 1); 3606 } 3607 3608 static void 3609 quiesce_fl(struct adapter *sc, struct sge_fl *fl) 3610 { 3611 mtx_lock(&sc->sfl_lock); 3612 FL_LOCK(fl); 3613 fl->flags |= FL_DOOMED; 3614 FL_UNLOCK(fl); 3615 mtx_unlock(&sc->sfl_lock); 3616 3617 callout_drain(&sc->sfl_callout); 3618 KASSERT((fl->flags & FL_STARVING) == 0, 3619 ("%s: still starving", __func__)); 3620 } 3621 3622 static int 3623 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 3624 driver_intr_t *handler, void *arg, char *name) 3625 { 3626 int rc; 3627 3628 irq->rid = rid; 3629 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 3630 RF_SHAREABLE | RF_ACTIVE); 3631 if (irq->res == NULL) { 3632 device_printf(sc->dev, 3633 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 3634 return (ENOMEM); 3635 } 3636 3637 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 3638 NULL, handler, arg, &irq->tag); 3639 if (rc != 0) { 3640 device_printf(sc->dev, 3641 "failed to setup interrupt for rid %d, name %s: %d\n", 3642 rid, name, rc); 3643 } else if (name) 3644 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 3645 3646 return (rc); 3647 } 3648 3649 static int 3650 t4_free_irq(struct adapter *sc, struct irq *irq) 3651 { 3652 if (irq->tag) 3653 bus_teardown_intr(sc->dev, irq->res, irq->tag); 3654 if (irq->res) 3655 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 3656 3657 bzero(irq, sizeof(*irq)); 3658 3659 return (0); 3660 } 3661 3662 static void 3663 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 3664 unsigned int end) 3665 { 3666 uint32_t *p = (uint32_t *)(buf + start); 3667 3668 for ( ; start <= end; start += sizeof(uint32_t)) 3669 *p++ = t4_read_reg(sc, start); 3670 } 3671 3672 static void 3673 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 3674 { 3675 int i, n; 3676 const unsigned int *reg_ranges; 3677 static const unsigned int t4_reg_ranges[] = { 3678 0x1008, 0x1108, 3679 0x1180, 0x11b4, 3680 0x11fc, 0x123c, 3681 0x1300, 0x173c, 3682 0x1800, 0x18fc, 3683 0x3000, 0x30d8, 3684 0x30e0, 0x5924, 3685 0x5960, 0x59d4, 3686 0x5a00, 0x5af8, 3687 0x6000, 0x6098, 3688 0x6100, 0x6150, 3689 0x6200, 0x6208, 3690 0x6240, 0x6248, 3691 0x6280, 0x6338, 3692 0x6370, 0x638c, 3693 0x6400, 0x643c, 3694 0x6500, 0x6524, 3695 0x6a00, 0x6a38, 3696 0x6a60, 0x6a78, 3697 0x6b00, 0x6b84, 3698 0x6bf0, 0x6c84, 3699 0x6cf0, 0x6d84, 3700 0x6df0, 0x6e84, 3701 0x6ef0, 0x6f84, 3702 0x6ff0, 0x7084, 3703 0x70f0, 0x7184, 3704 0x71f0, 0x7284, 3705 0x72f0, 0x7384, 3706 0x73f0, 0x7450, 3707 0x7500, 0x7530, 3708 0x7600, 0x761c, 3709 0x7680, 0x76cc, 3710 0x7700, 0x7798, 3711 0x77c0, 0x77fc, 3712 0x7900, 0x79fc, 3713 0x7b00, 0x7c38, 3714 0x7d00, 0x7efc, 3715 0x8dc0, 0x8e1c, 3716 0x8e30, 0x8e78, 3717 0x8ea0, 0x8f6c, 3718 0x8fc0, 0x9074, 3719 0x90fc, 0x90fc, 3720 0x9400, 0x9458, 3721 0x9600, 0x96bc, 3722 0x9800, 0x9808, 3723 0x9820, 0x983c, 3724 0x9850, 0x9864, 3725 0x9c00, 0x9c6c, 3726 0x9c80, 0x9cec, 3727 0x9d00, 0x9d6c, 3728 0x9d80, 0x9dec, 3729 0x9e00, 0x9e6c, 3730 0x9e80, 0x9eec, 3731 0x9f00, 0x9f6c, 3732 0x9f80, 0x9fec, 3733 0xd004, 0xd03c, 3734 0xdfc0, 0xdfe0, 3735 0xe000, 0xea7c, 3736 0xf000, 0x11110, 3737 0x11118, 0x11190, 3738 0x19040, 0x1906c, 3739 0x19078, 0x19080, 3740 0x1908c, 0x19124, 3741 0x19150, 0x191b0, 3742 0x191d0, 0x191e8, 3743 0x19238, 0x1924c, 3744 0x193f8, 0x19474, 3745 0x19490, 0x194f8, 3746 0x19800, 0x19f30, 3747 0x1a000, 0x1a06c, 3748 0x1a0b0, 0x1a120, 3749 0x1a128, 0x1a138, 3750 0x1a190, 0x1a1c4, 3751 0x1a1fc, 0x1a1fc, 3752 0x1e040, 0x1e04c, 3753 0x1e284, 0x1e28c, 3754 0x1e2c0, 0x1e2c0, 3755 0x1e2e0, 0x1e2e0, 3756 0x1e300, 0x1e384, 3757 0x1e3c0, 0x1e3c8, 3758 0x1e440, 0x1e44c, 3759 0x1e684, 0x1e68c, 3760 0x1e6c0, 0x1e6c0, 3761 0x1e6e0, 0x1e6e0, 3762 0x1e700, 0x1e784, 3763 0x1e7c0, 0x1e7c8, 3764 0x1e840, 0x1e84c, 3765 0x1ea84, 0x1ea8c, 3766 0x1eac0, 0x1eac0, 3767 0x1eae0, 0x1eae0, 3768 0x1eb00, 0x1eb84, 3769 0x1ebc0, 0x1ebc8, 3770 0x1ec40, 0x1ec4c, 3771 0x1ee84, 0x1ee8c, 3772 0x1eec0, 0x1eec0, 3773 0x1eee0, 0x1eee0, 3774 0x1ef00, 0x1ef84, 3775 0x1efc0, 0x1efc8, 3776 0x1f040, 0x1f04c, 3777 0x1f284, 0x1f28c, 3778 0x1f2c0, 0x1f2c0, 3779 0x1f2e0, 0x1f2e0, 3780 0x1f300, 0x1f384, 3781 0x1f3c0, 0x1f3c8, 3782 0x1f440, 0x1f44c, 3783 0x1f684, 0x1f68c, 3784 0x1f6c0, 0x1f6c0, 3785 0x1f6e0, 0x1f6e0, 3786 0x1f700, 0x1f784, 3787 0x1f7c0, 0x1f7c8, 3788 0x1f840, 0x1f84c, 3789 0x1fa84, 0x1fa8c, 3790 0x1fac0, 0x1fac0, 3791 0x1fae0, 0x1fae0, 3792 0x1fb00, 0x1fb84, 3793 0x1fbc0, 0x1fbc8, 3794 0x1fc40, 0x1fc4c, 3795 0x1fe84, 0x1fe8c, 3796 0x1fec0, 0x1fec0, 3797 0x1fee0, 0x1fee0, 3798 0x1ff00, 0x1ff84, 3799 0x1ffc0, 0x1ffc8, 3800 0x20000, 0x2002c, 3801 0x20100, 0x2013c, 3802 0x20190, 0x201c8, 3803 0x20200, 0x20318, 3804 0x20400, 0x20528, 3805 0x20540, 0x20614, 3806 0x21000, 0x21040, 3807 0x2104c, 0x21060, 3808 0x210c0, 0x210ec, 3809 0x21200, 0x21268, 3810 0x21270, 0x21284, 3811 0x212fc, 0x21388, 3812 0x21400, 0x21404, 3813 0x21500, 0x21518, 3814 0x2152c, 0x2153c, 3815 0x21550, 0x21554, 3816 0x21600, 0x21600, 3817 0x21608, 0x21628, 3818 0x21630, 0x2163c, 3819 0x21700, 0x2171c, 3820 0x21780, 0x2178c, 3821 0x21800, 0x21c38, 3822 0x21c80, 0x21d7c, 3823 0x21e00, 0x21e04, 3824 0x22000, 0x2202c, 3825 0x22100, 0x2213c, 3826 0x22190, 0x221c8, 3827 0x22200, 0x22318, 3828 0x22400, 0x22528, 3829 0x22540, 0x22614, 3830 0x23000, 0x23040, 3831 0x2304c, 0x23060, 3832 0x230c0, 0x230ec, 3833 0x23200, 0x23268, 3834 0x23270, 0x23284, 3835 0x232fc, 0x23388, 3836 0x23400, 0x23404, 3837 0x23500, 0x23518, 3838 0x2352c, 0x2353c, 3839 0x23550, 0x23554, 3840 0x23600, 0x23600, 3841 0x23608, 0x23628, 3842 0x23630, 0x2363c, 3843 0x23700, 0x2371c, 3844 0x23780, 0x2378c, 3845 0x23800, 0x23c38, 3846 0x23c80, 0x23d7c, 3847 0x23e00, 0x23e04, 3848 0x24000, 0x2402c, 3849 0x24100, 0x2413c, 3850 0x24190, 0x241c8, 3851 0x24200, 0x24318, 3852 0x24400, 0x24528, 3853 0x24540, 0x24614, 3854 0x25000, 0x25040, 3855 0x2504c, 0x25060, 3856 0x250c0, 0x250ec, 3857 0x25200, 0x25268, 3858 0x25270, 0x25284, 3859 0x252fc, 0x25388, 3860 0x25400, 0x25404, 3861 0x25500, 0x25518, 3862 0x2552c, 0x2553c, 3863 0x25550, 0x25554, 3864 0x25600, 0x25600, 3865 0x25608, 0x25628, 3866 0x25630, 0x2563c, 3867 0x25700, 0x2571c, 3868 0x25780, 0x2578c, 3869 0x25800, 0x25c38, 3870 0x25c80, 0x25d7c, 3871 0x25e00, 0x25e04, 3872 0x26000, 0x2602c, 3873 0x26100, 0x2613c, 3874 0x26190, 0x261c8, 3875 0x26200, 0x26318, 3876 0x26400, 0x26528, 3877 0x26540, 0x26614, 3878 0x27000, 0x27040, 3879 0x2704c, 0x27060, 3880 0x270c0, 0x270ec, 3881 0x27200, 0x27268, 3882 0x27270, 0x27284, 3883 0x272fc, 0x27388, 3884 0x27400, 0x27404, 3885 0x27500, 0x27518, 3886 0x2752c, 0x2753c, 3887 0x27550, 0x27554, 3888 0x27600, 0x27600, 3889 0x27608, 0x27628, 3890 0x27630, 0x2763c, 3891 0x27700, 0x2771c, 3892 0x27780, 0x2778c, 3893 0x27800, 0x27c38, 3894 0x27c80, 0x27d7c, 3895 0x27e00, 0x27e04 3896 }; 3897 static const unsigned int t5_reg_ranges[] = { 3898 0x1008, 0x1148, 3899 0x1180, 0x11b4, 3900 0x11fc, 0x123c, 3901 0x1280, 0x173c, 3902 0x1800, 0x18fc, 3903 0x3000, 0x3028, 3904 0x3060, 0x30d8, 3905 0x30e0, 0x30fc, 3906 0x3140, 0x357c, 3907 0x35a8, 0x35cc, 3908 0x35ec, 0x35ec, 3909 0x3600, 0x5624, 3910 0x56cc, 0x575c, 3911 0x580c, 0x5814, 3912 0x5890, 0x58bc, 3913 0x5940, 0x59dc, 3914 0x59fc, 0x5a18, 3915 0x5a60, 0x5a9c, 3916 0x5b94, 0x5bfc, 3917 0x6000, 0x6040, 3918 0x6058, 0x614c, 3919 0x7700, 0x7798, 3920 0x77c0, 0x78fc, 3921 0x7b00, 0x7c54, 3922 0x7d00, 0x7efc, 3923 0x8dc0, 0x8de0, 3924 0x8df8, 0x8e84, 3925 0x8ea0, 0x8f84, 3926 0x8fc0, 0x90f8, 3927 0x9400, 0x9470, 3928 0x9600, 0x96f4, 3929 0x9800, 0x9808, 3930 0x9820, 0x983c, 3931 0x9850, 0x9864, 3932 0x9c00, 0x9c6c, 3933 0x9c80, 0x9cec, 3934 0x9d00, 0x9d6c, 3935 0x9d80, 0x9dec, 3936 0x9e00, 0x9e6c, 3937 0x9e80, 0x9eec, 3938 0x9f00, 0x9f6c, 3939 0x9f80, 0xa020, 3940 0xd004, 0xd03c, 3941 0xdfc0, 0xdfe0, 3942 0xe000, 0x11088, 3943 0x1109c, 0x11110, 3944 0x11118, 0x1117c, 3945 0x11190, 0x11204, 3946 0x19040, 0x1906c, 3947 0x19078, 0x19080, 3948 0x1908c, 0x19124, 3949 0x19150, 0x191b0, 3950 0x191d0, 0x191e8, 3951 0x19238, 0x19290, 3952 0x193f8, 0x19474, 3953 0x19490, 0x194cc, 3954 0x194f0, 0x194f8, 3955 0x19c00, 0x19c60, 3956 0x19c94, 0x19e10, 3957 0x19e50, 0x19f34, 3958 0x19f40, 0x19f50, 3959 0x19f90, 0x19fe4, 3960 0x1a000, 0x1a06c, 3961 0x1a0b0, 0x1a120, 3962 0x1a128, 0x1a138, 3963 0x1a190, 0x1a1c4, 3964 0x1a1fc, 0x1a1fc, 3965 0x1e008, 0x1e00c, 3966 0x1e040, 0x1e04c, 3967 0x1e284, 0x1e290, 3968 0x1e2c0, 0x1e2c0, 3969 0x1e2e0, 0x1e2e0, 3970 0x1e300, 0x1e384, 3971 0x1e3c0, 0x1e3c8, 3972 0x1e408, 0x1e40c, 3973 0x1e440, 0x1e44c, 3974 0x1e684, 0x1e690, 3975 0x1e6c0, 0x1e6c0, 3976 0x1e6e0, 0x1e6e0, 3977 0x1e700, 0x1e784, 3978 0x1e7c0, 0x1e7c8, 3979 0x1e808, 0x1e80c, 3980 0x1e840, 0x1e84c, 3981 0x1ea84, 0x1ea90, 3982 0x1eac0, 0x1eac0, 3983 0x1eae0, 0x1eae0, 3984 0x1eb00, 0x1eb84, 3985 0x1ebc0, 0x1ebc8, 3986 0x1ec08, 0x1ec0c, 3987 0x1ec40, 0x1ec4c, 3988 0x1ee84, 0x1ee90, 3989 0x1eec0, 0x1eec0, 3990 0x1eee0, 0x1eee0, 3991 0x1ef00, 0x1ef84, 3992 0x1efc0, 0x1efc8, 3993 0x1f008, 0x1f00c, 3994 0x1f040, 0x1f04c, 3995 0x1f284, 0x1f290, 3996 0x1f2c0, 0x1f2c0, 3997 0x1f2e0, 0x1f2e0, 3998 0x1f300, 0x1f384, 3999 0x1f3c0, 0x1f3c8, 4000 0x1f408, 0x1f40c, 4001 0x1f440, 0x1f44c, 4002 0x1f684, 0x1f690, 4003 0x1f6c0, 0x1f6c0, 4004 0x1f6e0, 0x1f6e0, 4005 0x1f700, 0x1f784, 4006 0x1f7c0, 0x1f7c8, 4007 0x1f808, 0x1f80c, 4008 0x1f840, 0x1f84c, 4009 0x1fa84, 0x1fa90, 4010 0x1fac0, 0x1fac0, 4011 0x1fae0, 0x1fae0, 4012 0x1fb00, 0x1fb84, 4013 0x1fbc0, 0x1fbc8, 4014 0x1fc08, 0x1fc0c, 4015 0x1fc40, 0x1fc4c, 4016 0x1fe84, 0x1fe90, 4017 0x1fec0, 0x1fec0, 4018 0x1fee0, 0x1fee0, 4019 0x1ff00, 0x1ff84, 4020 0x1ffc0, 0x1ffc8, 4021 0x30000, 0x30030, 4022 0x30100, 0x30144, 4023 0x30190, 0x301d0, 4024 0x30200, 0x30318, 4025 0x30400, 0x3052c, 4026 0x30540, 0x3061c, 4027 0x30800, 0x30834, 4028 0x308c0, 0x30908, 4029 0x30910, 0x309ac, 4030 0x30a00, 0x30a2c, 4031 0x30a44, 0x30a50, 4032 0x30a74, 0x30c24, 4033 0x30d00, 0x30d00, 4034 0x30d08, 0x30d14, 4035 0x30d1c, 0x30d20, 4036 0x30d3c, 0x30d50, 4037 0x31200, 0x3120c, 4038 0x31220, 0x31220, 4039 0x31240, 0x31240, 4040 0x31600, 0x3160c, 4041 0x31a00, 0x31a1c, 4042 0x31e00, 0x31e20, 4043 0x31e38, 0x31e3c, 4044 0x31e80, 0x31e80, 4045 0x31e88, 0x31ea8, 4046 0x31eb0, 0x31eb4, 4047 0x31ec8, 0x31ed4, 4048 0x31fb8, 0x32004, 4049 0x32200, 0x32200, 4050 0x32208, 0x32240, 4051 0x32248, 0x32280, 4052 0x32288, 0x322c0, 4053 0x322c8, 0x322fc, 4054 0x32600, 0x32630, 4055 0x32a00, 0x32abc, 4056 0x32b00, 0x32b70, 4057 0x33000, 0x33048, 4058 0x33060, 0x3309c, 4059 0x330f0, 0x33148, 4060 0x33160, 0x3319c, 4061 0x331f0, 0x332e4, 4062 0x332f8, 0x333e4, 4063 0x333f8, 0x33448, 4064 0x33460, 0x3349c, 4065 0x334f0, 0x33548, 4066 0x33560, 0x3359c, 4067 0x335f0, 0x336e4, 4068 0x336f8, 0x337e4, 4069 0x337f8, 0x337fc, 4070 0x33814, 0x33814, 4071 0x3382c, 0x3382c, 4072 0x33880, 0x3388c, 4073 0x338e8, 0x338ec, 4074 0x33900, 0x33948, 4075 0x33960, 0x3399c, 4076 0x339f0, 0x33ae4, 4077 0x33af8, 0x33b10, 4078 0x33b28, 0x33b28, 4079 0x33b3c, 0x33b50, 4080 0x33bf0, 0x33c10, 4081 0x33c28, 0x33c28, 4082 0x33c3c, 0x33c50, 4083 0x33cf0, 0x33cfc, 4084 0x34000, 0x34030, 4085 0x34100, 0x34144, 4086 0x34190, 0x341d0, 4087 0x34200, 0x34318, 4088 0x34400, 0x3452c, 4089 0x34540, 0x3461c, 4090 0x34800, 0x34834, 4091 0x348c0, 0x34908, 4092 0x34910, 0x349ac, 4093 0x34a00, 0x34a2c, 4094 0x34a44, 0x34a50, 4095 0x34a74, 0x34c24, 4096 0x34d00, 0x34d00, 4097 0x34d08, 0x34d14, 4098 0x34d1c, 0x34d20, 4099 0x34d3c, 0x34d50, 4100 0x35200, 0x3520c, 4101 0x35220, 0x35220, 4102 0x35240, 0x35240, 4103 0x35600, 0x3560c, 4104 0x35a00, 0x35a1c, 4105 0x35e00, 0x35e20, 4106 0x35e38, 0x35e3c, 4107 0x35e80, 0x35e80, 4108 0x35e88, 0x35ea8, 4109 0x35eb0, 0x35eb4, 4110 0x35ec8, 0x35ed4, 4111 0x35fb8, 0x36004, 4112 0x36200, 0x36200, 4113 0x36208, 0x36240, 4114 0x36248, 0x36280, 4115 0x36288, 0x362c0, 4116 0x362c8, 0x362fc, 4117 0x36600, 0x36630, 4118 0x36a00, 0x36abc, 4119 0x36b00, 0x36b70, 4120 0x37000, 0x37048, 4121 0x37060, 0x3709c, 4122 0x370f0, 0x37148, 4123 0x37160, 0x3719c, 4124 0x371f0, 0x372e4, 4125 0x372f8, 0x373e4, 4126 0x373f8, 0x37448, 4127 0x37460, 0x3749c, 4128 0x374f0, 0x37548, 4129 0x37560, 0x3759c, 4130 0x375f0, 0x376e4, 4131 0x376f8, 0x377e4, 4132 0x377f8, 0x377fc, 4133 0x37814, 0x37814, 4134 0x3782c, 0x3782c, 4135 0x37880, 0x3788c, 4136 0x378e8, 0x378ec, 4137 0x37900, 0x37948, 4138 0x37960, 0x3799c, 4139 0x379f0, 0x37ae4, 4140 0x37af8, 0x37b10, 4141 0x37b28, 0x37b28, 4142 0x37b3c, 0x37b50, 4143 0x37bf0, 0x37c10, 4144 0x37c28, 0x37c28, 4145 0x37c3c, 0x37c50, 4146 0x37cf0, 0x37cfc, 4147 0x38000, 0x38030, 4148 0x38100, 0x38144, 4149 0x38190, 0x381d0, 4150 0x38200, 0x38318, 4151 0x38400, 0x3852c, 4152 0x38540, 0x3861c, 4153 0x38800, 0x38834, 4154 0x388c0, 0x38908, 4155 0x38910, 0x389ac, 4156 0x38a00, 0x38a2c, 4157 0x38a44, 0x38a50, 4158 0x38a74, 0x38c24, 4159 0x38d00, 0x38d00, 4160 0x38d08, 0x38d14, 4161 0x38d1c, 0x38d20, 4162 0x38d3c, 0x38d50, 4163 0x39200, 0x3920c, 4164 0x39220, 0x39220, 4165 0x39240, 0x39240, 4166 0x39600, 0x3960c, 4167 0x39a00, 0x39a1c, 4168 0x39e00, 0x39e20, 4169 0x39e38, 0x39e3c, 4170 0x39e80, 0x39e80, 4171 0x39e88, 0x39ea8, 4172 0x39eb0, 0x39eb4, 4173 0x39ec8, 0x39ed4, 4174 0x39fb8, 0x3a004, 4175 0x3a200, 0x3a200, 4176 0x3a208, 0x3a240, 4177 0x3a248, 0x3a280, 4178 0x3a288, 0x3a2c0, 4179 0x3a2c8, 0x3a2fc, 4180 0x3a600, 0x3a630, 4181 0x3aa00, 0x3aabc, 4182 0x3ab00, 0x3ab70, 4183 0x3b000, 0x3b048, 4184 0x3b060, 0x3b09c, 4185 0x3b0f0, 0x3b148, 4186 0x3b160, 0x3b19c, 4187 0x3b1f0, 0x3b2e4, 4188 0x3b2f8, 0x3b3e4, 4189 0x3b3f8, 0x3b448, 4190 0x3b460, 0x3b49c, 4191 0x3b4f0, 0x3b548, 4192 0x3b560, 0x3b59c, 4193 0x3b5f0, 0x3b6e4, 4194 0x3b6f8, 0x3b7e4, 4195 0x3b7f8, 0x3b7fc, 4196 0x3b814, 0x3b814, 4197 0x3b82c, 0x3b82c, 4198 0x3b880, 0x3b88c, 4199 0x3b8e8, 0x3b8ec, 4200 0x3b900, 0x3b948, 4201 0x3b960, 0x3b99c, 4202 0x3b9f0, 0x3bae4, 4203 0x3baf8, 0x3bb10, 4204 0x3bb28, 0x3bb28, 4205 0x3bb3c, 0x3bb50, 4206 0x3bbf0, 0x3bc10, 4207 0x3bc28, 0x3bc28, 4208 0x3bc3c, 0x3bc50, 4209 0x3bcf0, 0x3bcfc, 4210 0x3c000, 0x3c030, 4211 0x3c100, 0x3c144, 4212 0x3c190, 0x3c1d0, 4213 0x3c200, 0x3c318, 4214 0x3c400, 0x3c52c, 4215 0x3c540, 0x3c61c, 4216 0x3c800, 0x3c834, 4217 0x3c8c0, 0x3c908, 4218 0x3c910, 0x3c9ac, 4219 0x3ca00, 0x3ca2c, 4220 0x3ca44, 0x3ca50, 4221 0x3ca74, 0x3cc24, 4222 0x3cd00, 0x3cd00, 4223 0x3cd08, 0x3cd14, 4224 0x3cd1c, 0x3cd20, 4225 0x3cd3c, 0x3cd50, 4226 0x3d200, 0x3d20c, 4227 0x3d220, 0x3d220, 4228 0x3d240, 0x3d240, 4229 0x3d600, 0x3d60c, 4230 0x3da00, 0x3da1c, 4231 0x3de00, 0x3de20, 4232 0x3de38, 0x3de3c, 4233 0x3de80, 0x3de80, 4234 0x3de88, 0x3dea8, 4235 0x3deb0, 0x3deb4, 4236 0x3dec8, 0x3ded4, 4237 0x3dfb8, 0x3e004, 4238 0x3e200, 0x3e200, 4239 0x3e208, 0x3e240, 4240 0x3e248, 0x3e280, 4241 0x3e288, 0x3e2c0, 4242 0x3e2c8, 0x3e2fc, 4243 0x3e600, 0x3e630, 4244 0x3ea00, 0x3eabc, 4245 0x3eb00, 0x3eb70, 4246 0x3f000, 0x3f048, 4247 0x3f060, 0x3f09c, 4248 0x3f0f0, 0x3f148, 4249 0x3f160, 0x3f19c, 4250 0x3f1f0, 0x3f2e4, 4251 0x3f2f8, 0x3f3e4, 4252 0x3f3f8, 0x3f448, 4253 0x3f460, 0x3f49c, 4254 0x3f4f0, 0x3f548, 4255 0x3f560, 0x3f59c, 4256 0x3f5f0, 0x3f6e4, 4257 0x3f6f8, 0x3f7e4, 4258 0x3f7f8, 0x3f7fc, 4259 0x3f814, 0x3f814, 4260 0x3f82c, 0x3f82c, 4261 0x3f880, 0x3f88c, 4262 0x3f8e8, 0x3f8ec, 4263 0x3f900, 0x3f948, 4264 0x3f960, 0x3f99c, 4265 0x3f9f0, 0x3fae4, 4266 0x3faf8, 0x3fb10, 4267 0x3fb28, 0x3fb28, 4268 0x3fb3c, 0x3fb50, 4269 0x3fbf0, 0x3fc10, 4270 0x3fc28, 0x3fc28, 4271 0x3fc3c, 0x3fc50, 4272 0x3fcf0, 0x3fcfc, 4273 0x40000, 0x4000c, 4274 0x40040, 0x40068, 4275 0x4007c, 0x40144, 4276 0x40180, 0x4018c, 4277 0x40200, 0x40298, 4278 0x402ac, 0x4033c, 4279 0x403f8, 0x403fc, 4280 0x41304, 0x413c4, 4281 0x41400, 0x4141c, 4282 0x41480, 0x414d0, 4283 0x44000, 0x44078, 4284 0x440c0, 0x44278, 4285 0x442c0, 0x44478, 4286 0x444c0, 0x44678, 4287 0x446c0, 0x44878, 4288 0x448c0, 0x449fc, 4289 0x45000, 0x45068, 4290 0x45080, 0x45084, 4291 0x450a0, 0x450b0, 4292 0x45200, 0x45268, 4293 0x45280, 0x45284, 4294 0x452a0, 0x452b0, 4295 0x460c0, 0x460e4, 4296 0x47000, 0x4708c, 4297 0x47200, 0x47250, 4298 0x47400, 0x47420, 4299 0x47600, 0x47618, 4300 0x47800, 0x47814, 4301 0x48000, 0x4800c, 4302 0x48040, 0x48068, 4303 0x4807c, 0x48144, 4304 0x48180, 0x4818c, 4305 0x48200, 0x48298, 4306 0x482ac, 0x4833c, 4307 0x483f8, 0x483fc, 4308 0x49304, 0x493c4, 4309 0x49400, 0x4941c, 4310 0x49480, 0x494d0, 4311 0x4c000, 0x4c078, 4312 0x4c0c0, 0x4c278, 4313 0x4c2c0, 0x4c478, 4314 0x4c4c0, 0x4c678, 4315 0x4c6c0, 0x4c878, 4316 0x4c8c0, 0x4c9fc, 4317 0x4d000, 0x4d068, 4318 0x4d080, 0x4d084, 4319 0x4d0a0, 0x4d0b0, 4320 0x4d200, 0x4d268, 4321 0x4d280, 0x4d284, 4322 0x4d2a0, 0x4d2b0, 4323 0x4e0c0, 0x4e0e4, 4324 0x4f000, 0x4f08c, 4325 0x4f200, 0x4f250, 4326 0x4f400, 0x4f420, 4327 0x4f600, 0x4f618, 4328 0x4f800, 0x4f814, 4329 0x50000, 0x500cc, 4330 0x50400, 0x50400, 4331 0x50800, 0x508cc, 4332 0x50c00, 0x50c00, 4333 0x51000, 0x5101c, 4334 0x51300, 0x51308, 4335 }; 4336 4337 if (is_t4(sc)) { 4338 reg_ranges = &t4_reg_ranges[0]; 4339 n = nitems(t4_reg_ranges); 4340 } else { 4341 reg_ranges = &t5_reg_ranges[0]; 4342 n = nitems(t5_reg_ranges); 4343 } 4344 4345 regs->version = chip_id(sc) | chip_rev(sc) << 10; 4346 for (i = 0; i < n; i += 2) 4347 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 4348 } 4349 4350 static void 4351 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi) 4352 { 4353 int i; 4354 u_int v, tnl_cong_drops; 4355 struct timeval tv; 4356 const struct timeval interval = {0, 250000}; /* 250ms */ 4357 4358 getmicrotime(&tv); 4359 timevalsub(&tv, &interval); 4360 if (timevalcmp(&tv, &pi->last_refreshed, <)) 4361 return; 4362 4363 tnl_cong_drops = 0; 4364 t4_get_port_stats(sc, pi->tx_chan, &pi->stats); 4365 for (i = 0; i < NCHAN; i++) { 4366 if (pi->rx_chan_map & (1 << i)) { 4367 mtx_lock(&sc->regwin_lock); 4368 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 4369 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 4370 mtx_unlock(&sc->regwin_lock); 4371 tnl_cong_drops += v; 4372 } 4373 } 4374 pi->tnl_cong_drops = tnl_cong_drops; 4375 getmicrotime(&pi->last_refreshed); 4376 } 4377 4378 static void 4379 cxgbe_tick(void *arg) 4380 { 4381 struct port_info *pi = arg; 4382 struct adapter *sc = pi->adapter; 4383 struct ifnet *ifp = pi->ifp; 4384 4385 PORT_LOCK(pi); 4386 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 4387 PORT_UNLOCK(pi); 4388 return; /* without scheduling another callout */ 4389 } 4390 4391 cxgbe_refresh_stats(sc, pi); 4392 4393 callout_schedule(&pi->tick, hz); 4394 PORT_UNLOCK(pi); 4395 } 4396 4397 static void 4398 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 4399 { 4400 struct ifnet *vlan; 4401 4402 if (arg != ifp || ifp->if_type != IFT_ETHER) 4403 return; 4404 4405 vlan = VLAN_DEVAT(ifp, vid); 4406 VLAN_SETCOOKIE(vlan, ifp); 4407 } 4408 4409 static int 4410 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 4411 { 4412 4413 #ifdef INVARIANTS 4414 panic("%s: opcode 0x%02x on iq %p with payload %p", 4415 __func__, rss->opcode, iq, m); 4416 #else 4417 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 4418 __func__, rss->opcode, iq, m); 4419 m_freem(m); 4420 #endif 4421 return (EDOOFUS); 4422 } 4423 4424 int 4425 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 4426 { 4427 uintptr_t *loc, new; 4428 4429 if (opcode >= nitems(sc->cpl_handler)) 4430 return (EINVAL); 4431 4432 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 4433 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 4434 atomic_store_rel_ptr(loc, new); 4435 4436 return (0); 4437 } 4438 4439 static int 4440 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 4441 { 4442 4443 #ifdef INVARIANTS 4444 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 4445 #else 4446 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 4447 __func__, iq, ctrl); 4448 #endif 4449 return (EDOOFUS); 4450 } 4451 4452 int 4453 t4_register_an_handler(struct adapter *sc, an_handler_t h) 4454 { 4455 uintptr_t *loc, new; 4456 4457 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 4458 loc = (uintptr_t *) &sc->an_handler; 4459 atomic_store_rel_ptr(loc, new); 4460 4461 return (0); 4462 } 4463 4464 static int 4465 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 4466 { 4467 const struct cpl_fw6_msg *cpl = 4468 __containerof(rpl, struct cpl_fw6_msg, data[0]); 4469 4470 #ifdef INVARIANTS 4471 panic("%s: fw_msg type %d", __func__, cpl->type); 4472 #else 4473 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 4474 #endif 4475 return (EDOOFUS); 4476 } 4477 4478 int 4479 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 4480 { 4481 uintptr_t *loc, new; 4482 4483 if (type >= nitems(sc->fw_msg_handler)) 4484 return (EINVAL); 4485 4486 /* 4487 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 4488 * handler dispatch table. Reject any attempt to install a handler for 4489 * this subtype. 4490 */ 4491 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 4492 return (EINVAL); 4493 4494 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 4495 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 4496 atomic_store_rel_ptr(loc, new); 4497 4498 return (0); 4499 } 4500 4501 static int 4502 t4_sysctls(struct adapter *sc) 4503 { 4504 struct sysctl_ctx_list *ctx; 4505 struct sysctl_oid *oid; 4506 struct sysctl_oid_list *children, *c0; 4507 static char *caps[] = { 4508 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 4509 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL" /* caps[1] niccaps */ 4510 "\6HASHFILTER\7ETHOFLD", 4511 "\20\1TOE", /* caps[2] toecaps */ 4512 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 4513 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 4514 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 4515 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 4516 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 4517 "\4PO_INITIAOR\5PO_TARGET" 4518 }; 4519 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 4520 4521 ctx = device_get_sysctl_ctx(sc->dev); 4522 4523 /* 4524 * dev.t4nex.X. 4525 */ 4526 oid = device_get_sysctl_tree(sc->dev); 4527 c0 = children = SYSCTL_CHILDREN(oid); 4528 4529 sc->sc_do_rxcopy = 1; 4530 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, 4531 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); 4532 4533 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 4534 sc->params.nports, "# of ports"); 4535 4536 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 4537 NULL, chip_rev(sc), "chip hardware revision"); 4538 4539 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 4540 CTLFLAG_RD, sc->fw_version, 0, "firmware version"); 4541 4542 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 4543 CTLFLAG_RD, sc->cfg_file, 0, "configuration file"); 4544 4545 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 4546 sc->cfcsum, "config file checksum"); 4547 4548 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 4549 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 4550 sysctl_bitfield, "A", "available doorbells"); 4551 4552 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 4553 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 4554 sysctl_bitfield, "A", "available link capabilities"); 4555 4556 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 4557 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 4558 sysctl_bitfield, "A", "available NIC capabilities"); 4559 4560 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 4561 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 4562 sysctl_bitfield, "A", "available TCP offload capabilities"); 4563 4564 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 4565 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 4566 sysctl_bitfield, "A", "available RDMA capabilities"); 4567 4568 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 4569 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 4570 sysctl_bitfield, "A", "available iSCSI capabilities"); 4571 4572 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 4573 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 4574 sysctl_bitfield, "A", "available FCoE capabilities"); 4575 4576 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 4577 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 4578 4579 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 4580 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 4581 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 4582 "interrupt holdoff timer values (us)"); 4583 4584 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 4585 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 4586 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 4587 "interrupt holdoff packet counter values"); 4588 4589 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, 4590 NULL, sc->tids.nftids, "number of filters"); 4591 4592 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | 4593 CTLFLAG_RD, sc, 0, sysctl_temperature, "I", 4594 "chip temperature (in Celsius)"); 4595 4596 t4_sge_sysctls(sc, ctx, children); 4597 4598 sc->lro_timeout = 100; 4599 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, 4600 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); 4601 4602 #ifdef SBUF_DRAIN 4603 /* 4604 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 4605 */ 4606 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 4607 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 4608 "logs and miscellaneous information"); 4609 children = SYSCTL_CHILDREN(oid); 4610 4611 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 4612 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4613 sysctl_cctrl, "A", "congestion control"); 4614 4615 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 4616 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4617 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 4618 4619 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 4620 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 4621 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 4622 4623 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 4624 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 4625 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 4626 4627 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 4628 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 4629 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 4630 4631 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 4632 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 4633 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 4634 4635 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 4636 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 4637 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 4638 4639 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 4640 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4641 sysctl_cim_la, "A", "CIM logic analyzer"); 4642 4643 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", 4644 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4645 sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); 4646 4647 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 4648 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 4649 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 4650 4651 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 4652 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 4653 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 4654 4655 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 4656 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 4657 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 4658 4659 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 4660 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 4661 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 4662 4663 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 4664 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 4665 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 4666 4667 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 4668 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 4669 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 4670 4671 if (is_t5(sc)) { 4672 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 4673 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 4674 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 4675 4676 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 4677 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 4678 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 4679 } 4680 4681 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", 4682 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4683 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); 4684 4685 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 4686 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4687 sysctl_cim_qcfg, "A", "CIM queue configuration"); 4688 4689 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 4690 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4691 sysctl_cpl_stats, "A", "CPL statistics"); 4692 4693 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 4694 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4695 sysctl_ddp_stats, "A", "DDP statistics"); 4696 4697 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 4698 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4699 sysctl_devlog, "A", "firmware's device log"); 4700 4701 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 4702 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4703 sysctl_fcoe_stats, "A", "FCoE statistics"); 4704 4705 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 4706 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4707 sysctl_hw_sched, "A", "hardware scheduler "); 4708 4709 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 4710 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4711 sysctl_l2t, "A", "hardware L2 table"); 4712 4713 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 4714 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4715 sysctl_lb_stats, "A", "loopback statistics"); 4716 4717 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 4718 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4719 sysctl_meminfo, "A", "memory regions"); 4720 4721 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", 4722 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4723 sysctl_mps_tcam, "A", "MPS TCAM entries"); 4724 4725 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 4726 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4727 sysctl_path_mtus, "A", "path MTUs"); 4728 4729 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 4730 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4731 sysctl_pm_stats, "A", "PM statistics"); 4732 4733 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 4734 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4735 sysctl_rdma_stats, "A", "RDMA statistics"); 4736 4737 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 4738 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4739 sysctl_tcp_stats, "A", "TCP statistics"); 4740 4741 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 4742 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4743 sysctl_tids, "A", "TID information"); 4744 4745 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 4746 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4747 sysctl_tp_err_stats, "A", "TP error statistics"); 4748 4749 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", 4750 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4751 sysctl_tp_la, "A", "TP logic analyzer"); 4752 4753 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 4754 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4755 sysctl_tx_rate, "A", "Tx rate"); 4756 4757 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", 4758 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4759 sysctl_ulprx_la, "A", "ULPRX logic analyzer"); 4760 4761 if (is_t5(sc)) { 4762 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 4763 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4764 sysctl_wcwr_stats, "A", "write combined work requests"); 4765 } 4766 #endif 4767 4768 #ifdef TCP_OFFLOAD 4769 if (is_offload(sc)) { 4770 /* 4771 * dev.t4nex.X.toe. 4772 */ 4773 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 4774 NULL, "TOE parameters"); 4775 children = SYSCTL_CHILDREN(oid); 4776 4777 sc->tt.sndbuf = 256 * 1024; 4778 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 4779 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 4780 4781 sc->tt.ddp = 0; 4782 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 4783 &sc->tt.ddp, 0, "DDP allowed"); 4784 4785 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 4786 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 4787 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 4788 4789 sc->tt.ddp_thres = 4790 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 4791 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 4792 &sc->tt.ddp_thres, 0, "DDP threshold"); 4793 4794 sc->tt.rx_coalesce = 1; 4795 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", 4796 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); 4797 } 4798 #endif 4799 4800 4801 return (0); 4802 } 4803 4804 static int 4805 cxgbe_sysctls(struct port_info *pi) 4806 { 4807 struct sysctl_ctx_list *ctx; 4808 struct sysctl_oid *oid; 4809 struct sysctl_oid_list *children; 4810 struct adapter *sc = pi->adapter; 4811 4812 ctx = device_get_sysctl_ctx(pi->dev); 4813 4814 /* 4815 * dev.cxgbe.X. 4816 */ 4817 oid = device_get_sysctl_tree(pi->dev); 4818 children = SYSCTL_CHILDREN(oid); 4819 4820 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | 4821 CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); 4822 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { 4823 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 4824 CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I", 4825 "PHY temperature (in Celsius)"); 4826 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", 4827 CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I", 4828 "PHY firmware version"); 4829 } 4830 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 4831 &pi->nrxq, 0, "# of rx queues"); 4832 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 4833 &pi->ntxq, 0, "# of tx queues"); 4834 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 4835 &pi->first_rxq, 0, "index of first rx queue"); 4836 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 4837 &pi->first_txq, 0, "index of first tx queue"); 4838 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT | 4839 CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU", 4840 "Reserve queue 0 for non-flowid packets"); 4841 4842 #ifdef TCP_OFFLOAD 4843 if (is_offload(sc)) { 4844 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 4845 &pi->nofldrxq, 0, 4846 "# of rx queues for offloaded TCP connections"); 4847 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 4848 &pi->nofldtxq, 0, 4849 "# of tx queues for offloaded TCP connections"); 4850 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 4851 CTLFLAG_RD, &pi->first_ofld_rxq, 0, 4852 "index of first TOE rx queue"); 4853 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 4854 CTLFLAG_RD, &pi->first_ofld_txq, 0, 4855 "index of first TOE tx queue"); 4856 } 4857 #endif 4858 #ifdef DEV_NETMAP 4859 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD, 4860 &pi->nnmrxq, 0, "# of rx queues for netmap"); 4861 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD, 4862 &pi->nnmtxq, 0, "# of tx queues for netmap"); 4863 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq", 4864 CTLFLAG_RD, &pi->first_nm_rxq, 0, 4865 "index of first netmap rx queue"); 4866 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq", 4867 CTLFLAG_RD, &pi->first_nm_txq, 0, 4868 "index of first netmap tx queue"); 4869 #endif 4870 4871 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 4872 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 4873 "holdoff timer index"); 4874 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 4875 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 4876 "holdoff packet counter index"); 4877 4878 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 4879 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 4880 "rx queue size"); 4881 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 4882 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 4883 "tx queue size"); 4884 4885 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings", 4886 CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings, 4887 "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)"); 4888 4889 /* 4890 * dev.cxgbe.X.stats. 4891 */ 4892 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 4893 NULL, "port statistics"); 4894 children = SYSCTL_CHILDREN(oid); 4895 4896 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 4897 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 4898 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ 4899 sysctl_handle_t4_reg64, "QU", desc) 4900 4901 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 4902 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 4903 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 4904 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 4905 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 4906 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 4907 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 4908 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 4909 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 4910 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 4911 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 4912 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 4913 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 4914 "# of tx frames in this range", 4915 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 4916 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 4917 "# of tx frames in this range", 4918 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 4919 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 4920 "# of tx frames in this range", 4921 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 4922 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 4923 "# of tx frames in this range", 4924 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 4925 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 4926 "# of tx frames in this range", 4927 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 4928 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 4929 "# of tx frames in this range", 4930 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 4931 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 4932 "# of tx frames in this range", 4933 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 4934 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 4935 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 4936 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 4937 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 4938 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 4939 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 4940 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 4941 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 4942 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 4943 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 4944 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 4945 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 4946 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 4947 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 4948 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 4949 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 4950 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 4951 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 4952 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 4953 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 4954 4955 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 4956 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 4957 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 4958 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 4959 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 4960 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 4961 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 4962 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 4963 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 4964 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 4965 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 4966 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 4967 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 4968 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 4969 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 4970 "# of frames received with bad FCS", 4971 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 4972 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 4973 "# of frames received with length error", 4974 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 4975 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 4976 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 4977 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 4978 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 4979 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 4980 "# of rx frames in this range", 4981 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 4982 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 4983 "# of rx frames in this range", 4984 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 4985 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 4986 "# of rx frames in this range", 4987 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 4988 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 4989 "# of rx frames in this range", 4990 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 4991 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 4992 "# of rx frames in this range", 4993 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 4994 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 4995 "# of rx frames in this range", 4996 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 4997 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 4998 "# of rx frames in this range", 4999 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 5000 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 5001 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 5002 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 5003 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 5004 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 5005 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 5006 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 5007 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 5008 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 5009 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 5010 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 5011 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 5012 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 5013 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 5014 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 5015 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 5016 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 5017 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 5018 5019 #undef SYSCTL_ADD_T4_REG64 5020 5021 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 5022 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 5023 &pi->stats.name, desc) 5024 5025 /* We get these from port_stats and they may be stale by upto 1s */ 5026 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 5027 "# drops due to buffer-group 0 overflows"); 5028 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 5029 "# drops due to buffer-group 1 overflows"); 5030 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 5031 "# drops due to buffer-group 2 overflows"); 5032 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 5033 "# drops due to buffer-group 3 overflows"); 5034 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 5035 "# of buffer-group 0 truncated packets"); 5036 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 5037 "# of buffer-group 1 truncated packets"); 5038 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 5039 "# of buffer-group 2 truncated packets"); 5040 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 5041 "# of buffer-group 3 truncated packets"); 5042 5043 #undef SYSCTL_ADD_T4_PORTSTAT 5044 5045 return (0); 5046 } 5047 5048 static int 5049 sysctl_int_array(SYSCTL_HANDLER_ARGS) 5050 { 5051 int rc, *i; 5052 struct sbuf sb; 5053 5054 sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND); 5055 for (i = arg1; arg2; arg2 -= sizeof(int), i++) 5056 sbuf_printf(&sb, "%d ", *i); 5057 sbuf_trim(&sb); 5058 sbuf_finish(&sb); 5059 rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 5060 sbuf_delete(&sb); 5061 return (rc); 5062 } 5063 5064 static int 5065 sysctl_bitfield(SYSCTL_HANDLER_ARGS) 5066 { 5067 int rc; 5068 struct sbuf *sb; 5069 5070 rc = sysctl_wire_old_buffer(req, 0); 5071 if (rc != 0) 5072 return(rc); 5073 5074 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5075 if (sb == NULL) 5076 return (ENOMEM); 5077 5078 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 5079 rc = sbuf_finish(sb); 5080 sbuf_delete(sb); 5081 5082 return (rc); 5083 } 5084 5085 static int 5086 sysctl_btphy(SYSCTL_HANDLER_ARGS) 5087 { 5088 struct port_info *pi = arg1; 5089 int op = arg2; 5090 struct adapter *sc = pi->adapter; 5091 u_int v; 5092 int rc; 5093 5094 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt"); 5095 if (rc) 5096 return (rc); 5097 /* XXX: magic numbers */ 5098 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, 5099 &v); 5100 end_synchronized_op(sc, 0); 5101 if (rc) 5102 return (rc); 5103 if (op == 0) 5104 v /= 256; 5105 5106 rc = sysctl_handle_int(oidp, &v, 0, req); 5107 return (rc); 5108 } 5109 5110 static int 5111 sysctl_noflowq(SYSCTL_HANDLER_ARGS) 5112 { 5113 struct port_info *pi = arg1; 5114 int rc, val; 5115 5116 val = pi->rsrv_noflowq; 5117 rc = sysctl_handle_int(oidp, &val, 0, req); 5118 if (rc != 0 || req->newptr == NULL) 5119 return (rc); 5120 5121 if ((val >= 1) && (pi->ntxq > 1)) 5122 pi->rsrv_noflowq = 1; 5123 else 5124 pi->rsrv_noflowq = 0; 5125 5126 return (rc); 5127 } 5128 5129 static int 5130 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 5131 { 5132 struct port_info *pi = arg1; 5133 struct adapter *sc = pi->adapter; 5134 int idx, rc, i; 5135 struct sge_rxq *rxq; 5136 #ifdef TCP_OFFLOAD 5137 struct sge_ofld_rxq *ofld_rxq; 5138 #endif 5139 uint8_t v; 5140 5141 idx = pi->tmr_idx; 5142 5143 rc = sysctl_handle_int(oidp, &idx, 0, req); 5144 if (rc != 0 || req->newptr == NULL) 5145 return (rc); 5146 5147 if (idx < 0 || idx >= SGE_NTIMERS) 5148 return (EINVAL); 5149 5150 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5151 "t4tmr"); 5152 if (rc) 5153 return (rc); 5154 5155 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 5156 for_each_rxq(pi, i, rxq) { 5157 #ifdef atomic_store_rel_8 5158 atomic_store_rel_8(&rxq->iq.intr_params, v); 5159 #else 5160 rxq->iq.intr_params = v; 5161 #endif 5162 } 5163 #ifdef TCP_OFFLOAD 5164 for_each_ofld_rxq(pi, i, ofld_rxq) { 5165 #ifdef atomic_store_rel_8 5166 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); 5167 #else 5168 ofld_rxq->iq.intr_params = v; 5169 #endif 5170 } 5171 #endif 5172 pi->tmr_idx = idx; 5173 5174 end_synchronized_op(sc, LOCK_HELD); 5175 return (0); 5176 } 5177 5178 static int 5179 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 5180 { 5181 struct port_info *pi = arg1; 5182 struct adapter *sc = pi->adapter; 5183 int idx, rc; 5184 5185 idx = pi->pktc_idx; 5186 5187 rc = sysctl_handle_int(oidp, &idx, 0, req); 5188 if (rc != 0 || req->newptr == NULL) 5189 return (rc); 5190 5191 if (idx < -1 || idx >= SGE_NCOUNTERS) 5192 return (EINVAL); 5193 5194 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5195 "t4pktc"); 5196 if (rc) 5197 return (rc); 5198 5199 if (pi->flags & PORT_INIT_DONE) 5200 rc = EBUSY; /* cannot be changed once the queues are created */ 5201 else 5202 pi->pktc_idx = idx; 5203 5204 end_synchronized_op(sc, LOCK_HELD); 5205 return (rc); 5206 } 5207 5208 static int 5209 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 5210 { 5211 struct port_info *pi = arg1; 5212 struct adapter *sc = pi->adapter; 5213 int qsize, rc; 5214 5215 qsize = pi->qsize_rxq; 5216 5217 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5218 if (rc != 0 || req->newptr == NULL) 5219 return (rc); 5220 5221 if (qsize < 128 || (qsize & 7)) 5222 return (EINVAL); 5223 5224 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5225 "t4rxqs"); 5226 if (rc) 5227 return (rc); 5228 5229 if (pi->flags & PORT_INIT_DONE) 5230 rc = EBUSY; /* cannot be changed once the queues are created */ 5231 else 5232 pi->qsize_rxq = qsize; 5233 5234 end_synchronized_op(sc, LOCK_HELD); 5235 return (rc); 5236 } 5237 5238 static int 5239 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 5240 { 5241 struct port_info *pi = arg1; 5242 struct adapter *sc = pi->adapter; 5243 int qsize, rc; 5244 5245 qsize = pi->qsize_txq; 5246 5247 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5248 if (rc != 0 || req->newptr == NULL) 5249 return (rc); 5250 5251 /* bufring size must be powerof2 */ 5252 if (qsize < 128 || !powerof2(qsize)) 5253 return (EINVAL); 5254 5255 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5256 "t4txqs"); 5257 if (rc) 5258 return (rc); 5259 5260 if (pi->flags & PORT_INIT_DONE) 5261 rc = EBUSY; /* cannot be changed once the queues are created */ 5262 else 5263 pi->qsize_txq = qsize; 5264 5265 end_synchronized_op(sc, LOCK_HELD); 5266 return (rc); 5267 } 5268 5269 static int 5270 sysctl_pause_settings(SYSCTL_HANDLER_ARGS) 5271 { 5272 struct port_info *pi = arg1; 5273 struct adapter *sc = pi->adapter; 5274 struct link_config *lc = &pi->link_cfg; 5275 int rc; 5276 5277 if (req->newptr == NULL) { 5278 struct sbuf *sb; 5279 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX"; 5280 5281 rc = sysctl_wire_old_buffer(req, 0); 5282 if (rc != 0) 5283 return(rc); 5284 5285 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5286 if (sb == NULL) 5287 return (ENOMEM); 5288 5289 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits); 5290 rc = sbuf_finish(sb); 5291 sbuf_delete(sb); 5292 } else { 5293 char s[2]; 5294 int n; 5295 5296 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX)); 5297 s[1] = 0; 5298 5299 rc = sysctl_handle_string(oidp, s, sizeof(s), req); 5300 if (rc != 0) 5301 return(rc); 5302 5303 if (s[1] != 0) 5304 return (EINVAL); 5305 if (s[0] < '0' || s[0] > '9') 5306 return (EINVAL); /* not a number */ 5307 n = s[0] - '0'; 5308 if (n & ~(PAUSE_TX | PAUSE_RX)) 5309 return (EINVAL); /* some other bit is set too */ 5310 5311 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE"); 5312 if (rc) 5313 return (rc); 5314 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) { 5315 int link_ok = lc->link_ok; 5316 5317 lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX); 5318 lc->requested_fc |= n; 5319 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc); 5320 lc->link_ok = link_ok; /* restore */ 5321 } 5322 end_synchronized_op(sc, 0); 5323 } 5324 5325 return (rc); 5326 } 5327 5328 static int 5329 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 5330 { 5331 struct adapter *sc = arg1; 5332 int reg = arg2; 5333 uint64_t val; 5334 5335 val = t4_read_reg64(sc, reg); 5336 5337 return (sysctl_handle_64(oidp, &val, 0, req)); 5338 } 5339 5340 static int 5341 sysctl_temperature(SYSCTL_HANDLER_ARGS) 5342 { 5343 struct adapter *sc = arg1; 5344 int rc, t; 5345 uint32_t param, val; 5346 5347 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); 5348 if (rc) 5349 return (rc); 5350 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 5351 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 5352 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); 5353 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 5354 end_synchronized_op(sc, 0); 5355 if (rc) 5356 return (rc); 5357 5358 /* unknown is returned as 0 but we display -1 in that case */ 5359 t = val == 0 ? -1 : val; 5360 5361 rc = sysctl_handle_int(oidp, &t, 0, req); 5362 return (rc); 5363 } 5364 5365 #ifdef SBUF_DRAIN 5366 static int 5367 sysctl_cctrl(SYSCTL_HANDLER_ARGS) 5368 { 5369 struct adapter *sc = arg1; 5370 struct sbuf *sb; 5371 int rc, i; 5372 uint16_t incr[NMTUS][NCCTRL_WIN]; 5373 static const char *dec_fac[] = { 5374 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 5375 "0.9375" 5376 }; 5377 5378 rc = sysctl_wire_old_buffer(req, 0); 5379 if (rc != 0) 5380 return (rc); 5381 5382 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5383 if (sb == NULL) 5384 return (ENOMEM); 5385 5386 t4_read_cong_tbl(sc, incr); 5387 5388 for (i = 0; i < NCCTRL_WIN; ++i) { 5389 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 5390 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 5391 incr[5][i], incr[6][i], incr[7][i]); 5392 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 5393 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 5394 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 5395 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 5396 } 5397 5398 rc = sbuf_finish(sb); 5399 sbuf_delete(sb); 5400 5401 return (rc); 5402 } 5403 5404 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 5405 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 5406 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 5407 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 5408 }; 5409 5410 static int 5411 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 5412 { 5413 struct adapter *sc = arg1; 5414 struct sbuf *sb; 5415 int rc, i, n, qid = arg2; 5416 uint32_t *buf, *p; 5417 char *qtype; 5418 u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5; 5419 5420 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 5421 ("%s: bad qid %d\n", __func__, qid)); 5422 5423 if (qid < CIM_NUM_IBQ) { 5424 /* inbound queue */ 5425 qtype = "IBQ"; 5426 n = 4 * CIM_IBQ_SIZE; 5427 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5428 rc = t4_read_cim_ibq(sc, qid, buf, n); 5429 } else { 5430 /* outbound queue */ 5431 qtype = "OBQ"; 5432 qid -= CIM_NUM_IBQ; 5433 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 5434 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5435 rc = t4_read_cim_obq(sc, qid, buf, n); 5436 } 5437 5438 if (rc < 0) { 5439 rc = -rc; 5440 goto done; 5441 } 5442 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 5443 5444 rc = sysctl_wire_old_buffer(req, 0); 5445 if (rc != 0) 5446 goto done; 5447 5448 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5449 if (sb == NULL) { 5450 rc = ENOMEM; 5451 goto done; 5452 } 5453 5454 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 5455 for (i = 0, p = buf; i < n; i += 16, p += 4) 5456 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 5457 p[2], p[3]); 5458 5459 rc = sbuf_finish(sb); 5460 sbuf_delete(sb); 5461 done: 5462 free(buf, M_CXGBE); 5463 return (rc); 5464 } 5465 5466 static int 5467 sysctl_cim_la(SYSCTL_HANDLER_ARGS) 5468 { 5469 struct adapter *sc = arg1; 5470 u_int cfg; 5471 struct sbuf *sb; 5472 uint32_t *buf, *p; 5473 int rc; 5474 5475 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 5476 if (rc != 0) 5477 return (rc); 5478 5479 rc = sysctl_wire_old_buffer(req, 0); 5480 if (rc != 0) 5481 return (rc); 5482 5483 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5484 if (sb == NULL) 5485 return (ENOMEM); 5486 5487 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 5488 M_ZERO | M_WAITOK); 5489 5490 rc = -t4_cim_read_la(sc, buf, NULL); 5491 if (rc != 0) 5492 goto done; 5493 5494 sbuf_printf(sb, "Status Data PC%s", 5495 cfg & F_UPDBGLACAPTPCONLY ? "" : 5496 " LS0Stat LS0Addr LS0Data"); 5497 5498 KASSERT((sc->params.cim_la_size & 7) == 0, 5499 ("%s: p will walk off the end of buf", __func__)); 5500 5501 for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) { 5502 if (cfg & F_UPDBGLACAPTPCONLY) { 5503 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 5504 p[6], p[7]); 5505 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 5506 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 5507 p[4] & 0xff, p[5] >> 8); 5508 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 5509 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5510 p[1] & 0xf, p[2] >> 4); 5511 } else { 5512 sbuf_printf(sb, 5513 "\n %02x %x%07x %x%07x %08x %08x " 5514 "%08x%08x%08x%08x", 5515 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5516 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 5517 p[6], p[7]); 5518 } 5519 } 5520 5521 rc = sbuf_finish(sb); 5522 sbuf_delete(sb); 5523 done: 5524 free(buf, M_CXGBE); 5525 return (rc); 5526 } 5527 5528 static int 5529 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) 5530 { 5531 struct adapter *sc = arg1; 5532 u_int i; 5533 struct sbuf *sb; 5534 uint32_t *buf, *p; 5535 int rc; 5536 5537 rc = sysctl_wire_old_buffer(req, 0); 5538 if (rc != 0) 5539 return (rc); 5540 5541 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5542 if (sb == NULL) 5543 return (ENOMEM); 5544 5545 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, 5546 M_ZERO | M_WAITOK); 5547 5548 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); 5549 p = buf; 5550 5551 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5552 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], 5553 p[1], p[0]); 5554 } 5555 5556 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); 5557 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5558 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", 5559 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, 5560 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, 5561 (p[1] >> 2) | ((p[2] & 3) << 30), 5562 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, 5563 p[0] & 1); 5564 } 5565 5566 rc = sbuf_finish(sb); 5567 sbuf_delete(sb); 5568 free(buf, M_CXGBE); 5569 return (rc); 5570 } 5571 5572 static int 5573 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) 5574 { 5575 struct adapter *sc = arg1; 5576 u_int i; 5577 struct sbuf *sb; 5578 uint32_t *buf, *p; 5579 int rc; 5580 5581 rc = sysctl_wire_old_buffer(req, 0); 5582 if (rc != 0) 5583 return (rc); 5584 5585 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5586 if (sb == NULL) 5587 return (ENOMEM); 5588 5589 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, 5590 M_ZERO | M_WAITOK); 5591 5592 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); 5593 p = buf; 5594 5595 sbuf_printf(sb, "Cntl ID DataBE Addr Data"); 5596 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5597 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", 5598 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, 5599 p[4], p[3], p[2], p[1], p[0]); 5600 } 5601 5602 sbuf_printf(sb, "\n\nCntl ID Data"); 5603 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5604 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", 5605 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); 5606 } 5607 5608 rc = sbuf_finish(sb); 5609 sbuf_delete(sb); 5610 free(buf, M_CXGBE); 5611 return (rc); 5612 } 5613 5614 static int 5615 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 5616 { 5617 struct adapter *sc = arg1; 5618 struct sbuf *sb; 5619 int rc, i; 5620 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5621 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5622 uint16_t thres[CIM_NUM_IBQ]; 5623 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 5624 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 5625 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 5626 5627 if (is_t4(sc)) { 5628 cim_num_obq = CIM_NUM_OBQ; 5629 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 5630 obq_rdaddr = A_UP_OBQ_0_REALADDR; 5631 } else { 5632 cim_num_obq = CIM_NUM_OBQ_T5; 5633 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 5634 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 5635 } 5636 nq = CIM_NUM_IBQ + cim_num_obq; 5637 5638 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 5639 if (rc == 0) 5640 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 5641 if (rc != 0) 5642 return (rc); 5643 5644 t4_read_cimq_cfg(sc, base, size, thres); 5645 5646 rc = sysctl_wire_old_buffer(req, 0); 5647 if (rc != 0) 5648 return (rc); 5649 5650 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5651 if (sb == NULL) 5652 return (ENOMEM); 5653 5654 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 5655 5656 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 5657 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 5658 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 5659 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5660 G_QUEREMFLITS(p[2]) * 16); 5661 for ( ; i < nq; i++, p += 4, wr += 2) 5662 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 5663 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 5664 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5665 G_QUEREMFLITS(p[2]) * 16); 5666 5667 rc = sbuf_finish(sb); 5668 sbuf_delete(sb); 5669 5670 return (rc); 5671 } 5672 5673 static int 5674 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 5675 { 5676 struct adapter *sc = arg1; 5677 struct sbuf *sb; 5678 int rc; 5679 struct tp_cpl_stats stats; 5680 5681 rc = sysctl_wire_old_buffer(req, 0); 5682 if (rc != 0) 5683 return (rc); 5684 5685 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5686 if (sb == NULL) 5687 return (ENOMEM); 5688 5689 t4_tp_get_cpl_stats(sc, &stats); 5690 5691 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 5692 "channel 3\n"); 5693 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 5694 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 5695 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 5696 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 5697 5698 rc = sbuf_finish(sb); 5699 sbuf_delete(sb); 5700 5701 return (rc); 5702 } 5703 5704 static int 5705 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 5706 { 5707 struct adapter *sc = arg1; 5708 struct sbuf *sb; 5709 int rc; 5710 struct tp_usm_stats stats; 5711 5712 rc = sysctl_wire_old_buffer(req, 0); 5713 if (rc != 0) 5714 return(rc); 5715 5716 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5717 if (sb == NULL) 5718 return (ENOMEM); 5719 5720 t4_get_usm_stats(sc, &stats); 5721 5722 sbuf_printf(sb, "Frames: %u\n", stats.frames); 5723 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 5724 sbuf_printf(sb, "Drops: %u", stats.drops); 5725 5726 rc = sbuf_finish(sb); 5727 sbuf_delete(sb); 5728 5729 return (rc); 5730 } 5731 5732 const char *devlog_level_strings[] = { 5733 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 5734 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 5735 [FW_DEVLOG_LEVEL_ERR] = "ERR", 5736 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 5737 [FW_DEVLOG_LEVEL_INFO] = "INFO", 5738 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 5739 }; 5740 5741 const char *devlog_facility_strings[] = { 5742 [FW_DEVLOG_FACILITY_CORE] = "CORE", 5743 [FW_DEVLOG_FACILITY_CF] = "CF", 5744 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 5745 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 5746 [FW_DEVLOG_FACILITY_RES] = "RES", 5747 [FW_DEVLOG_FACILITY_HW] = "HW", 5748 [FW_DEVLOG_FACILITY_FLR] = "FLR", 5749 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 5750 [FW_DEVLOG_FACILITY_PHY] = "PHY", 5751 [FW_DEVLOG_FACILITY_MAC] = "MAC", 5752 [FW_DEVLOG_FACILITY_PORT] = "PORT", 5753 [FW_DEVLOG_FACILITY_VI] = "VI", 5754 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 5755 [FW_DEVLOG_FACILITY_ACL] = "ACL", 5756 [FW_DEVLOG_FACILITY_TM] = "TM", 5757 [FW_DEVLOG_FACILITY_QFC] = "QFC", 5758 [FW_DEVLOG_FACILITY_DCB] = "DCB", 5759 [FW_DEVLOG_FACILITY_ETH] = "ETH", 5760 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 5761 [FW_DEVLOG_FACILITY_RI] = "RI", 5762 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 5763 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 5764 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 5765 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 5766 }; 5767 5768 static int 5769 sysctl_devlog(SYSCTL_HANDLER_ARGS) 5770 { 5771 struct adapter *sc = arg1; 5772 struct devlog_params *dparams = &sc->params.devlog; 5773 struct fw_devlog_e *buf, *e; 5774 int i, j, rc, nentries, first = 0, m; 5775 struct sbuf *sb; 5776 uint64_t ftstamp = UINT64_MAX; 5777 5778 if (dparams->start == 0) { 5779 dparams->memtype = FW_MEMTYPE_EDC0; 5780 dparams->start = 0x84000; 5781 dparams->size = 32768; 5782 } 5783 5784 nentries = dparams->size / sizeof(struct fw_devlog_e); 5785 5786 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 5787 if (buf == NULL) 5788 return (ENOMEM); 5789 5790 m = fwmtype_to_hwmtype(dparams->memtype); 5791 rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf); 5792 if (rc != 0) 5793 goto done; 5794 5795 for (i = 0; i < nentries; i++) { 5796 e = &buf[i]; 5797 5798 if (e->timestamp == 0) 5799 break; /* end */ 5800 5801 e->timestamp = be64toh(e->timestamp); 5802 e->seqno = be32toh(e->seqno); 5803 for (j = 0; j < 8; j++) 5804 e->params[j] = be32toh(e->params[j]); 5805 5806 if (e->timestamp < ftstamp) { 5807 ftstamp = e->timestamp; 5808 first = i; 5809 } 5810 } 5811 5812 if (buf[first].timestamp == 0) 5813 goto done; /* nothing in the log */ 5814 5815 rc = sysctl_wire_old_buffer(req, 0); 5816 if (rc != 0) 5817 goto done; 5818 5819 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5820 if (sb == NULL) { 5821 rc = ENOMEM; 5822 goto done; 5823 } 5824 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 5825 "Seq#", "Tstamp", "Level", "Facility", "Message"); 5826 5827 i = first; 5828 do { 5829 e = &buf[i]; 5830 if (e->timestamp == 0) 5831 break; /* end */ 5832 5833 sbuf_printf(sb, "%10d %15ju %8s %8s ", 5834 e->seqno, e->timestamp, 5835 (e->level < nitems(devlog_level_strings) ? 5836 devlog_level_strings[e->level] : "UNKNOWN"), 5837 (e->facility < nitems(devlog_facility_strings) ? 5838 devlog_facility_strings[e->facility] : "UNKNOWN")); 5839 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 5840 e->params[2], e->params[3], e->params[4], 5841 e->params[5], e->params[6], e->params[7]); 5842 5843 if (++i == nentries) 5844 i = 0; 5845 } while (i != first); 5846 5847 rc = sbuf_finish(sb); 5848 sbuf_delete(sb); 5849 done: 5850 free(buf, M_CXGBE); 5851 return (rc); 5852 } 5853 5854 static int 5855 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 5856 { 5857 struct adapter *sc = arg1; 5858 struct sbuf *sb; 5859 int rc; 5860 struct tp_fcoe_stats stats[4]; 5861 5862 rc = sysctl_wire_old_buffer(req, 0); 5863 if (rc != 0) 5864 return (rc); 5865 5866 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5867 if (sb == NULL) 5868 return (ENOMEM); 5869 5870 t4_get_fcoe_stats(sc, 0, &stats[0]); 5871 t4_get_fcoe_stats(sc, 1, &stats[1]); 5872 t4_get_fcoe_stats(sc, 2, &stats[2]); 5873 t4_get_fcoe_stats(sc, 3, &stats[3]); 5874 5875 sbuf_printf(sb, " channel 0 channel 1 " 5876 "channel 2 channel 3\n"); 5877 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 5878 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 5879 stats[3].octetsDDP); 5880 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 5881 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 5882 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 5883 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 5884 stats[3].framesDrop); 5885 5886 rc = sbuf_finish(sb); 5887 sbuf_delete(sb); 5888 5889 return (rc); 5890 } 5891 5892 static int 5893 sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 5894 { 5895 struct adapter *sc = arg1; 5896 struct sbuf *sb; 5897 int rc, i; 5898 unsigned int map, kbps, ipg, mode; 5899 unsigned int pace_tab[NTX_SCHED]; 5900 5901 rc = sysctl_wire_old_buffer(req, 0); 5902 if (rc != 0) 5903 return (rc); 5904 5905 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5906 if (sb == NULL) 5907 return (ENOMEM); 5908 5909 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 5910 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 5911 t4_read_pace_tbl(sc, pace_tab); 5912 5913 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 5914 "Class IPG (0.1 ns) Flow IPG (us)"); 5915 5916 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 5917 t4_get_tx_sched(sc, i, &kbps, &ipg); 5918 sbuf_printf(sb, "\n %u %-5s %u ", i, 5919 (mode & (1 << i)) ? "flow" : "class", map & 3); 5920 if (kbps) 5921 sbuf_printf(sb, "%9u ", kbps); 5922 else 5923 sbuf_printf(sb, " disabled "); 5924 5925 if (ipg) 5926 sbuf_printf(sb, "%13u ", ipg); 5927 else 5928 sbuf_printf(sb, " disabled "); 5929 5930 if (pace_tab[i]) 5931 sbuf_printf(sb, "%10u", pace_tab[i]); 5932 else 5933 sbuf_printf(sb, " disabled"); 5934 } 5935 5936 rc = sbuf_finish(sb); 5937 sbuf_delete(sb); 5938 5939 return (rc); 5940 } 5941 5942 static int 5943 sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 5944 { 5945 struct adapter *sc = arg1; 5946 struct sbuf *sb; 5947 int rc, i, j; 5948 uint64_t *p0, *p1; 5949 struct lb_port_stats s[2]; 5950 static const char *stat_name[] = { 5951 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 5952 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 5953 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 5954 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 5955 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 5956 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 5957 "BG2FramesTrunc:", "BG3FramesTrunc:" 5958 }; 5959 5960 rc = sysctl_wire_old_buffer(req, 0); 5961 if (rc != 0) 5962 return (rc); 5963 5964 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5965 if (sb == NULL) 5966 return (ENOMEM); 5967 5968 memset(s, 0, sizeof(s)); 5969 5970 for (i = 0; i < 4; i += 2) { 5971 t4_get_lb_stats(sc, i, &s[0]); 5972 t4_get_lb_stats(sc, i + 1, &s[1]); 5973 5974 p0 = &s[0].octets; 5975 p1 = &s[1].octets; 5976 sbuf_printf(sb, "%s Loopback %u" 5977 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 5978 5979 for (j = 0; j < nitems(stat_name); j++) 5980 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 5981 *p0++, *p1++); 5982 } 5983 5984 rc = sbuf_finish(sb); 5985 sbuf_delete(sb); 5986 5987 return (rc); 5988 } 5989 5990 static int 5991 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) 5992 { 5993 int rc = 0; 5994 struct port_info *pi = arg1; 5995 struct sbuf *sb; 5996 static const char *linkdnreasons[] = { 5997 "non-specific", "remote fault", "autoneg failed", "reserved3", 5998 "PHY overheated", "unknown", "rx los", "reserved7" 5999 }; 6000 6001 rc = sysctl_wire_old_buffer(req, 0); 6002 if (rc != 0) 6003 return(rc); 6004 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); 6005 if (sb == NULL) 6006 return (ENOMEM); 6007 6008 if (pi->linkdnrc < 0) 6009 sbuf_printf(sb, "n/a"); 6010 else if (pi->linkdnrc < nitems(linkdnreasons)) 6011 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]); 6012 else 6013 sbuf_printf(sb, "%d", pi->linkdnrc); 6014 6015 rc = sbuf_finish(sb); 6016 sbuf_delete(sb); 6017 6018 return (rc); 6019 } 6020 6021 struct mem_desc { 6022 unsigned int base; 6023 unsigned int limit; 6024 unsigned int idx; 6025 }; 6026 6027 static int 6028 mem_desc_cmp(const void *a, const void *b) 6029 { 6030 return ((const struct mem_desc *)a)->base - 6031 ((const struct mem_desc *)b)->base; 6032 } 6033 6034 static void 6035 mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 6036 unsigned int to) 6037 { 6038 unsigned int size; 6039 6040 size = to - from + 1; 6041 if (size == 0) 6042 return; 6043 6044 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 6045 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 6046 } 6047 6048 static int 6049 sysctl_meminfo(SYSCTL_HANDLER_ARGS) 6050 { 6051 struct adapter *sc = arg1; 6052 struct sbuf *sb; 6053 int rc, i, n; 6054 uint32_t lo, hi, used, alloc; 6055 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 6056 static const char *region[] = { 6057 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 6058 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 6059 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 6060 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 6061 "RQUDP region:", "PBL region:", "TXPBL region:", 6062 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 6063 "On-chip queues:" 6064 }; 6065 struct mem_desc avail[4]; 6066 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 6067 struct mem_desc *md = mem; 6068 6069 rc = sysctl_wire_old_buffer(req, 0); 6070 if (rc != 0) 6071 return (rc); 6072 6073 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6074 if (sb == NULL) 6075 return (ENOMEM); 6076 6077 for (i = 0; i < nitems(mem); i++) { 6078 mem[i].limit = 0; 6079 mem[i].idx = i; 6080 } 6081 6082 /* Find and sort the populated memory ranges */ 6083 i = 0; 6084 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 6085 if (lo & F_EDRAM0_ENABLE) { 6086 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 6087 avail[i].base = G_EDRAM0_BASE(hi) << 20; 6088 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 6089 avail[i].idx = 0; 6090 i++; 6091 } 6092 if (lo & F_EDRAM1_ENABLE) { 6093 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 6094 avail[i].base = G_EDRAM1_BASE(hi) << 20; 6095 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 6096 avail[i].idx = 1; 6097 i++; 6098 } 6099 if (lo & F_EXT_MEM_ENABLE) { 6100 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 6101 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 6102 avail[i].limit = avail[i].base + 6103 (G_EXT_MEM_SIZE(hi) << 20); 6104 avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */ 6105 i++; 6106 } 6107 if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) { 6108 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 6109 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 6110 avail[i].limit = avail[i].base + 6111 (G_EXT_MEM1_SIZE(hi) << 20); 6112 avail[i].idx = 4; 6113 i++; 6114 } 6115 if (!i) /* no memory available */ 6116 return 0; 6117 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 6118 6119 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 6120 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 6121 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 6122 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 6123 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 6124 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 6125 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 6126 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 6127 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 6128 6129 /* the next few have explicit upper bounds */ 6130 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 6131 md->limit = md->base - 1 + 6132 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 6133 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 6134 md++; 6135 6136 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 6137 md->limit = md->base - 1 + 6138 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 6139 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 6140 md++; 6141 6142 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6143 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 6144 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 6145 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 6146 } else { 6147 md->base = 0; 6148 md->idx = nitems(region); /* hide it */ 6149 } 6150 md++; 6151 6152 #define ulp_region(reg) \ 6153 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 6154 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 6155 6156 ulp_region(RX_ISCSI); 6157 ulp_region(RX_TDDP); 6158 ulp_region(TX_TPT); 6159 ulp_region(RX_STAG); 6160 ulp_region(RX_RQ); 6161 ulp_region(RX_RQUDP); 6162 ulp_region(RX_PBL); 6163 ulp_region(TX_PBL); 6164 #undef ulp_region 6165 6166 md->base = 0; 6167 md->idx = nitems(region); 6168 if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) { 6169 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); 6170 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc, 6171 A_SGE_DBVFIFO_SIZE))) << 2) - 1; 6172 } 6173 md++; 6174 6175 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 6176 md->limit = md->base + sc->tids.ntids - 1; 6177 md++; 6178 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 6179 md->limit = md->base + sc->tids.ntids - 1; 6180 md++; 6181 6182 md->base = sc->vres.ocq.start; 6183 if (sc->vres.ocq.size) 6184 md->limit = md->base + sc->vres.ocq.size - 1; 6185 else 6186 md->idx = nitems(region); /* hide it */ 6187 md++; 6188 6189 /* add any address-space holes, there can be up to 3 */ 6190 for (n = 0; n < i - 1; n++) 6191 if (avail[n].limit < avail[n + 1].base) 6192 (md++)->base = avail[n].limit; 6193 if (avail[n].limit) 6194 (md++)->base = avail[n].limit; 6195 6196 n = md - mem; 6197 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 6198 6199 for (lo = 0; lo < i; lo++) 6200 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 6201 avail[lo].limit - 1); 6202 6203 sbuf_printf(sb, "\n"); 6204 for (i = 0; i < n; i++) { 6205 if (mem[i].idx >= nitems(region)) 6206 continue; /* skip holes */ 6207 if (!mem[i].limit) 6208 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 6209 mem_region_show(sb, region[mem[i].idx], mem[i].base, 6210 mem[i].limit); 6211 } 6212 6213 sbuf_printf(sb, "\n"); 6214 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 6215 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 6216 mem_region_show(sb, "uP RAM:", lo, hi); 6217 6218 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 6219 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 6220 mem_region_show(sb, "uP Extmem2:", lo, hi); 6221 6222 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 6223 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 6224 G_PMRXMAXPAGE(lo), 6225 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 6226 (lo & F_PMRXNUMCHN) ? 2 : 1); 6227 6228 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 6229 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 6230 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 6231 G_PMTXMAXPAGE(lo), 6232 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 6233 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 6234 sbuf_printf(sb, "%u p-structs\n", 6235 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 6236 6237 for (i = 0; i < 4; i++) { 6238 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 6239 if (is_t4(sc)) { 6240 used = G_USED(lo); 6241 alloc = G_ALLOC(lo); 6242 } else { 6243 used = G_T5_USED(lo); 6244 alloc = G_T5_ALLOC(lo); 6245 } 6246 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 6247 i, used, alloc); 6248 } 6249 for (i = 0; i < 4; i++) { 6250 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 6251 if (is_t4(sc)) { 6252 used = G_USED(lo); 6253 alloc = G_ALLOC(lo); 6254 } else { 6255 used = G_T5_USED(lo); 6256 alloc = G_T5_ALLOC(lo); 6257 } 6258 sbuf_printf(sb, 6259 "\nLoopback %d using %u pages out of %u allocated", 6260 i, used, alloc); 6261 } 6262 6263 rc = sbuf_finish(sb); 6264 sbuf_delete(sb); 6265 6266 return (rc); 6267 } 6268 6269 static inline void 6270 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) 6271 { 6272 *mask = x | y; 6273 y = htobe64(y); 6274 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); 6275 } 6276 6277 static int 6278 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) 6279 { 6280 struct adapter *sc = arg1; 6281 struct sbuf *sb; 6282 int rc, i, n; 6283 6284 rc = sysctl_wire_old_buffer(req, 0); 6285 if (rc != 0) 6286 return (rc); 6287 6288 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6289 if (sb == NULL) 6290 return (ENOMEM); 6291 6292 sbuf_printf(sb, 6293 "Idx Ethernet address Mask Vld Ports PF" 6294 " VF Replication P0 P1 P2 P3 ML"); 6295 n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES : 6296 NUM_MPS_T5_CLS_SRAM_L_INSTANCES; 6297 for (i = 0; i < n; i++) { 6298 uint64_t tcamx, tcamy, mask; 6299 uint32_t cls_lo, cls_hi; 6300 uint8_t addr[ETHER_ADDR_LEN]; 6301 6302 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); 6303 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); 6304 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 6305 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 6306 6307 if (tcamx & tcamy) 6308 continue; 6309 6310 tcamxy2valmask(tcamx, tcamy, addr, &mask); 6311 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" 6312 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], 6313 addr[3], addr[4], addr[5], (uintmax_t)mask, 6314 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', 6315 G_PORTMAP(cls_hi), G_PF(cls_lo), 6316 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); 6317 6318 if (cls_lo & F_REPLICATE) { 6319 struct fw_ldst_cmd ldst_cmd; 6320 6321 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 6322 ldst_cmd.op_to_addrspace = 6323 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 6324 F_FW_CMD_REQUEST | F_FW_CMD_READ | 6325 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 6326 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 6327 ldst_cmd.u.mps.fid_ctl = 6328 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 6329 V_FW_LDST_CMD_CTL(i)); 6330 6331 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 6332 "t4mps"); 6333 if (rc) 6334 break; 6335 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 6336 sizeof(ldst_cmd), &ldst_cmd); 6337 end_synchronized_op(sc, 0); 6338 6339 if (rc != 0) { 6340 sbuf_printf(sb, 6341 " ------------ error %3u ------------", rc); 6342 rc = 0; 6343 } else { 6344 sbuf_printf(sb, " %08x %08x %08x %08x", 6345 be32toh(ldst_cmd.u.mps.rplc127_96), 6346 be32toh(ldst_cmd.u.mps.rplc95_64), 6347 be32toh(ldst_cmd.u.mps.rplc63_32), 6348 be32toh(ldst_cmd.u.mps.rplc31_0)); 6349 } 6350 } else 6351 sbuf_printf(sb, "%36s", ""); 6352 6353 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), 6354 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), 6355 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); 6356 } 6357 6358 if (rc) 6359 (void) sbuf_finish(sb); 6360 else 6361 rc = sbuf_finish(sb); 6362 sbuf_delete(sb); 6363 6364 return (rc); 6365 } 6366 6367 static int 6368 sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 6369 { 6370 struct adapter *sc = arg1; 6371 struct sbuf *sb; 6372 int rc; 6373 uint16_t mtus[NMTUS]; 6374 6375 rc = sysctl_wire_old_buffer(req, 0); 6376 if (rc != 0) 6377 return (rc); 6378 6379 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6380 if (sb == NULL) 6381 return (ENOMEM); 6382 6383 t4_read_mtu_tbl(sc, mtus, NULL); 6384 6385 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 6386 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 6387 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 6388 mtus[14], mtus[15]); 6389 6390 rc = sbuf_finish(sb); 6391 sbuf_delete(sb); 6392 6393 return (rc); 6394 } 6395 6396 static int 6397 sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 6398 { 6399 struct adapter *sc = arg1; 6400 struct sbuf *sb; 6401 int rc, i; 6402 uint32_t cnt[PM_NSTATS]; 6403 uint64_t cyc[PM_NSTATS]; 6404 static const char *rx_stats[] = { 6405 "Read:", "Write bypass:", "Write mem:", "Flush:" 6406 }; 6407 static const char *tx_stats[] = { 6408 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:" 6409 }; 6410 6411 rc = sysctl_wire_old_buffer(req, 0); 6412 if (rc != 0) 6413 return (rc); 6414 6415 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6416 if (sb == NULL) 6417 return (ENOMEM); 6418 6419 t4_pmtx_get_stats(sc, cnt, cyc); 6420 sbuf_printf(sb, " Tx pcmds Tx bytes"); 6421 for (i = 0; i < ARRAY_SIZE(tx_stats); i++) 6422 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i], 6423 cyc[i]); 6424 6425 t4_pmrx_get_stats(sc, cnt, cyc); 6426 sbuf_printf(sb, "\n Rx pcmds Rx bytes"); 6427 for (i = 0; i < ARRAY_SIZE(rx_stats); i++) 6428 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i], 6429 cyc[i]); 6430 6431 rc = sbuf_finish(sb); 6432 sbuf_delete(sb); 6433 6434 return (rc); 6435 } 6436 6437 static int 6438 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 6439 { 6440 struct adapter *sc = arg1; 6441 struct sbuf *sb; 6442 int rc; 6443 struct tp_rdma_stats stats; 6444 6445 rc = sysctl_wire_old_buffer(req, 0); 6446 if (rc != 0) 6447 return (rc); 6448 6449 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6450 if (sb == NULL) 6451 return (ENOMEM); 6452 6453 t4_tp_get_rdma_stats(sc, &stats); 6454 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 6455 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 6456 6457 rc = sbuf_finish(sb); 6458 sbuf_delete(sb); 6459 6460 return (rc); 6461 } 6462 6463 static int 6464 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 6465 { 6466 struct adapter *sc = arg1; 6467 struct sbuf *sb; 6468 int rc; 6469 struct tp_tcp_stats v4, v6; 6470 6471 rc = sysctl_wire_old_buffer(req, 0); 6472 if (rc != 0) 6473 return (rc); 6474 6475 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6476 if (sb == NULL) 6477 return (ENOMEM); 6478 6479 t4_tp_get_tcp_stats(sc, &v4, &v6); 6480 sbuf_printf(sb, 6481 " IP IPv6\n"); 6482 sbuf_printf(sb, "OutRsts: %20u %20u\n", 6483 v4.tcpOutRsts, v6.tcpOutRsts); 6484 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 6485 v4.tcpInSegs, v6.tcpInSegs); 6486 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 6487 v4.tcpOutSegs, v6.tcpOutSegs); 6488 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 6489 v4.tcpRetransSegs, v6.tcpRetransSegs); 6490 6491 rc = sbuf_finish(sb); 6492 sbuf_delete(sb); 6493 6494 return (rc); 6495 } 6496 6497 static int 6498 sysctl_tids(SYSCTL_HANDLER_ARGS) 6499 { 6500 struct adapter *sc = arg1; 6501 struct sbuf *sb; 6502 int rc; 6503 struct tid_info *t = &sc->tids; 6504 6505 rc = sysctl_wire_old_buffer(req, 0); 6506 if (rc != 0) 6507 return (rc); 6508 6509 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6510 if (sb == NULL) 6511 return (ENOMEM); 6512 6513 if (t->natids) { 6514 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 6515 t->atids_in_use); 6516 } 6517 6518 if (t->ntids) { 6519 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6520 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 6521 6522 if (b) { 6523 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 6524 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6525 t->ntids - 1); 6526 } else { 6527 sbuf_printf(sb, "TID range: %u-%u", 6528 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6529 t->ntids - 1); 6530 } 6531 } else 6532 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 6533 sbuf_printf(sb, ", in use: %u\n", 6534 atomic_load_acq_int(&t->tids_in_use)); 6535 } 6536 6537 if (t->nstids) { 6538 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 6539 t->stid_base + t->nstids - 1, t->stids_in_use); 6540 } 6541 6542 if (t->nftids) { 6543 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 6544 t->ftid_base + t->nftids - 1); 6545 } 6546 6547 if (t->netids) { 6548 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base, 6549 t->etid_base + t->netids - 1); 6550 } 6551 6552 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 6553 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 6554 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 6555 6556 rc = sbuf_finish(sb); 6557 sbuf_delete(sb); 6558 6559 return (rc); 6560 } 6561 6562 static int 6563 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 6564 { 6565 struct adapter *sc = arg1; 6566 struct sbuf *sb; 6567 int rc; 6568 struct tp_err_stats stats; 6569 6570 rc = sysctl_wire_old_buffer(req, 0); 6571 if (rc != 0) 6572 return (rc); 6573 6574 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6575 if (sb == NULL) 6576 return (ENOMEM); 6577 6578 t4_tp_get_err_stats(sc, &stats); 6579 6580 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6581 "channel 3\n"); 6582 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 6583 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 6584 stats.macInErrs[3]); 6585 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 6586 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 6587 stats.hdrInErrs[3]); 6588 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 6589 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 6590 stats.tcpInErrs[3]); 6591 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 6592 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 6593 stats.tcp6InErrs[3]); 6594 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 6595 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 6596 stats.tnlCongDrops[3]); 6597 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 6598 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 6599 stats.tnlTxDrops[3]); 6600 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 6601 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 6602 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 6603 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 6604 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 6605 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 6606 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 6607 stats.ofldNoNeigh, stats.ofldCongDefer); 6608 6609 rc = sbuf_finish(sb); 6610 sbuf_delete(sb); 6611 6612 return (rc); 6613 } 6614 6615 struct field_desc { 6616 const char *name; 6617 u_int start; 6618 u_int width; 6619 }; 6620 6621 static void 6622 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) 6623 { 6624 char buf[32]; 6625 int line_size = 0; 6626 6627 while (f->name) { 6628 uint64_t mask = (1ULL << f->width) - 1; 6629 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, 6630 ((uintmax_t)v >> f->start) & mask); 6631 6632 if (line_size + len >= 79) { 6633 line_size = 8; 6634 sbuf_printf(sb, "\n "); 6635 } 6636 sbuf_printf(sb, "%s ", buf); 6637 line_size += len + 1; 6638 f++; 6639 } 6640 sbuf_printf(sb, "\n"); 6641 } 6642 6643 static struct field_desc tp_la0[] = { 6644 { "RcfOpCodeOut", 60, 4 }, 6645 { "State", 56, 4 }, 6646 { "WcfState", 52, 4 }, 6647 { "RcfOpcSrcOut", 50, 2 }, 6648 { "CRxError", 49, 1 }, 6649 { "ERxError", 48, 1 }, 6650 { "SanityFailed", 47, 1 }, 6651 { "SpuriousMsg", 46, 1 }, 6652 { "FlushInputMsg", 45, 1 }, 6653 { "FlushInputCpl", 44, 1 }, 6654 { "RssUpBit", 43, 1 }, 6655 { "RssFilterHit", 42, 1 }, 6656 { "Tid", 32, 10 }, 6657 { "InitTcb", 31, 1 }, 6658 { "LineNumber", 24, 7 }, 6659 { "Emsg", 23, 1 }, 6660 { "EdataOut", 22, 1 }, 6661 { "Cmsg", 21, 1 }, 6662 { "CdataOut", 20, 1 }, 6663 { "EreadPdu", 19, 1 }, 6664 { "CreadPdu", 18, 1 }, 6665 { "TunnelPkt", 17, 1 }, 6666 { "RcfPeerFin", 16, 1 }, 6667 { "RcfReasonOut", 12, 4 }, 6668 { "TxCchannel", 10, 2 }, 6669 { "RcfTxChannel", 8, 2 }, 6670 { "RxEchannel", 6, 2 }, 6671 { "RcfRxChannel", 5, 1 }, 6672 { "RcfDataOutSrdy", 4, 1 }, 6673 { "RxDvld", 3, 1 }, 6674 { "RxOoDvld", 2, 1 }, 6675 { "RxCongestion", 1, 1 }, 6676 { "TxCongestion", 0, 1 }, 6677 { NULL } 6678 }; 6679 6680 static struct field_desc tp_la1[] = { 6681 { "CplCmdIn", 56, 8 }, 6682 { "CplCmdOut", 48, 8 }, 6683 { "ESynOut", 47, 1 }, 6684 { "EAckOut", 46, 1 }, 6685 { "EFinOut", 45, 1 }, 6686 { "ERstOut", 44, 1 }, 6687 { "SynIn", 43, 1 }, 6688 { "AckIn", 42, 1 }, 6689 { "FinIn", 41, 1 }, 6690 { "RstIn", 40, 1 }, 6691 { "DataIn", 39, 1 }, 6692 { "DataInVld", 38, 1 }, 6693 { "PadIn", 37, 1 }, 6694 { "RxBufEmpty", 36, 1 }, 6695 { "RxDdp", 35, 1 }, 6696 { "RxFbCongestion", 34, 1 }, 6697 { "TxFbCongestion", 33, 1 }, 6698 { "TxPktSumSrdy", 32, 1 }, 6699 { "RcfUlpType", 28, 4 }, 6700 { "Eread", 27, 1 }, 6701 { "Ebypass", 26, 1 }, 6702 { "Esave", 25, 1 }, 6703 { "Static0", 24, 1 }, 6704 { "Cread", 23, 1 }, 6705 { "Cbypass", 22, 1 }, 6706 { "Csave", 21, 1 }, 6707 { "CPktOut", 20, 1 }, 6708 { "RxPagePoolFull", 18, 2 }, 6709 { "RxLpbkPkt", 17, 1 }, 6710 { "TxLpbkPkt", 16, 1 }, 6711 { "RxVfValid", 15, 1 }, 6712 { "SynLearned", 14, 1 }, 6713 { "SetDelEntry", 13, 1 }, 6714 { "SetInvEntry", 12, 1 }, 6715 { "CpcmdDvld", 11, 1 }, 6716 { "CpcmdSave", 10, 1 }, 6717 { "RxPstructsFull", 8, 2 }, 6718 { "EpcmdDvld", 7, 1 }, 6719 { "EpcmdFlush", 6, 1 }, 6720 { "EpcmdTrimPrefix", 5, 1 }, 6721 { "EpcmdTrimPostfix", 4, 1 }, 6722 { "ERssIp4Pkt", 3, 1 }, 6723 { "ERssIp6Pkt", 2, 1 }, 6724 { "ERssTcpUdpPkt", 1, 1 }, 6725 { "ERssFceFipPkt", 0, 1 }, 6726 { NULL } 6727 }; 6728 6729 static struct field_desc tp_la2[] = { 6730 { "CplCmdIn", 56, 8 }, 6731 { "MpsVfVld", 55, 1 }, 6732 { "MpsPf", 52, 3 }, 6733 { "MpsVf", 44, 8 }, 6734 { "SynIn", 43, 1 }, 6735 { "AckIn", 42, 1 }, 6736 { "FinIn", 41, 1 }, 6737 { "RstIn", 40, 1 }, 6738 { "DataIn", 39, 1 }, 6739 { "DataInVld", 38, 1 }, 6740 { "PadIn", 37, 1 }, 6741 { "RxBufEmpty", 36, 1 }, 6742 { "RxDdp", 35, 1 }, 6743 { "RxFbCongestion", 34, 1 }, 6744 { "TxFbCongestion", 33, 1 }, 6745 { "TxPktSumSrdy", 32, 1 }, 6746 { "RcfUlpType", 28, 4 }, 6747 { "Eread", 27, 1 }, 6748 { "Ebypass", 26, 1 }, 6749 { "Esave", 25, 1 }, 6750 { "Static0", 24, 1 }, 6751 { "Cread", 23, 1 }, 6752 { "Cbypass", 22, 1 }, 6753 { "Csave", 21, 1 }, 6754 { "CPktOut", 20, 1 }, 6755 { "RxPagePoolFull", 18, 2 }, 6756 { "RxLpbkPkt", 17, 1 }, 6757 { "TxLpbkPkt", 16, 1 }, 6758 { "RxVfValid", 15, 1 }, 6759 { "SynLearned", 14, 1 }, 6760 { "SetDelEntry", 13, 1 }, 6761 { "SetInvEntry", 12, 1 }, 6762 { "CpcmdDvld", 11, 1 }, 6763 { "CpcmdSave", 10, 1 }, 6764 { "RxPstructsFull", 8, 2 }, 6765 { "EpcmdDvld", 7, 1 }, 6766 { "EpcmdFlush", 6, 1 }, 6767 { "EpcmdTrimPrefix", 5, 1 }, 6768 { "EpcmdTrimPostfix", 4, 1 }, 6769 { "ERssIp4Pkt", 3, 1 }, 6770 { "ERssIp6Pkt", 2, 1 }, 6771 { "ERssTcpUdpPkt", 1, 1 }, 6772 { "ERssFceFipPkt", 0, 1 }, 6773 { NULL } 6774 }; 6775 6776 static void 6777 tp_la_show(struct sbuf *sb, uint64_t *p, int idx) 6778 { 6779 6780 field_desc_show(sb, *p, tp_la0); 6781 } 6782 6783 static void 6784 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) 6785 { 6786 6787 if (idx) 6788 sbuf_printf(sb, "\n"); 6789 field_desc_show(sb, p[0], tp_la0); 6790 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6791 field_desc_show(sb, p[1], tp_la0); 6792 } 6793 6794 static void 6795 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) 6796 { 6797 6798 if (idx) 6799 sbuf_printf(sb, "\n"); 6800 field_desc_show(sb, p[0], tp_la0); 6801 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6802 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); 6803 } 6804 6805 static int 6806 sysctl_tp_la(SYSCTL_HANDLER_ARGS) 6807 { 6808 struct adapter *sc = arg1; 6809 struct sbuf *sb; 6810 uint64_t *buf, *p; 6811 int rc; 6812 u_int i, inc; 6813 void (*show_func)(struct sbuf *, uint64_t *, int); 6814 6815 rc = sysctl_wire_old_buffer(req, 0); 6816 if (rc != 0) 6817 return (rc); 6818 6819 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6820 if (sb == NULL) 6821 return (ENOMEM); 6822 6823 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); 6824 6825 t4_tp_read_la(sc, buf, NULL); 6826 p = buf; 6827 6828 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { 6829 case 2: 6830 inc = 2; 6831 show_func = tp_la_show2; 6832 break; 6833 case 3: 6834 inc = 2; 6835 show_func = tp_la_show3; 6836 break; 6837 default: 6838 inc = 1; 6839 show_func = tp_la_show; 6840 } 6841 6842 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) 6843 (*show_func)(sb, p, i); 6844 6845 rc = sbuf_finish(sb); 6846 sbuf_delete(sb); 6847 free(buf, M_CXGBE); 6848 return (rc); 6849 } 6850 6851 static int 6852 sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 6853 { 6854 struct adapter *sc = arg1; 6855 struct sbuf *sb; 6856 int rc; 6857 u64 nrate[NCHAN], orate[NCHAN]; 6858 6859 rc = sysctl_wire_old_buffer(req, 0); 6860 if (rc != 0) 6861 return (rc); 6862 6863 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6864 if (sb == NULL) 6865 return (ENOMEM); 6866 6867 t4_get_chan_txrate(sc, nrate, orate); 6868 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6869 "channel 3\n"); 6870 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 6871 nrate[0], nrate[1], nrate[2], nrate[3]); 6872 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 6873 orate[0], orate[1], orate[2], orate[3]); 6874 6875 rc = sbuf_finish(sb); 6876 sbuf_delete(sb); 6877 6878 return (rc); 6879 } 6880 6881 static int 6882 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) 6883 { 6884 struct adapter *sc = arg1; 6885 struct sbuf *sb; 6886 uint32_t *buf, *p; 6887 int rc, i; 6888 6889 rc = sysctl_wire_old_buffer(req, 0); 6890 if (rc != 0) 6891 return (rc); 6892 6893 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6894 if (sb == NULL) 6895 return (ENOMEM); 6896 6897 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, 6898 M_ZERO | M_WAITOK); 6899 6900 t4_ulprx_read_la(sc, buf); 6901 p = buf; 6902 6903 sbuf_printf(sb, " Pcmd Type Message" 6904 " Data"); 6905 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { 6906 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", 6907 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); 6908 } 6909 6910 rc = sbuf_finish(sb); 6911 sbuf_delete(sb); 6912 free(buf, M_CXGBE); 6913 return (rc); 6914 } 6915 6916 static int 6917 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 6918 { 6919 struct adapter *sc = arg1; 6920 struct sbuf *sb; 6921 int rc, v; 6922 6923 rc = sysctl_wire_old_buffer(req, 0); 6924 if (rc != 0) 6925 return (rc); 6926 6927 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6928 if (sb == NULL) 6929 return (ENOMEM); 6930 6931 v = t4_read_reg(sc, A_SGE_STAT_CFG); 6932 if (G_STATSOURCE_T5(v) == 7) { 6933 if (G_STATMODE(v) == 0) { 6934 sbuf_printf(sb, "total %d, incomplete %d", 6935 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6936 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6937 } else if (G_STATMODE(v) == 1) { 6938 sbuf_printf(sb, "total %d, data overflow %d", 6939 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6940 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6941 } 6942 } 6943 rc = sbuf_finish(sb); 6944 sbuf_delete(sb); 6945 6946 return (rc); 6947 } 6948 #endif 6949 6950 static inline void 6951 txq_start(struct ifnet *ifp, struct sge_txq *txq) 6952 { 6953 struct buf_ring *br; 6954 struct mbuf *m; 6955 6956 TXQ_LOCK_ASSERT_OWNED(txq); 6957 6958 br = txq->br; 6959 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 6960 if (m) 6961 t4_eth_tx(ifp, txq, m); 6962 } 6963 6964 void 6965 t4_tx_callout(void *arg) 6966 { 6967 struct sge_eq *eq = arg; 6968 struct adapter *sc; 6969 6970 if (EQ_TRYLOCK(eq) == 0) 6971 goto reschedule; 6972 6973 if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) { 6974 EQ_UNLOCK(eq); 6975 reschedule: 6976 if (__predict_true(!(eq->flags && EQ_DOOMED))) 6977 callout_schedule(&eq->tx_callout, 1); 6978 return; 6979 } 6980 6981 EQ_LOCK_ASSERT_OWNED(eq); 6982 6983 if (__predict_true((eq->flags & EQ_DOOMED) == 0)) { 6984 6985 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 6986 struct sge_txq *txq = arg; 6987 struct port_info *pi = txq->ifp->if_softc; 6988 6989 sc = pi->adapter; 6990 } else { 6991 struct sge_wrq *wrq = arg; 6992 6993 sc = wrq->adapter; 6994 } 6995 6996 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task); 6997 } 6998 6999 EQ_UNLOCK(eq); 7000 } 7001 7002 void 7003 t4_tx_task(void *arg, int count) 7004 { 7005 struct sge_eq *eq = arg; 7006 7007 EQ_LOCK(eq); 7008 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 7009 struct sge_txq *txq = arg; 7010 txq_start(txq->ifp, txq); 7011 } else { 7012 struct sge_wrq *wrq = arg; 7013 t4_wrq_tx_locked(wrq->adapter, wrq, NULL); 7014 } 7015 EQ_UNLOCK(eq); 7016 } 7017 7018 static uint32_t 7019 fconf_to_mode(uint32_t fconf) 7020 { 7021 uint32_t mode; 7022 7023 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 7024 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 7025 7026 if (fconf & F_FRAGMENTATION) 7027 mode |= T4_FILTER_IP_FRAGMENT; 7028 7029 if (fconf & F_MPSHITTYPE) 7030 mode |= T4_FILTER_MPS_HIT_TYPE; 7031 7032 if (fconf & F_MACMATCH) 7033 mode |= T4_FILTER_MAC_IDX; 7034 7035 if (fconf & F_ETHERTYPE) 7036 mode |= T4_FILTER_ETH_TYPE; 7037 7038 if (fconf & F_PROTOCOL) 7039 mode |= T4_FILTER_IP_PROTO; 7040 7041 if (fconf & F_TOS) 7042 mode |= T4_FILTER_IP_TOS; 7043 7044 if (fconf & F_VLAN) 7045 mode |= T4_FILTER_VLAN; 7046 7047 if (fconf & F_VNIC_ID) 7048 mode |= T4_FILTER_VNIC; 7049 7050 if (fconf & F_PORT) 7051 mode |= T4_FILTER_PORT; 7052 7053 if (fconf & F_FCOE) 7054 mode |= T4_FILTER_FCoE; 7055 7056 return (mode); 7057 } 7058 7059 static uint32_t 7060 mode_to_fconf(uint32_t mode) 7061 { 7062 uint32_t fconf = 0; 7063 7064 if (mode & T4_FILTER_IP_FRAGMENT) 7065 fconf |= F_FRAGMENTATION; 7066 7067 if (mode & T4_FILTER_MPS_HIT_TYPE) 7068 fconf |= F_MPSHITTYPE; 7069 7070 if (mode & T4_FILTER_MAC_IDX) 7071 fconf |= F_MACMATCH; 7072 7073 if (mode & T4_FILTER_ETH_TYPE) 7074 fconf |= F_ETHERTYPE; 7075 7076 if (mode & T4_FILTER_IP_PROTO) 7077 fconf |= F_PROTOCOL; 7078 7079 if (mode & T4_FILTER_IP_TOS) 7080 fconf |= F_TOS; 7081 7082 if (mode & T4_FILTER_VLAN) 7083 fconf |= F_VLAN; 7084 7085 if (mode & T4_FILTER_VNIC) 7086 fconf |= F_VNIC_ID; 7087 7088 if (mode & T4_FILTER_PORT) 7089 fconf |= F_PORT; 7090 7091 if (mode & T4_FILTER_FCoE) 7092 fconf |= F_FCOE; 7093 7094 return (fconf); 7095 } 7096 7097 static uint32_t 7098 fspec_to_fconf(struct t4_filter_specification *fs) 7099 { 7100 uint32_t fconf = 0; 7101 7102 if (fs->val.frag || fs->mask.frag) 7103 fconf |= F_FRAGMENTATION; 7104 7105 if (fs->val.matchtype || fs->mask.matchtype) 7106 fconf |= F_MPSHITTYPE; 7107 7108 if (fs->val.macidx || fs->mask.macidx) 7109 fconf |= F_MACMATCH; 7110 7111 if (fs->val.ethtype || fs->mask.ethtype) 7112 fconf |= F_ETHERTYPE; 7113 7114 if (fs->val.proto || fs->mask.proto) 7115 fconf |= F_PROTOCOL; 7116 7117 if (fs->val.tos || fs->mask.tos) 7118 fconf |= F_TOS; 7119 7120 if (fs->val.vlan_vld || fs->mask.vlan_vld) 7121 fconf |= F_VLAN; 7122 7123 if (fs->val.vnic_vld || fs->mask.vnic_vld) 7124 fconf |= F_VNIC_ID; 7125 7126 if (fs->val.iport || fs->mask.iport) 7127 fconf |= F_PORT; 7128 7129 if (fs->val.fcoe || fs->mask.fcoe) 7130 fconf |= F_FCOE; 7131 7132 return (fconf); 7133 } 7134 7135 static int 7136 get_filter_mode(struct adapter *sc, uint32_t *mode) 7137 { 7138 int rc; 7139 uint32_t fconf; 7140 7141 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7142 "t4getfm"); 7143 if (rc) 7144 return (rc); 7145 7146 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 7147 A_TP_VLAN_PRI_MAP); 7148 7149 if (sc->params.tp.vlan_pri_map != fconf) { 7150 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 7151 device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map, 7152 fconf); 7153 sc->params.tp.vlan_pri_map = fconf; 7154 } 7155 7156 *mode = fconf_to_mode(sc->params.tp.vlan_pri_map); 7157 7158 end_synchronized_op(sc, LOCK_HELD); 7159 return (0); 7160 } 7161 7162 static int 7163 set_filter_mode(struct adapter *sc, uint32_t mode) 7164 { 7165 uint32_t fconf; 7166 int rc; 7167 7168 fconf = mode_to_fconf(mode); 7169 7170 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7171 "t4setfm"); 7172 if (rc) 7173 return (rc); 7174 7175 if (sc->tids.ftids_in_use > 0) { 7176 rc = EBUSY; 7177 goto done; 7178 } 7179 7180 #ifdef TCP_OFFLOAD 7181 if (sc->offload_map) { 7182 rc = EBUSY; 7183 goto done; 7184 } 7185 #endif 7186 7187 #ifdef notyet 7188 rc = -t4_set_filter_mode(sc, fconf); 7189 if (rc == 0) 7190 sc->filter_mode = fconf; 7191 #else 7192 rc = ENOTSUP; 7193 #endif 7194 7195 done: 7196 end_synchronized_op(sc, LOCK_HELD); 7197 return (rc); 7198 } 7199 7200 static inline uint64_t 7201 get_filter_hits(struct adapter *sc, uint32_t fid) 7202 { 7203 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 7204 uint64_t hits; 7205 7206 memwin_info(sc, 0, &mw_base, NULL); 7207 off = position_memwin(sc, 0, 7208 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 7209 if (is_t4(sc)) { 7210 hits = t4_read_reg64(sc, mw_base + off + 16); 7211 hits = be64toh(hits); 7212 } else { 7213 hits = t4_read_reg(sc, mw_base + off + 24); 7214 hits = be32toh(hits); 7215 } 7216 7217 return (hits); 7218 } 7219 7220 static int 7221 get_filter(struct adapter *sc, struct t4_filter *t) 7222 { 7223 int i, rc, nfilters = sc->tids.nftids; 7224 struct filter_entry *f; 7225 7226 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7227 "t4getf"); 7228 if (rc) 7229 return (rc); 7230 7231 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 7232 t->idx >= nfilters) { 7233 t->idx = 0xffffffff; 7234 goto done; 7235 } 7236 7237 f = &sc->tids.ftid_tab[t->idx]; 7238 for (i = t->idx; i < nfilters; i++, f++) { 7239 if (f->valid) { 7240 t->idx = i; 7241 t->l2tidx = f->l2t ? f->l2t->idx : 0; 7242 t->smtidx = f->smtidx; 7243 if (f->fs.hitcnts) 7244 t->hits = get_filter_hits(sc, t->idx); 7245 else 7246 t->hits = UINT64_MAX; 7247 t->fs = f->fs; 7248 7249 goto done; 7250 } 7251 } 7252 7253 t->idx = 0xffffffff; 7254 done: 7255 end_synchronized_op(sc, LOCK_HELD); 7256 return (0); 7257 } 7258 7259 static int 7260 set_filter(struct adapter *sc, struct t4_filter *t) 7261 { 7262 unsigned int nfilters, nports; 7263 struct filter_entry *f; 7264 int i, rc; 7265 7266 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 7267 if (rc) 7268 return (rc); 7269 7270 nfilters = sc->tids.nftids; 7271 nports = sc->params.nports; 7272 7273 if (nfilters == 0) { 7274 rc = ENOTSUP; 7275 goto done; 7276 } 7277 7278 if (!(sc->flags & FULL_INIT_DONE)) { 7279 rc = EAGAIN; 7280 goto done; 7281 } 7282 7283 if (t->idx >= nfilters) { 7284 rc = EINVAL; 7285 goto done; 7286 } 7287 7288 /* Validate against the global filter mode */ 7289 if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) != 7290 sc->params.tp.vlan_pri_map) { 7291 rc = E2BIG; 7292 goto done; 7293 } 7294 7295 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 7296 rc = EINVAL; 7297 goto done; 7298 } 7299 7300 if (t->fs.val.iport >= nports) { 7301 rc = EINVAL; 7302 goto done; 7303 } 7304 7305 /* Can't specify an iq if not steering to it */ 7306 if (!t->fs.dirsteer && t->fs.iq) { 7307 rc = EINVAL; 7308 goto done; 7309 } 7310 7311 /* IPv6 filter idx must be 4 aligned */ 7312 if (t->fs.type == 1 && 7313 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 7314 rc = EINVAL; 7315 goto done; 7316 } 7317 7318 if (sc->tids.ftid_tab == NULL) { 7319 KASSERT(sc->tids.ftids_in_use == 0, 7320 ("%s: no memory allocated but filters_in_use > 0", 7321 __func__)); 7322 7323 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 7324 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 7325 if (sc->tids.ftid_tab == NULL) { 7326 rc = ENOMEM; 7327 goto done; 7328 } 7329 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 7330 } 7331 7332 for (i = 0; i < 4; i++) { 7333 f = &sc->tids.ftid_tab[t->idx + i]; 7334 7335 if (f->pending || f->valid) { 7336 rc = EBUSY; 7337 goto done; 7338 } 7339 if (f->locked) { 7340 rc = EPERM; 7341 goto done; 7342 } 7343 7344 if (t->fs.type == 0) 7345 break; 7346 } 7347 7348 f = &sc->tids.ftid_tab[t->idx]; 7349 f->fs = t->fs; 7350 7351 rc = set_filter_wr(sc, t->idx); 7352 done: 7353 end_synchronized_op(sc, 0); 7354 7355 if (rc == 0) { 7356 mtx_lock(&sc->tids.ftid_lock); 7357 for (;;) { 7358 if (f->pending == 0) { 7359 rc = f->valid ? 0 : EIO; 7360 break; 7361 } 7362 7363 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7364 PCATCH, "t4setfw", 0)) { 7365 rc = EINPROGRESS; 7366 break; 7367 } 7368 } 7369 mtx_unlock(&sc->tids.ftid_lock); 7370 } 7371 return (rc); 7372 } 7373 7374 static int 7375 del_filter(struct adapter *sc, struct t4_filter *t) 7376 { 7377 unsigned int nfilters; 7378 struct filter_entry *f; 7379 int rc; 7380 7381 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 7382 if (rc) 7383 return (rc); 7384 7385 nfilters = sc->tids.nftids; 7386 7387 if (nfilters == 0) { 7388 rc = ENOTSUP; 7389 goto done; 7390 } 7391 7392 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 7393 t->idx >= nfilters) { 7394 rc = EINVAL; 7395 goto done; 7396 } 7397 7398 if (!(sc->flags & FULL_INIT_DONE)) { 7399 rc = EAGAIN; 7400 goto done; 7401 } 7402 7403 f = &sc->tids.ftid_tab[t->idx]; 7404 7405 if (f->pending) { 7406 rc = EBUSY; 7407 goto done; 7408 } 7409 if (f->locked) { 7410 rc = EPERM; 7411 goto done; 7412 } 7413 7414 if (f->valid) { 7415 t->fs = f->fs; /* extra info for the caller */ 7416 rc = del_filter_wr(sc, t->idx); 7417 } 7418 7419 done: 7420 end_synchronized_op(sc, 0); 7421 7422 if (rc == 0) { 7423 mtx_lock(&sc->tids.ftid_lock); 7424 for (;;) { 7425 if (f->pending == 0) { 7426 rc = f->valid ? EIO : 0; 7427 break; 7428 } 7429 7430 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7431 PCATCH, "t4delfw", 0)) { 7432 rc = EINPROGRESS; 7433 break; 7434 } 7435 } 7436 mtx_unlock(&sc->tids.ftid_lock); 7437 } 7438 7439 return (rc); 7440 } 7441 7442 static void 7443 clear_filter(struct filter_entry *f) 7444 { 7445 if (f->l2t) 7446 t4_l2t_release(f->l2t); 7447 7448 bzero(f, sizeof (*f)); 7449 } 7450 7451 static int 7452 set_filter_wr(struct adapter *sc, int fidx) 7453 { 7454 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7455 struct wrqe *wr; 7456 struct fw_filter_wr *fwr; 7457 unsigned int ftid; 7458 7459 ASSERT_SYNCHRONIZED_OP(sc); 7460 7461 if (f->fs.newdmac || f->fs.newvlan) { 7462 /* This filter needs an L2T entry; allocate one. */ 7463 f->l2t = t4_l2t_alloc_switching(sc->l2t); 7464 if (f->l2t == NULL) 7465 return (EAGAIN); 7466 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 7467 f->fs.dmac)) { 7468 t4_l2t_release(f->l2t); 7469 f->l2t = NULL; 7470 return (ENOMEM); 7471 } 7472 } 7473 7474 ftid = sc->tids.ftid_base + fidx; 7475 7476 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 7477 if (wr == NULL) 7478 return (ENOMEM); 7479 7480 fwr = wrtod(wr); 7481 bzero(fwr, sizeof (*fwr)); 7482 7483 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 7484 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 7485 fwr->tid_to_iq = 7486 htobe32(V_FW_FILTER_WR_TID(ftid) | 7487 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 7488 V_FW_FILTER_WR_NOREPLY(0) | 7489 V_FW_FILTER_WR_IQ(f->fs.iq)); 7490 fwr->del_filter_to_l2tix = 7491 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 7492 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 7493 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 7494 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 7495 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 7496 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 7497 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 7498 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 7499 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 7500 f->fs.newvlan == VLAN_REWRITE) | 7501 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 7502 f->fs.newvlan == VLAN_REWRITE) | 7503 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 7504 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 7505 V_FW_FILTER_WR_PRIO(f->fs.prio) | 7506 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 7507 fwr->ethtype = htobe16(f->fs.val.ethtype); 7508 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 7509 fwr->frag_to_ovlan_vldm = 7510 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 7511 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 7512 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 7513 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 7514 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 7515 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 7516 fwr->smac_sel = 0; 7517 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 7518 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 7519 fwr->maci_to_matchtypem = 7520 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 7521 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 7522 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 7523 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 7524 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 7525 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 7526 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 7527 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 7528 fwr->ptcl = f->fs.val.proto; 7529 fwr->ptclm = f->fs.mask.proto; 7530 fwr->ttyp = f->fs.val.tos; 7531 fwr->ttypm = f->fs.mask.tos; 7532 fwr->ivlan = htobe16(f->fs.val.vlan); 7533 fwr->ivlanm = htobe16(f->fs.mask.vlan); 7534 fwr->ovlan = htobe16(f->fs.val.vnic); 7535 fwr->ovlanm = htobe16(f->fs.mask.vnic); 7536 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 7537 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 7538 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 7539 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 7540 fwr->lp = htobe16(f->fs.val.dport); 7541 fwr->lpm = htobe16(f->fs.mask.dport); 7542 fwr->fp = htobe16(f->fs.val.sport); 7543 fwr->fpm = htobe16(f->fs.mask.sport); 7544 if (f->fs.newsmac) 7545 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 7546 7547 f->pending = 1; 7548 sc->tids.ftids_in_use++; 7549 7550 t4_wrq_tx(sc, wr); 7551 return (0); 7552 } 7553 7554 static int 7555 del_filter_wr(struct adapter *sc, int fidx) 7556 { 7557 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7558 struct wrqe *wr; 7559 struct fw_filter_wr *fwr; 7560 unsigned int ftid; 7561 7562 ftid = sc->tids.ftid_base + fidx; 7563 7564 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 7565 if (wr == NULL) 7566 return (ENOMEM); 7567 fwr = wrtod(wr); 7568 bzero(fwr, sizeof (*fwr)); 7569 7570 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 7571 7572 f->pending = 1; 7573 t4_wrq_tx(sc, wr); 7574 return (0); 7575 } 7576 7577 int 7578 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 7579 { 7580 struct adapter *sc = iq->adapter; 7581 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 7582 unsigned int idx = GET_TID(rpl); 7583 unsigned int rc; 7584 struct filter_entry *f; 7585 7586 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 7587 rss->opcode)); 7588 7589 if (is_ftid(sc, idx)) { 7590 7591 idx -= sc->tids.ftid_base; 7592 f = &sc->tids.ftid_tab[idx]; 7593 rc = G_COOKIE(rpl->cookie); 7594 7595 mtx_lock(&sc->tids.ftid_lock); 7596 if (rc == FW_FILTER_WR_FLT_ADDED) { 7597 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 7598 __func__, idx)); 7599 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 7600 f->pending = 0; /* asynchronous setup completed */ 7601 f->valid = 1; 7602 } else { 7603 if (rc != FW_FILTER_WR_FLT_DELETED) { 7604 /* Add or delete failed, display an error */ 7605 log(LOG_ERR, 7606 "filter %u setup failed with error %u\n", 7607 idx, rc); 7608 } 7609 7610 clear_filter(f); 7611 sc->tids.ftids_in_use--; 7612 } 7613 wakeup(&sc->tids.ftid_tab); 7614 mtx_unlock(&sc->tids.ftid_lock); 7615 } 7616 7617 return (0); 7618 } 7619 7620 static int 7621 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 7622 { 7623 int rc; 7624 7625 if (cntxt->cid > M_CTXTQID) 7626 return (EINVAL); 7627 7628 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 7629 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 7630 return (EINVAL); 7631 7632 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 7633 if (rc) 7634 return (rc); 7635 7636 if (sc->flags & FW_OK) { 7637 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 7638 &cntxt->data[0]); 7639 if (rc == 0) 7640 goto done; 7641 } 7642 7643 /* 7644 * Read via firmware failed or wasn't even attempted. Read directly via 7645 * the backdoor. 7646 */ 7647 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 7648 done: 7649 end_synchronized_op(sc, 0); 7650 return (rc); 7651 } 7652 7653 static int 7654 load_fw(struct adapter *sc, struct t4_data *fw) 7655 { 7656 int rc; 7657 uint8_t *fw_data; 7658 7659 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 7660 if (rc) 7661 return (rc); 7662 7663 if (sc->flags & FULL_INIT_DONE) { 7664 rc = EBUSY; 7665 goto done; 7666 } 7667 7668 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 7669 if (fw_data == NULL) { 7670 rc = ENOMEM; 7671 goto done; 7672 } 7673 7674 rc = copyin(fw->data, fw_data, fw->len); 7675 if (rc == 0) 7676 rc = -t4_load_fw(sc, fw_data, fw->len); 7677 7678 free(fw_data, M_CXGBE); 7679 done: 7680 end_synchronized_op(sc, 0); 7681 return (rc); 7682 } 7683 7684 static int 7685 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 7686 { 7687 uint32_t addr, off, remaining, i, n; 7688 uint32_t *buf, *b; 7689 uint32_t mw_base, mw_aperture; 7690 int rc; 7691 uint8_t *dst; 7692 7693 rc = validate_mem_range(sc, mr->addr, mr->len); 7694 if (rc != 0) 7695 return (rc); 7696 7697 memwin_info(sc, win, &mw_base, &mw_aperture); 7698 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 7699 addr = mr->addr; 7700 remaining = mr->len; 7701 dst = (void *)mr->data; 7702 7703 while (remaining) { 7704 off = position_memwin(sc, win, addr); 7705 7706 /* number of bytes that we'll copy in the inner loop */ 7707 n = min(remaining, mw_aperture - off); 7708 for (i = 0; i < n; i += 4) 7709 *b++ = t4_read_reg(sc, mw_base + off + i); 7710 7711 rc = copyout(buf, dst, n); 7712 if (rc != 0) 7713 break; 7714 7715 b = buf; 7716 dst += n; 7717 remaining -= n; 7718 addr += n; 7719 } 7720 7721 free(buf, M_CXGBE); 7722 return (rc); 7723 } 7724 7725 static int 7726 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 7727 { 7728 int rc; 7729 7730 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 7731 return (EINVAL); 7732 7733 if (i2cd->len > sizeof(i2cd->data)) 7734 return (EFBIG); 7735 7736 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 7737 if (rc) 7738 return (rc); 7739 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 7740 i2cd->offset, i2cd->len, &i2cd->data[0]); 7741 end_synchronized_op(sc, 0); 7742 7743 return (rc); 7744 } 7745 7746 static int 7747 in_range(int val, int lo, int hi) 7748 { 7749 7750 return (val < 0 || (val <= hi && val >= lo)); 7751 } 7752 7753 static int 7754 set_sched_class(struct adapter *sc, struct t4_sched_params *p) 7755 { 7756 int fw_subcmd, fw_type, rc; 7757 7758 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc"); 7759 if (rc) 7760 return (rc); 7761 7762 if (!(sc->flags & FULL_INIT_DONE)) { 7763 rc = EAGAIN; 7764 goto done; 7765 } 7766 7767 /* 7768 * Translate the cxgbetool parameters into T4 firmware parameters. (The 7769 * sub-command and type are in common locations.) 7770 */ 7771 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG) 7772 fw_subcmd = FW_SCHED_SC_CONFIG; 7773 else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS) 7774 fw_subcmd = FW_SCHED_SC_PARAMS; 7775 else { 7776 rc = EINVAL; 7777 goto done; 7778 } 7779 if (p->type == SCHED_CLASS_TYPE_PACKET) 7780 fw_type = FW_SCHED_TYPE_PKTSCHED; 7781 else { 7782 rc = EINVAL; 7783 goto done; 7784 } 7785 7786 if (fw_subcmd == FW_SCHED_SC_CONFIG) { 7787 /* Vet our parameters ..*/ 7788 if (p->u.config.minmax < 0) { 7789 rc = EINVAL; 7790 goto done; 7791 } 7792 7793 /* And pass the request to the firmware ...*/ 7794 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1); 7795 goto done; 7796 } 7797 7798 if (fw_subcmd == FW_SCHED_SC_PARAMS) { 7799 int fw_level; 7800 int fw_mode; 7801 int fw_rateunit; 7802 int fw_ratemode; 7803 7804 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL) 7805 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL; 7806 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) 7807 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR; 7808 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) 7809 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL; 7810 else { 7811 rc = EINVAL; 7812 goto done; 7813 } 7814 7815 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS) 7816 fw_mode = FW_SCHED_PARAMS_MODE_CLASS; 7817 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW) 7818 fw_mode = FW_SCHED_PARAMS_MODE_FLOW; 7819 else { 7820 rc = EINVAL; 7821 goto done; 7822 } 7823 7824 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS) 7825 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 7826 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS) 7827 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE; 7828 else { 7829 rc = EINVAL; 7830 goto done; 7831 } 7832 7833 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL) 7834 fw_ratemode = FW_SCHED_PARAMS_RATE_REL; 7835 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS) 7836 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS; 7837 else { 7838 rc = EINVAL; 7839 goto done; 7840 } 7841 7842 /* Vet our parameters ... */ 7843 if (!in_range(p->u.params.channel, 0, 3) || 7844 !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) || 7845 !in_range(p->u.params.minrate, 0, 10000000) || 7846 !in_range(p->u.params.maxrate, 0, 10000000) || 7847 !in_range(p->u.params.weight, 0, 100)) { 7848 rc = ERANGE; 7849 goto done; 7850 } 7851 7852 /* 7853 * Translate any unset parameters into the firmware's 7854 * nomenclature and/or fail the call if the parameters 7855 * are required ... 7856 */ 7857 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 || 7858 p->u.params.channel < 0 || p->u.params.cl < 0) { 7859 rc = EINVAL; 7860 goto done; 7861 } 7862 if (p->u.params.minrate < 0) 7863 p->u.params.minrate = 0; 7864 if (p->u.params.maxrate < 0) { 7865 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7866 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7867 rc = EINVAL; 7868 goto done; 7869 } else 7870 p->u.params.maxrate = 0; 7871 } 7872 if (p->u.params.weight < 0) { 7873 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) { 7874 rc = EINVAL; 7875 goto done; 7876 } else 7877 p->u.params.weight = 0; 7878 } 7879 if (p->u.params.pktsize < 0) { 7880 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7881 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7882 rc = EINVAL; 7883 goto done; 7884 } else 7885 p->u.params.pktsize = 0; 7886 } 7887 7888 /* See what the firmware thinks of the request ... */ 7889 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode, 7890 fw_rateunit, fw_ratemode, p->u.params.channel, 7891 p->u.params.cl, p->u.params.minrate, p->u.params.maxrate, 7892 p->u.params.weight, p->u.params.pktsize, 1); 7893 goto done; 7894 } 7895 7896 rc = EINVAL; 7897 done: 7898 end_synchronized_op(sc, 0); 7899 return (rc); 7900 } 7901 7902 static int 7903 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p) 7904 { 7905 struct port_info *pi = NULL; 7906 struct sge_txq *txq; 7907 uint32_t fw_mnem, fw_queue, fw_class; 7908 int i, rc; 7909 7910 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq"); 7911 if (rc) 7912 return (rc); 7913 7914 if (!(sc->flags & FULL_INIT_DONE)) { 7915 rc = EAGAIN; 7916 goto done; 7917 } 7918 7919 if (p->port >= sc->params.nports) { 7920 rc = EINVAL; 7921 goto done; 7922 } 7923 7924 pi = sc->port[p->port]; 7925 if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) { 7926 rc = EINVAL; 7927 goto done; 7928 } 7929 7930 /* 7931 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX 7932 * Scheduling Class in this case). 7933 */ 7934 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 7935 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH)); 7936 fw_class = p->cl < 0 ? 0xffffffff : p->cl; 7937 7938 /* 7939 * If op.queue is non-negative, then we're only changing the scheduling 7940 * on a single specified TX queue. 7941 */ 7942 if (p->queue >= 0) { 7943 txq = &sc->sge.txq[pi->first_txq + p->queue]; 7944 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7945 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7946 &fw_class); 7947 goto done; 7948 } 7949 7950 /* 7951 * Change the scheduling on all the TX queues for the 7952 * interface. 7953 */ 7954 for_each_txq(pi, i, txq) { 7955 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7956 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7957 &fw_class); 7958 if (rc) 7959 goto done; 7960 } 7961 7962 rc = 0; 7963 done: 7964 end_synchronized_op(sc, 0); 7965 return (rc); 7966 } 7967 7968 int 7969 t4_os_find_pci_capability(struct adapter *sc, int cap) 7970 { 7971 int i; 7972 7973 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 7974 } 7975 7976 int 7977 t4_os_pci_save_state(struct adapter *sc) 7978 { 7979 device_t dev; 7980 struct pci_devinfo *dinfo; 7981 7982 dev = sc->dev; 7983 dinfo = device_get_ivars(dev); 7984 7985 pci_cfg_save(dev, dinfo, 0); 7986 return (0); 7987 } 7988 7989 int 7990 t4_os_pci_restore_state(struct adapter *sc) 7991 { 7992 device_t dev; 7993 struct pci_devinfo *dinfo; 7994 7995 dev = sc->dev; 7996 dinfo = device_get_ivars(dev); 7997 7998 pci_cfg_restore(dev, dinfo); 7999 return (0); 8000 } 8001 8002 void 8003 t4_os_portmod_changed(const struct adapter *sc, int idx) 8004 { 8005 struct port_info *pi = sc->port[idx]; 8006 static const char *mod_str[] = { 8007 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 8008 }; 8009 8010 build_medialist(pi, &pi->media); 8011 #ifdef DEV_NETMAP 8012 build_medialist(pi, &pi->nm_media); 8013 #endif 8014 8015 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 8016 if_printf(pi->ifp, "transceiver unplugged.\n"); 8017 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 8018 if_printf(pi->ifp, "unknown transceiver inserted.\n"); 8019 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 8020 if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 8021 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 8022 if_printf(pi->ifp, "%s transceiver inserted.\n", 8023 mod_str[pi->mod_type]); 8024 } else { 8025 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 8026 pi->mod_type); 8027 } 8028 } 8029 8030 void 8031 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason) 8032 { 8033 struct port_info *pi = sc->port[idx]; 8034 struct ifnet *ifp = pi->ifp; 8035 8036 if (link_stat) { 8037 pi->linkdnrc = -1; 8038 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 8039 if_link_state_change(ifp, LINK_STATE_UP); 8040 } else { 8041 if (reason >= 0) 8042 pi->linkdnrc = reason; 8043 if_link_state_change(ifp, LINK_STATE_DOWN); 8044 } 8045 } 8046 8047 void 8048 t4_iterate(void (*func)(struct adapter *, void *), void *arg) 8049 { 8050 struct adapter *sc; 8051 8052 sx_slock(&t4_list_lock); 8053 SLIST_FOREACH(sc, &t4_list, link) { 8054 /* 8055 * func should not make any assumptions about what state sc is 8056 * in - the only guarantee is that sc->sc_lock is a valid lock. 8057 */ 8058 func(sc, arg); 8059 } 8060 sx_sunlock(&t4_list_lock); 8061 } 8062 8063 static int 8064 t4_open(struct cdev *dev, int flags, int type, struct thread *td) 8065 { 8066 return (0); 8067 } 8068 8069 static int 8070 t4_close(struct cdev *dev, int flags, int type, struct thread *td) 8071 { 8072 return (0); 8073 } 8074 8075 static int 8076 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 8077 struct thread *td) 8078 { 8079 int rc; 8080 struct adapter *sc = dev->si_drv1; 8081 8082 rc = priv_check(td, PRIV_DRIVER); 8083 if (rc != 0) 8084 return (rc); 8085 8086 switch (cmd) { 8087 case CHELSIO_T4_GETREG: { 8088 struct t4_reg *edata = (struct t4_reg *)data; 8089 8090 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8091 return (EFAULT); 8092 8093 if (edata->size == 4) 8094 edata->val = t4_read_reg(sc, edata->addr); 8095 else if (edata->size == 8) 8096 edata->val = t4_read_reg64(sc, edata->addr); 8097 else 8098 return (EINVAL); 8099 8100 break; 8101 } 8102 case CHELSIO_T4_SETREG: { 8103 struct t4_reg *edata = (struct t4_reg *)data; 8104 8105 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8106 return (EFAULT); 8107 8108 if (edata->size == 4) { 8109 if (edata->val & 0xffffffff00000000) 8110 return (EINVAL); 8111 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 8112 } else if (edata->size == 8) 8113 t4_write_reg64(sc, edata->addr, edata->val); 8114 else 8115 return (EINVAL); 8116 break; 8117 } 8118 case CHELSIO_T4_REGDUMP: { 8119 struct t4_regdump *regs = (struct t4_regdump *)data; 8120 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 8121 uint8_t *buf; 8122 8123 if (regs->len < reglen) { 8124 regs->len = reglen; /* hint to the caller */ 8125 return (ENOBUFS); 8126 } 8127 8128 regs->len = reglen; 8129 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 8130 t4_get_regs(sc, regs, buf); 8131 rc = copyout(buf, regs->data, reglen); 8132 free(buf, M_CXGBE); 8133 break; 8134 } 8135 case CHELSIO_T4_GET_FILTER_MODE: 8136 rc = get_filter_mode(sc, (uint32_t *)data); 8137 break; 8138 case CHELSIO_T4_SET_FILTER_MODE: 8139 rc = set_filter_mode(sc, *(uint32_t *)data); 8140 break; 8141 case CHELSIO_T4_GET_FILTER: 8142 rc = get_filter(sc, (struct t4_filter *)data); 8143 break; 8144 case CHELSIO_T4_SET_FILTER: 8145 rc = set_filter(sc, (struct t4_filter *)data); 8146 break; 8147 case CHELSIO_T4_DEL_FILTER: 8148 rc = del_filter(sc, (struct t4_filter *)data); 8149 break; 8150 case CHELSIO_T4_GET_SGE_CONTEXT: 8151 rc = get_sge_context(sc, (struct t4_sge_context *)data); 8152 break; 8153 case CHELSIO_T4_LOAD_FW: 8154 rc = load_fw(sc, (struct t4_data *)data); 8155 break; 8156 case CHELSIO_T4_GET_MEM: 8157 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 8158 break; 8159 case CHELSIO_T4_GET_I2C: 8160 rc = read_i2c(sc, (struct t4_i2c_data *)data); 8161 break; 8162 case CHELSIO_T4_CLEAR_STATS: { 8163 int i; 8164 u_int port_id = *(uint32_t *)data; 8165 struct port_info *pi; 8166 8167 if (port_id >= sc->params.nports) 8168 return (EINVAL); 8169 pi = sc->port[port_id]; 8170 8171 /* MAC stats */ 8172 t4_clr_port_stats(sc, pi->tx_chan); 8173 8174 if (pi->flags & PORT_INIT_DONE) { 8175 struct sge_rxq *rxq; 8176 struct sge_txq *txq; 8177 struct sge_wrq *wrq; 8178 8179 for_each_rxq(pi, i, rxq) { 8180 #if defined(INET) || defined(INET6) 8181 rxq->lro.lro_queued = 0; 8182 rxq->lro.lro_flushed = 0; 8183 #endif 8184 rxq->rxcsum = 0; 8185 rxq->vlan_extraction = 0; 8186 } 8187 8188 for_each_txq(pi, i, txq) { 8189 txq->txcsum = 0; 8190 txq->tso_wrs = 0; 8191 txq->vlan_insertion = 0; 8192 txq->imm_wrs = 0; 8193 txq->sgl_wrs = 0; 8194 txq->txpkt_wrs = 0; 8195 txq->txpkts_wrs = 0; 8196 txq->txpkts_pkts = 0; 8197 txq->br->br_drops = 0; 8198 txq->no_dmamap = 0; 8199 txq->no_desc = 0; 8200 } 8201 8202 #ifdef TCP_OFFLOAD 8203 /* nothing to clear for each ofld_rxq */ 8204 8205 for_each_ofld_txq(pi, i, wrq) { 8206 wrq->tx_wrs = 0; 8207 wrq->no_desc = 0; 8208 } 8209 #endif 8210 wrq = &sc->sge.ctrlq[pi->port_id]; 8211 wrq->tx_wrs = 0; 8212 wrq->no_desc = 0; 8213 } 8214 break; 8215 } 8216 case CHELSIO_T4_SCHED_CLASS: 8217 rc = set_sched_class(sc, (struct t4_sched_params *)data); 8218 break; 8219 case CHELSIO_T4_SCHED_QUEUE: 8220 rc = set_sched_queue(sc, (struct t4_sched_queue *)data); 8221 break; 8222 case CHELSIO_T4_GET_TRACER: 8223 rc = t4_get_tracer(sc, (struct t4_tracer *)data); 8224 break; 8225 case CHELSIO_T4_SET_TRACER: 8226 rc = t4_set_tracer(sc, (struct t4_tracer *)data); 8227 break; 8228 default: 8229 rc = EINVAL; 8230 } 8231 8232 return (rc); 8233 } 8234 8235 #ifdef TCP_OFFLOAD 8236 void 8237 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask, 8238 const unsigned int *pgsz_order) 8239 { 8240 struct port_info *pi = ifp->if_softc; 8241 struct adapter *sc = pi->adapter; 8242 8243 t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask); 8244 t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) | 8245 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) | 8246 V_HPZ3(pgsz_order[3])); 8247 } 8248 8249 static int 8250 toe_capability(struct port_info *pi, int enable) 8251 { 8252 int rc; 8253 struct adapter *sc = pi->adapter; 8254 8255 ASSERT_SYNCHRONIZED_OP(sc); 8256 8257 if (!is_offload(sc)) 8258 return (ENODEV); 8259 8260 if (enable) { 8261 if (!(sc->flags & FULL_INIT_DONE)) { 8262 rc = cxgbe_init_synchronized(pi); 8263 if (rc) 8264 return (rc); 8265 } 8266 8267 if (isset(&sc->offload_map, pi->port_id)) 8268 return (0); 8269 8270 if (!(sc->flags & TOM_INIT_DONE)) { 8271 rc = t4_activate_uld(sc, ULD_TOM); 8272 if (rc == EAGAIN) { 8273 log(LOG_WARNING, 8274 "You must kldload t4_tom.ko before trying " 8275 "to enable TOE on a cxgbe interface.\n"); 8276 } 8277 if (rc != 0) 8278 return (rc); 8279 KASSERT(sc->tom_softc != NULL, 8280 ("%s: TOM activated but softc NULL", __func__)); 8281 KASSERT(sc->flags & TOM_INIT_DONE, 8282 ("%s: TOM activated but flag not set", __func__)); 8283 } 8284 8285 setbit(&sc->offload_map, pi->port_id); 8286 } else { 8287 if (!isset(&sc->offload_map, pi->port_id)) 8288 return (0); 8289 8290 KASSERT(sc->flags & TOM_INIT_DONE, 8291 ("%s: TOM never initialized?", __func__)); 8292 clrbit(&sc->offload_map, pi->port_id); 8293 } 8294 8295 return (0); 8296 } 8297 8298 /* 8299 * Add an upper layer driver to the global list. 8300 */ 8301 int 8302 t4_register_uld(struct uld_info *ui) 8303 { 8304 int rc = 0; 8305 struct uld_info *u; 8306 8307 sx_xlock(&t4_uld_list_lock); 8308 SLIST_FOREACH(u, &t4_uld_list, link) { 8309 if (u->uld_id == ui->uld_id) { 8310 rc = EEXIST; 8311 goto done; 8312 } 8313 } 8314 8315 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 8316 ui->refcount = 0; 8317 done: 8318 sx_xunlock(&t4_uld_list_lock); 8319 return (rc); 8320 } 8321 8322 int 8323 t4_unregister_uld(struct uld_info *ui) 8324 { 8325 int rc = EINVAL; 8326 struct uld_info *u; 8327 8328 sx_xlock(&t4_uld_list_lock); 8329 8330 SLIST_FOREACH(u, &t4_uld_list, link) { 8331 if (u == ui) { 8332 if (ui->refcount > 0) { 8333 rc = EBUSY; 8334 goto done; 8335 } 8336 8337 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 8338 rc = 0; 8339 goto done; 8340 } 8341 } 8342 done: 8343 sx_xunlock(&t4_uld_list_lock); 8344 return (rc); 8345 } 8346 8347 int 8348 t4_activate_uld(struct adapter *sc, int id) 8349 { 8350 int rc = EAGAIN; 8351 struct uld_info *ui; 8352 8353 ASSERT_SYNCHRONIZED_OP(sc); 8354 8355 sx_slock(&t4_uld_list_lock); 8356 8357 SLIST_FOREACH(ui, &t4_uld_list, link) { 8358 if (ui->uld_id == id) { 8359 if (!(sc->flags & FULL_INIT_DONE)) { 8360 rc = adapter_full_init(sc); 8361 if (rc != 0) 8362 goto done; 8363 } 8364 8365 rc = ui->activate(sc); 8366 if (rc == 0) 8367 ui->refcount++; 8368 goto done; 8369 } 8370 } 8371 done: 8372 sx_sunlock(&t4_uld_list_lock); 8373 8374 return (rc); 8375 } 8376 8377 int 8378 t4_deactivate_uld(struct adapter *sc, int id) 8379 { 8380 int rc = EINVAL; 8381 struct uld_info *ui; 8382 8383 ASSERT_SYNCHRONIZED_OP(sc); 8384 8385 sx_slock(&t4_uld_list_lock); 8386 8387 SLIST_FOREACH(ui, &t4_uld_list, link) { 8388 if (ui->uld_id == id) { 8389 rc = ui->deactivate(sc); 8390 if (rc == 0) 8391 ui->refcount--; 8392 goto done; 8393 } 8394 } 8395 done: 8396 sx_sunlock(&t4_uld_list_lock); 8397 8398 return (rc); 8399 } 8400 #endif 8401 8402 /* 8403 * Come up with reasonable defaults for some of the tunables, provided they're 8404 * not set by the user (in which case we'll use the values as is). 8405 */ 8406 static void 8407 tweak_tunables(void) 8408 { 8409 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 8410 8411 if (t4_ntxq10g < 1) 8412 t4_ntxq10g = min(nc, NTXQ_10G); 8413 8414 if (t4_ntxq1g < 1) 8415 t4_ntxq1g = min(nc, NTXQ_1G); 8416 8417 if (t4_nrxq10g < 1) 8418 t4_nrxq10g = min(nc, NRXQ_10G); 8419 8420 if (t4_nrxq1g < 1) 8421 t4_nrxq1g = min(nc, NRXQ_1G); 8422 8423 #ifdef TCP_OFFLOAD 8424 if (t4_nofldtxq10g < 1) 8425 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 8426 8427 if (t4_nofldtxq1g < 1) 8428 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 8429 8430 if (t4_nofldrxq10g < 1) 8431 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 8432 8433 if (t4_nofldrxq1g < 1) 8434 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 8435 8436 if (t4_toecaps_allowed == -1) 8437 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 8438 #else 8439 if (t4_toecaps_allowed == -1) 8440 t4_toecaps_allowed = 0; 8441 #endif 8442 8443 #ifdef DEV_NETMAP 8444 if (t4_nnmtxq10g < 1) 8445 t4_nnmtxq10g = min(nc, NNMTXQ_10G); 8446 8447 if (t4_nnmtxq1g < 1) 8448 t4_nnmtxq1g = min(nc, NNMTXQ_1G); 8449 8450 if (t4_nnmrxq10g < 1) 8451 t4_nnmrxq10g = min(nc, NNMRXQ_10G); 8452 8453 if (t4_nnmrxq1g < 1) 8454 t4_nnmrxq1g = min(nc, NNMRXQ_1G); 8455 #endif 8456 8457 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 8458 t4_tmr_idx_10g = TMR_IDX_10G; 8459 8460 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 8461 t4_pktc_idx_10g = PKTC_IDX_10G; 8462 8463 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 8464 t4_tmr_idx_1g = TMR_IDX_1G; 8465 8466 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 8467 t4_pktc_idx_1g = PKTC_IDX_1G; 8468 8469 if (t4_qsize_txq < 128) 8470 t4_qsize_txq = 128; 8471 8472 if (t4_qsize_rxq < 128) 8473 t4_qsize_rxq = 128; 8474 while (t4_qsize_rxq & 7) 8475 t4_qsize_rxq++; 8476 8477 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 8478 } 8479 8480 static struct sx mlu; /* mod load unload */ 8481 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload"); 8482 8483 static int 8484 mod_event(module_t mod, int cmd, void *arg) 8485 { 8486 int rc = 0; 8487 static int loaded = 0; 8488 8489 switch (cmd) { 8490 case MOD_LOAD: 8491 sx_xlock(&mlu); 8492 if (loaded++ == 0) { 8493 t4_sge_modload(); 8494 sx_init(&t4_list_lock, "T4/T5 adapters"); 8495 SLIST_INIT(&t4_list); 8496 #ifdef TCP_OFFLOAD 8497 sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); 8498 SLIST_INIT(&t4_uld_list); 8499 #endif 8500 t4_tracer_modload(); 8501 tweak_tunables(); 8502 } 8503 sx_xunlock(&mlu); 8504 break; 8505 8506 case MOD_UNLOAD: 8507 sx_xlock(&mlu); 8508 if (--loaded == 0) { 8509 int tries; 8510 8511 sx_slock(&t4_list_lock); 8512 if (!SLIST_EMPTY(&t4_list)) { 8513 rc = EBUSY; 8514 sx_sunlock(&t4_list_lock); 8515 goto done_unload; 8516 } 8517 #ifdef TCP_OFFLOAD 8518 sx_slock(&t4_uld_list_lock); 8519 if (!SLIST_EMPTY(&t4_uld_list)) { 8520 rc = EBUSY; 8521 sx_sunlock(&t4_uld_list_lock); 8522 sx_sunlock(&t4_list_lock); 8523 goto done_unload; 8524 } 8525 #endif 8526 tries = 0; 8527 while (tries++ < 5 && t4_sge_extfree_refs() != 0) { 8528 uprintf("%ju clusters with custom free routine " 8529 "still is use.\n", t4_sge_extfree_refs()); 8530 pause("t4unload", 2 * hz); 8531 } 8532 #ifdef TCP_OFFLOAD 8533 sx_sunlock(&t4_uld_list_lock); 8534 #endif 8535 sx_sunlock(&t4_list_lock); 8536 8537 if (t4_sge_extfree_refs() == 0) { 8538 t4_tracer_modunload(); 8539 #ifdef TCP_OFFLOAD 8540 sx_destroy(&t4_uld_list_lock); 8541 #endif 8542 sx_destroy(&t4_list_lock); 8543 t4_sge_modunload(); 8544 loaded = 0; 8545 } else { 8546 rc = EBUSY; 8547 loaded++; /* undo earlier decrement */ 8548 } 8549 } 8550 done_unload: 8551 sx_xunlock(&mlu); 8552 break; 8553 } 8554 8555 return (rc); 8556 } 8557 8558 static devclass_t t4_devclass, t5_devclass; 8559 static devclass_t cxgbe_devclass, cxl_devclass; 8560 8561 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 8562 MODULE_VERSION(t4nex, 1); 8563 MODULE_DEPEND(t4nex, firmware, 1, 1, 1); 8564 8565 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 8566 MODULE_VERSION(t5nex, 1); 8567 MODULE_DEPEND(t5nex, firmware, 1, 1, 1); 8568 8569 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 8570 MODULE_VERSION(cxgbe, 1); 8571 8572 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 8573 MODULE_VERSION(cxl, 1); 8574