1 /*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_inet.h" 32 #include "opt_inet6.h" 33 34 #include <sys/param.h> 35 #include <sys/conf.h> 36 #include <sys/priv.h> 37 #include <sys/kernel.h> 38 #include <sys/bus.h> 39 #include <sys/module.h> 40 #include <sys/malloc.h> 41 #include <sys/queue.h> 42 #include <sys/taskqueue.h> 43 #include <sys/pciio.h> 44 #include <dev/pci/pcireg.h> 45 #include <dev/pci/pcivar.h> 46 #include <dev/pci/pci_private.h> 47 #include <sys/firmware.h> 48 #include <sys/sbuf.h> 49 #include <sys/smp.h> 50 #include <sys/socket.h> 51 #include <sys/sockio.h> 52 #include <sys/sysctl.h> 53 #include <net/ethernet.h> 54 #include <net/if.h> 55 #include <net/if_types.h> 56 #include <net/if_dl.h> 57 #include <net/if_vlan_var.h> 58 #if defined(__i386__) || defined(__amd64__) 59 #include <vm/vm.h> 60 #include <vm/pmap.h> 61 #endif 62 63 #include "common/common.h" 64 #include "common/t4_msg.h" 65 #include "common/t4_regs.h" 66 #include "common/t4_regs_values.h" 67 #include "t4_ioctl.h" 68 #include "t4_l2t.h" 69 70 /* T4 bus driver interface */ 71 static int t4_probe(device_t); 72 static int t4_attach(device_t); 73 static int t4_detach(device_t); 74 static device_method_t t4_methods[] = { 75 DEVMETHOD(device_probe, t4_probe), 76 DEVMETHOD(device_attach, t4_attach), 77 DEVMETHOD(device_detach, t4_detach), 78 79 DEVMETHOD_END 80 }; 81 static driver_t t4_driver = { 82 "t4nex", 83 t4_methods, 84 sizeof(struct adapter) 85 }; 86 87 88 /* T4 port (cxgbe) interface */ 89 static int cxgbe_probe(device_t); 90 static int cxgbe_attach(device_t); 91 static int cxgbe_detach(device_t); 92 static device_method_t cxgbe_methods[] = { 93 DEVMETHOD(device_probe, cxgbe_probe), 94 DEVMETHOD(device_attach, cxgbe_attach), 95 DEVMETHOD(device_detach, cxgbe_detach), 96 { 0, 0 } 97 }; 98 static driver_t cxgbe_driver = { 99 "cxgbe", 100 cxgbe_methods, 101 sizeof(struct port_info) 102 }; 103 104 static d_ioctl_t t4_ioctl; 105 static d_open_t t4_open; 106 static d_close_t t4_close; 107 108 static struct cdevsw t4_cdevsw = { 109 .d_version = D_VERSION, 110 .d_flags = 0, 111 .d_open = t4_open, 112 .d_close = t4_close, 113 .d_ioctl = t4_ioctl, 114 .d_name = "t4nex", 115 }; 116 117 /* T5 bus driver interface */ 118 static int t5_probe(device_t); 119 static device_method_t t5_methods[] = { 120 DEVMETHOD(device_probe, t5_probe), 121 DEVMETHOD(device_attach, t4_attach), 122 DEVMETHOD(device_detach, t4_detach), 123 124 DEVMETHOD_END 125 }; 126 static driver_t t5_driver = { 127 "t5nex", 128 t5_methods, 129 sizeof(struct adapter) 130 }; 131 132 133 /* T5 port (cxl) interface */ 134 static driver_t cxl_driver = { 135 "cxl", 136 cxgbe_methods, 137 sizeof(struct port_info) 138 }; 139 140 static struct cdevsw t5_cdevsw = { 141 .d_version = D_VERSION, 142 .d_flags = 0, 143 .d_open = t4_open, 144 .d_close = t4_close, 145 .d_ioctl = t4_ioctl, 146 .d_name = "t5nex", 147 }; 148 149 /* ifnet + media interface */ 150 static void cxgbe_init(void *); 151 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 152 static int cxgbe_transmit(struct ifnet *, struct mbuf *); 153 static void cxgbe_qflush(struct ifnet *); 154 static uint64_t cxgbe_get_counter(struct ifnet *, ift_counter); 155 static int cxgbe_media_change(struct ifnet *); 156 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 157 158 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 159 160 /* 161 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 162 * then ADAPTER_LOCK, then t4_uld_list_lock. 163 */ 164 static struct sx t4_list_lock; 165 SLIST_HEAD(, adapter) t4_list; 166 #ifdef TCP_OFFLOAD 167 static struct sx t4_uld_list_lock; 168 SLIST_HEAD(, uld_info) t4_uld_list; 169 #endif 170 171 /* 172 * Tunables. See tweak_tunables() too. 173 * 174 * Each tunable is set to a default value here if it's known at compile-time. 175 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 176 * provide a reasonable default when the driver is loaded. 177 * 178 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 179 * T5 are under hw.cxl. 180 */ 181 182 /* 183 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 184 */ 185 #define NTXQ_10G 16 186 static int t4_ntxq10g = -1; 187 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 188 189 #define NRXQ_10G 8 190 static int t4_nrxq10g = -1; 191 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 192 193 #define NTXQ_1G 4 194 static int t4_ntxq1g = -1; 195 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 196 197 #define NRXQ_1G 2 198 static int t4_nrxq1g = -1; 199 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 200 201 static int t4_rsrv_noflowq = 0; 202 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq); 203 204 #ifdef TCP_OFFLOAD 205 #define NOFLDTXQ_10G 8 206 static int t4_nofldtxq10g = -1; 207 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 208 209 #define NOFLDRXQ_10G 2 210 static int t4_nofldrxq10g = -1; 211 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 212 213 #define NOFLDTXQ_1G 2 214 static int t4_nofldtxq1g = -1; 215 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 216 217 #define NOFLDRXQ_1G 1 218 static int t4_nofldrxq1g = -1; 219 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 220 #endif 221 222 #ifdef DEV_NETMAP 223 #define NNMTXQ_10G 2 224 static int t4_nnmtxq10g = -1; 225 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g); 226 227 #define NNMRXQ_10G 2 228 static int t4_nnmrxq10g = -1; 229 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g); 230 231 #define NNMTXQ_1G 1 232 static int t4_nnmtxq1g = -1; 233 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g); 234 235 #define NNMRXQ_1G 1 236 static int t4_nnmrxq1g = -1; 237 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g); 238 #endif 239 240 /* 241 * Holdoff parameters for 10G and 1G ports. 242 */ 243 #define TMR_IDX_10G 1 244 static int t4_tmr_idx_10g = TMR_IDX_10G; 245 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 246 247 #define PKTC_IDX_10G (-1) 248 static int t4_pktc_idx_10g = PKTC_IDX_10G; 249 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 250 251 #define TMR_IDX_1G 1 252 static int t4_tmr_idx_1g = TMR_IDX_1G; 253 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 254 255 #define PKTC_IDX_1G (-1) 256 static int t4_pktc_idx_1g = PKTC_IDX_1G; 257 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 258 259 /* 260 * Size (# of entries) of each tx and rx queue. 261 */ 262 static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 263 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 264 265 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 266 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 267 268 /* 269 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 270 */ 271 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 272 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 273 274 /* 275 * Configuration file. 276 */ 277 #define DEFAULT_CF "default" 278 #define FLASH_CF "flash" 279 #define UWIRE_CF "uwire" 280 #define FPGA_CF "fpga" 281 static char t4_cfg_file[32] = DEFAULT_CF; 282 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 283 284 /* 285 * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively). 286 * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them. 287 * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water 288 * mark or when signalled to do so, 0 to never emit PAUSE. 289 */ 290 static int t4_pause_settings = PAUSE_TX | PAUSE_RX; 291 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings); 292 293 /* 294 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 295 * encouraged respectively). 296 */ 297 static unsigned int t4_fw_install = 1; 298 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 299 300 /* 301 * ASIC features that will be used. Disable the ones you don't want so that the 302 * chip resources aren't wasted on features that will not be used. 303 */ 304 static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 305 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 306 307 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 308 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 309 310 static int t4_toecaps_allowed = -1; 311 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 312 313 static int t4_rdmacaps_allowed = 0; 314 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 315 316 static int t4_iscsicaps_allowed = 0; 317 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 318 319 static int t4_fcoecaps_allowed = 0; 320 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 321 322 static int t5_write_combine = 0; 323 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 324 325 struct intrs_and_queues { 326 uint16_t intr_type; /* INTx, MSI, or MSI-X */ 327 uint16_t nirq; /* Total # of vectors */ 328 uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */ 329 uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */ 330 uint16_t ntxq10g; /* # of NIC txq's for each 10G port */ 331 uint16_t nrxq10g; /* # of NIC rxq's for each 10G port */ 332 uint16_t ntxq1g; /* # of NIC txq's for each 1G port */ 333 uint16_t nrxq1g; /* # of NIC rxq's for each 1G port */ 334 uint16_t rsrv_noflowq; /* Flag whether to reserve queue 0 */ 335 #ifdef TCP_OFFLOAD 336 uint16_t nofldtxq10g; /* # of TOE txq's for each 10G port */ 337 uint16_t nofldrxq10g; /* # of TOE rxq's for each 10G port */ 338 uint16_t nofldtxq1g; /* # of TOE txq's for each 1G port */ 339 uint16_t nofldrxq1g; /* # of TOE rxq's for each 1G port */ 340 #endif 341 #ifdef DEV_NETMAP 342 uint16_t nnmtxq10g; /* # of netmap txq's for each 10G port */ 343 uint16_t nnmrxq10g; /* # of netmap rxq's for each 10G port */ 344 uint16_t nnmtxq1g; /* # of netmap txq's for each 1G port */ 345 uint16_t nnmrxq1g; /* # of netmap rxq's for each 1G port */ 346 #endif 347 }; 348 349 struct filter_entry { 350 uint32_t valid:1; /* filter allocated and valid */ 351 uint32_t locked:1; /* filter is administratively locked */ 352 uint32_t pending:1; /* filter action is pending firmware reply */ 353 uint32_t smtidx:8; /* Source MAC Table index for smac */ 354 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 355 356 struct t4_filter_specification fs; 357 }; 358 359 static int map_bars_0_and_4(struct adapter *); 360 static int map_bar_2(struct adapter *); 361 static void setup_memwin(struct adapter *); 362 static int validate_mem_range(struct adapter *, uint32_t, int); 363 static int fwmtype_to_hwmtype(int); 364 static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 365 uint32_t *); 366 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 367 static uint32_t position_memwin(struct adapter *, int, uint32_t); 368 static int cfg_itype_and_nqueues(struct adapter *, int, int, 369 struct intrs_and_queues *); 370 static int prep_firmware(struct adapter *); 371 static int partition_resources(struct adapter *, const struct firmware *, 372 const char *); 373 static int get_params__pre_init(struct adapter *); 374 static int get_params__post_init(struct adapter *); 375 static int set_params__post_init(struct adapter *); 376 static void t4_set_desc(struct adapter *); 377 static void build_medialist(struct port_info *, struct ifmedia *); 378 static int cxgbe_init_synchronized(struct port_info *); 379 static int cxgbe_uninit_synchronized(struct port_info *); 380 static int setup_intr_handlers(struct adapter *); 381 static void quiesce_eq(struct adapter *, struct sge_eq *); 382 static void quiesce_iq(struct adapter *, struct sge_iq *); 383 static void quiesce_fl(struct adapter *, struct sge_fl *); 384 static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 385 driver_intr_t *, void *, char *); 386 static int t4_free_irq(struct adapter *, struct irq *); 387 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 388 unsigned int); 389 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 390 static void cxgbe_refresh_stats(struct adapter *, struct port_info *); 391 static void cxgbe_tick(void *); 392 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 393 static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 394 struct mbuf *); 395 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 396 static int fw_msg_not_handled(struct adapter *, const __be64 *); 397 static int t4_sysctls(struct adapter *); 398 static int cxgbe_sysctls(struct port_info *); 399 static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 400 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 401 static int sysctl_btphy(SYSCTL_HANDLER_ARGS); 402 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); 403 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 404 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 405 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 406 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 407 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS); 408 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 409 static int sysctl_temperature(SYSCTL_HANDLER_ARGS); 410 #ifdef SBUF_DRAIN 411 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 412 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 413 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 414 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); 415 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); 416 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 417 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 418 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 419 static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 420 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 421 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 422 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 423 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); 424 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 425 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); 426 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 427 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 428 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 429 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 430 static int sysctl_tids(SYSCTL_HANDLER_ARGS); 431 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 432 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 433 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 434 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 435 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 436 #endif 437 static inline void txq_start(struct ifnet *, struct sge_txq *); 438 static uint32_t fconf_to_mode(uint32_t); 439 static uint32_t mode_to_fconf(uint32_t); 440 static uint32_t fspec_to_fconf(struct t4_filter_specification *); 441 static int get_filter_mode(struct adapter *, uint32_t *); 442 static int set_filter_mode(struct adapter *, uint32_t); 443 static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 444 static int get_filter(struct adapter *, struct t4_filter *); 445 static int set_filter(struct adapter *, struct t4_filter *); 446 static int del_filter(struct adapter *, struct t4_filter *); 447 static void clear_filter(struct filter_entry *); 448 static int set_filter_wr(struct adapter *, int); 449 static int del_filter_wr(struct adapter *, int); 450 static int get_sge_context(struct adapter *, struct t4_sge_context *); 451 static int load_fw(struct adapter *, struct t4_data *); 452 static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 453 static int read_i2c(struct adapter *, struct t4_i2c_data *); 454 static int set_sched_class(struct adapter *, struct t4_sched_params *); 455 static int set_sched_queue(struct adapter *, struct t4_sched_queue *); 456 #ifdef TCP_OFFLOAD 457 static int toe_capability(struct port_info *, int); 458 #endif 459 static int mod_event(module_t, int, void *); 460 461 struct { 462 uint16_t device; 463 char *desc; 464 } t4_pciids[] = { 465 {0xa000, "Chelsio Terminator 4 FPGA"}, 466 {0x4400, "Chelsio T440-dbg"}, 467 {0x4401, "Chelsio T420-CR"}, 468 {0x4402, "Chelsio T422-CR"}, 469 {0x4403, "Chelsio T440-CR"}, 470 {0x4404, "Chelsio T420-BCH"}, 471 {0x4405, "Chelsio T440-BCH"}, 472 {0x4406, "Chelsio T440-CH"}, 473 {0x4407, "Chelsio T420-SO"}, 474 {0x4408, "Chelsio T420-CX"}, 475 {0x4409, "Chelsio T420-BT"}, 476 {0x440a, "Chelsio T404-BT"}, 477 {0x440e, "Chelsio T440-LP-CR"}, 478 }, t5_pciids[] = { 479 {0xb000, "Chelsio Terminator 5 FPGA"}, 480 {0x5400, "Chelsio T580-dbg"}, 481 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ 482 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ 483 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ 484 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ 485 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ 486 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ 487 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ 488 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ 489 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 490 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ 491 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ 492 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ 493 {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */ 494 #ifdef notyet 495 {0x5404, "Chelsio T520-BCH"}, 496 {0x5405, "Chelsio T540-BCH"}, 497 {0x5406, "Chelsio T540-CH"}, 498 {0x5408, "Chelsio T520-CX"}, 499 {0x540b, "Chelsio B520-SR"}, 500 {0x540c, "Chelsio B504-BT"}, 501 {0x540f, "Chelsio Amsterdam"}, 502 {0x5413, "Chelsio T580-CHR"}, 503 #endif 504 }; 505 506 #ifdef TCP_OFFLOAD 507 /* 508 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 509 * exactly the same for both rxq and ofld_rxq. 510 */ 511 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 512 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 513 #endif 514 515 /* No easy way to include t4_msg.h before adapter.h so we check this way */ 516 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 517 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 518 519 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); 520 521 static int 522 t4_probe(device_t dev) 523 { 524 int i; 525 uint16_t v = pci_get_vendor(dev); 526 uint16_t d = pci_get_device(dev); 527 uint8_t f = pci_get_function(dev); 528 529 if (v != PCI_VENDOR_ID_CHELSIO) 530 return (ENXIO); 531 532 /* Attach only to PF0 of the FPGA */ 533 if (d == 0xa000 && f != 0) 534 return (ENXIO); 535 536 for (i = 0; i < nitems(t4_pciids); i++) { 537 if (d == t4_pciids[i].device) { 538 device_set_desc(dev, t4_pciids[i].desc); 539 return (BUS_PROBE_DEFAULT); 540 } 541 } 542 543 return (ENXIO); 544 } 545 546 static int 547 t5_probe(device_t dev) 548 { 549 int i; 550 uint16_t v = pci_get_vendor(dev); 551 uint16_t d = pci_get_device(dev); 552 uint8_t f = pci_get_function(dev); 553 554 if (v != PCI_VENDOR_ID_CHELSIO) 555 return (ENXIO); 556 557 /* Attach only to PF0 of the FPGA */ 558 if (d == 0xb000 && f != 0) 559 return (ENXIO); 560 561 for (i = 0; i < nitems(t5_pciids); i++) { 562 if (d == t5_pciids[i].device) { 563 device_set_desc(dev, t5_pciids[i].desc); 564 return (BUS_PROBE_DEFAULT); 565 } 566 } 567 568 return (ENXIO); 569 } 570 571 static int 572 t4_attach(device_t dev) 573 { 574 struct adapter *sc; 575 int rc = 0, i, n10g, n1g, rqidx, tqidx; 576 struct intrs_and_queues iaq; 577 struct sge *s; 578 #ifdef TCP_OFFLOAD 579 int ofld_rqidx, ofld_tqidx; 580 #endif 581 #ifdef DEV_NETMAP 582 int nm_rqidx, nm_tqidx; 583 #endif 584 const char *pcie_ts; 585 586 sc = device_get_softc(dev); 587 sc->dev = dev; 588 589 pci_enable_busmaster(dev); 590 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 591 uint32_t v; 592 593 pci_set_max_read_req(dev, 4096); 594 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 595 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 596 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 597 } 598 599 sc->traceq = -1; 600 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); 601 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", 602 device_get_nameunit(dev)); 603 604 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 605 device_get_nameunit(dev)); 606 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 607 sx_xlock(&t4_list_lock); 608 SLIST_INSERT_HEAD(&t4_list, sc, link); 609 sx_xunlock(&t4_list_lock); 610 611 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 612 TAILQ_INIT(&sc->sfl); 613 callout_init(&sc->sfl_callout, CALLOUT_MPSAFE); 614 615 mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF); 616 617 rc = map_bars_0_and_4(sc); 618 if (rc != 0) 619 goto done; /* error message displayed already */ 620 621 /* 622 * This is the real PF# to which we're attaching. Works from within PCI 623 * passthrough environments too, where pci_get_function() could return a 624 * different PF# depending on the passthrough configuration. We need to 625 * use the real PF# in all our communication with the firmware. 626 */ 627 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 628 sc->mbox = sc->pf; 629 630 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 631 sc->an_handler = an_not_handled; 632 for (i = 0; i < nitems(sc->cpl_handler); i++) 633 sc->cpl_handler[i] = cpl_not_handled; 634 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 635 sc->fw_msg_handler[i] = fw_msg_not_handled; 636 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 637 t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt); 638 t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt); 639 t4_init_sge_cpl_handlers(sc); 640 641 /* Prepare the adapter for operation */ 642 rc = -t4_prep_adapter(sc); 643 if (rc != 0) { 644 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 645 goto done; 646 } 647 648 /* 649 * Do this really early, with the memory windows set up even before the 650 * character device. The userland tool's register i/o and mem read 651 * will work even in "recovery mode". 652 */ 653 setup_memwin(sc); 654 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 655 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 656 device_get_nameunit(dev)); 657 if (sc->cdev == NULL) 658 device_printf(dev, "failed to create nexus char device.\n"); 659 else 660 sc->cdev->si_drv1 = sc; 661 662 /* Go no further if recovery mode has been requested. */ 663 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 664 device_printf(dev, "recovery mode.\n"); 665 goto done; 666 } 667 668 /* Prepare the firmware for operation */ 669 rc = prep_firmware(sc); 670 if (rc != 0) 671 goto done; /* error message displayed already */ 672 673 rc = get_params__post_init(sc); 674 if (rc != 0) 675 goto done; /* error message displayed already */ 676 677 rc = set_params__post_init(sc); 678 if (rc != 0) 679 goto done; /* error message displayed already */ 680 681 rc = map_bar_2(sc); 682 if (rc != 0) 683 goto done; /* error message displayed already */ 684 685 rc = t4_create_dma_tag(sc); 686 if (rc != 0) 687 goto done; /* error message displayed already */ 688 689 /* 690 * First pass over all the ports - allocate VIs and initialize some 691 * basic parameters like mac address, port type, etc. We also figure 692 * out whether a port is 10G or 1G and use that information when 693 * calculating how many interrupts to attempt to allocate. 694 */ 695 n10g = n1g = 0; 696 for_each_port(sc, i) { 697 struct port_info *pi; 698 699 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 700 sc->port[i] = pi; 701 702 /* These must be set before t4_port_init */ 703 pi->adapter = sc; 704 pi->port_id = i; 705 706 /* Allocate the vi and initialize parameters like mac addr */ 707 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 708 if (rc != 0) { 709 device_printf(dev, "unable to initialize port %d: %d\n", 710 i, rc); 711 free(pi, M_CXGBE); 712 sc->port[i] = NULL; 713 goto done; 714 } 715 716 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX); 717 pi->link_cfg.requested_fc |= t4_pause_settings; 718 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX); 719 pi->link_cfg.fc |= t4_pause_settings; 720 721 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 722 if (rc != 0) { 723 device_printf(dev, "port %d l1cfg failed: %d\n", i, rc); 724 free(pi, M_CXGBE); 725 sc->port[i] = NULL; 726 goto done; 727 } 728 729 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 730 device_get_nameunit(dev), i); 731 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 732 sc->chan_map[pi->tx_chan] = i; 733 734 if (is_10G_port(pi) || is_40G_port(pi)) { 735 n10g++; 736 pi->tmr_idx = t4_tmr_idx_10g; 737 pi->pktc_idx = t4_pktc_idx_10g; 738 } else { 739 n1g++; 740 pi->tmr_idx = t4_tmr_idx_1g; 741 pi->pktc_idx = t4_pktc_idx_1g; 742 } 743 744 pi->xact_addr_filt = -1; 745 pi->linkdnrc = -1; 746 747 pi->qsize_rxq = t4_qsize_rxq; 748 pi->qsize_txq = t4_qsize_txq; 749 750 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 751 if (pi->dev == NULL) { 752 device_printf(dev, 753 "failed to add device for port %d.\n", i); 754 rc = ENXIO; 755 goto done; 756 } 757 device_set_softc(pi->dev, pi); 758 } 759 760 /* 761 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 762 */ 763 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 764 if (rc != 0) 765 goto done; /* error message displayed already */ 766 767 sc->intr_type = iaq.intr_type; 768 sc->intr_count = iaq.nirq; 769 770 s = &sc->sge; 771 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 772 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 773 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 774 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 775 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 776 #ifdef TCP_OFFLOAD 777 if (is_offload(sc)) { 778 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 779 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 780 s->neq += s->nofldtxq + s->nofldrxq; 781 s->niq += s->nofldrxq; 782 783 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 784 M_CXGBE, M_ZERO | M_WAITOK); 785 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 786 M_CXGBE, M_ZERO | M_WAITOK); 787 } 788 #endif 789 #ifdef DEV_NETMAP 790 s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g; 791 s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g; 792 s->neq += s->nnmtxq + s->nnmrxq; 793 s->niq += s->nnmrxq; 794 795 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq), 796 M_CXGBE, M_ZERO | M_WAITOK); 797 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq), 798 M_CXGBE, M_ZERO | M_WAITOK); 799 #endif 800 801 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 802 M_ZERO | M_WAITOK); 803 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 804 M_ZERO | M_WAITOK); 805 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 806 M_ZERO | M_WAITOK); 807 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 808 M_ZERO | M_WAITOK); 809 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 810 M_ZERO | M_WAITOK); 811 812 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 813 M_ZERO | M_WAITOK); 814 815 t4_init_l2t(sc, M_WAITOK); 816 817 /* 818 * Second pass over the ports. This time we know the number of rx and 819 * tx queues that each port should get. 820 */ 821 rqidx = tqidx = 0; 822 #ifdef TCP_OFFLOAD 823 ofld_rqidx = ofld_tqidx = 0; 824 #endif 825 #ifdef DEV_NETMAP 826 nm_rqidx = nm_tqidx = 0; 827 #endif 828 for_each_port(sc, i) { 829 struct port_info *pi = sc->port[i]; 830 831 if (pi == NULL) 832 continue; 833 834 pi->first_rxq = rqidx; 835 pi->first_txq = tqidx; 836 if (is_10G_port(pi) || is_40G_port(pi)) { 837 pi->flags |= iaq.intr_flags_10g; 838 pi->nrxq = iaq.nrxq10g; 839 pi->ntxq = iaq.ntxq10g; 840 } else { 841 pi->flags |= iaq.intr_flags_1g; 842 pi->nrxq = iaq.nrxq1g; 843 pi->ntxq = iaq.ntxq1g; 844 } 845 846 if (pi->ntxq > 1) 847 pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0; 848 else 849 pi->rsrv_noflowq = 0; 850 851 rqidx += pi->nrxq; 852 tqidx += pi->ntxq; 853 #ifdef TCP_OFFLOAD 854 if (is_offload(sc)) { 855 pi->first_ofld_rxq = ofld_rqidx; 856 pi->first_ofld_txq = ofld_tqidx; 857 if (is_10G_port(pi) || is_40G_port(pi)) { 858 pi->nofldrxq = iaq.nofldrxq10g; 859 pi->nofldtxq = iaq.nofldtxq10g; 860 } else { 861 pi->nofldrxq = iaq.nofldrxq1g; 862 pi->nofldtxq = iaq.nofldtxq1g; 863 } 864 ofld_rqidx += pi->nofldrxq; 865 ofld_tqidx += pi->nofldtxq; 866 } 867 #endif 868 #ifdef DEV_NETMAP 869 pi->first_nm_rxq = nm_rqidx; 870 pi->first_nm_txq = nm_tqidx; 871 if (is_10G_port(pi) || is_40G_port(pi)) { 872 pi->nnmrxq = iaq.nnmrxq10g; 873 pi->nnmtxq = iaq.nnmtxq10g; 874 } else { 875 pi->nnmrxq = iaq.nnmrxq1g; 876 pi->nnmtxq = iaq.nnmtxq1g; 877 } 878 nm_rqidx += pi->nnmrxq; 879 nm_tqidx += pi->nnmtxq; 880 #endif 881 } 882 883 rc = setup_intr_handlers(sc); 884 if (rc != 0) { 885 device_printf(dev, 886 "failed to setup interrupt handlers: %d\n", rc); 887 goto done; 888 } 889 890 rc = bus_generic_attach(dev); 891 if (rc != 0) { 892 device_printf(dev, 893 "failed to attach all child ports: %d\n", rc); 894 goto done; 895 } 896 897 switch (sc->params.pci.speed) { 898 case 0x1: 899 pcie_ts = "2.5"; 900 break; 901 case 0x2: 902 pcie_ts = "5.0"; 903 break; 904 case 0x3: 905 pcie_ts = "8.0"; 906 break; 907 default: 908 pcie_ts = "??"; 909 break; 910 } 911 device_printf(dev, 912 "PCIe x%d (%s GTS/s) (%d), %d ports, %d %s interrupt%s, %d eq, %d iq\n", 913 sc->params.pci.width, pcie_ts, sc->params.pci.speed, 914 sc->params.nports, sc->intr_count, 915 sc->intr_type == INTR_MSIX ? "MSI-X" : 916 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 917 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 918 919 t4_set_desc(sc); 920 921 done: 922 if (rc != 0 && sc->cdev) { 923 /* cdev was created and so cxgbetool works; recover that way. */ 924 device_printf(dev, 925 "error during attach, adapter is now in recovery mode.\n"); 926 rc = 0; 927 } 928 929 if (rc != 0) 930 t4_detach(dev); 931 else 932 t4_sysctls(sc); 933 934 return (rc); 935 } 936 937 /* 938 * Idempotent 939 */ 940 static int 941 t4_detach(device_t dev) 942 { 943 struct adapter *sc; 944 struct port_info *pi; 945 int i, rc; 946 947 sc = device_get_softc(dev); 948 949 if (sc->flags & FULL_INIT_DONE) 950 t4_intr_disable(sc); 951 952 if (sc->cdev) { 953 destroy_dev(sc->cdev); 954 sc->cdev = NULL; 955 } 956 957 rc = bus_generic_detach(dev); 958 if (rc) { 959 device_printf(dev, 960 "failed to detach child devices: %d\n", rc); 961 return (rc); 962 } 963 964 for (i = 0; i < sc->intr_count; i++) 965 t4_free_irq(sc, &sc->irq[i]); 966 967 for (i = 0; i < MAX_NPORTS; i++) { 968 pi = sc->port[i]; 969 if (pi) { 970 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid); 971 if (pi->dev) 972 device_delete_child(dev, pi->dev); 973 974 mtx_destroy(&pi->pi_lock); 975 free(pi, M_CXGBE); 976 } 977 } 978 979 if (sc->flags & FULL_INIT_DONE) 980 adapter_full_uninit(sc); 981 982 if (sc->flags & FW_OK) 983 t4_fw_bye(sc, sc->mbox); 984 985 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 986 pci_release_msi(dev); 987 988 if (sc->regs_res) 989 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 990 sc->regs_res); 991 992 if (sc->udbs_res) 993 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 994 sc->udbs_res); 995 996 if (sc->msix_res) 997 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 998 sc->msix_res); 999 1000 if (sc->l2t) 1001 t4_free_l2t(sc->l2t); 1002 1003 #ifdef TCP_OFFLOAD 1004 free(sc->sge.ofld_rxq, M_CXGBE); 1005 free(sc->sge.ofld_txq, M_CXGBE); 1006 #endif 1007 #ifdef DEV_NETMAP 1008 free(sc->sge.nm_rxq, M_CXGBE); 1009 free(sc->sge.nm_txq, M_CXGBE); 1010 #endif 1011 free(sc->irq, M_CXGBE); 1012 free(sc->sge.rxq, M_CXGBE); 1013 free(sc->sge.txq, M_CXGBE); 1014 free(sc->sge.ctrlq, M_CXGBE); 1015 free(sc->sge.iqmap, M_CXGBE); 1016 free(sc->sge.eqmap, M_CXGBE); 1017 free(sc->tids.ftid_tab, M_CXGBE); 1018 t4_destroy_dma_tag(sc); 1019 if (mtx_initialized(&sc->sc_lock)) { 1020 sx_xlock(&t4_list_lock); 1021 SLIST_REMOVE(&t4_list, sc, adapter, link); 1022 sx_xunlock(&t4_list_lock); 1023 mtx_destroy(&sc->sc_lock); 1024 } 1025 1026 if (mtx_initialized(&sc->tids.ftid_lock)) 1027 mtx_destroy(&sc->tids.ftid_lock); 1028 if (mtx_initialized(&sc->sfl_lock)) 1029 mtx_destroy(&sc->sfl_lock); 1030 if (mtx_initialized(&sc->ifp_lock)) 1031 mtx_destroy(&sc->ifp_lock); 1032 if (mtx_initialized(&sc->regwin_lock)) 1033 mtx_destroy(&sc->regwin_lock); 1034 1035 bzero(sc, sizeof(*sc)); 1036 1037 return (0); 1038 } 1039 1040 static int 1041 cxgbe_probe(device_t dev) 1042 { 1043 char buf[128]; 1044 struct port_info *pi = device_get_softc(dev); 1045 1046 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 1047 device_set_desc_copy(dev, buf); 1048 1049 return (BUS_PROBE_DEFAULT); 1050 } 1051 1052 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 1053 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 1054 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS) 1055 #define T4_CAP_ENABLE (T4_CAP) 1056 1057 static int 1058 cxgbe_attach(device_t dev) 1059 { 1060 struct port_info *pi = device_get_softc(dev); 1061 struct ifnet *ifp; 1062 char *s; 1063 int n, o; 1064 1065 /* Allocate an ifnet and set it up */ 1066 ifp = if_alloc(IFT_ETHER); 1067 if (ifp == NULL) { 1068 device_printf(dev, "Cannot allocate ifnet\n"); 1069 return (ENOMEM); 1070 } 1071 pi->ifp = ifp; 1072 ifp->if_softc = pi; 1073 1074 callout_init(&pi->tick, CALLOUT_MPSAFE); 1075 1076 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1077 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1078 1079 ifp->if_init = cxgbe_init; 1080 ifp->if_ioctl = cxgbe_ioctl; 1081 ifp->if_transmit = cxgbe_transmit; 1082 ifp->if_qflush = cxgbe_qflush; 1083 ifp->if_get_counter = cxgbe_get_counter; 1084 1085 ifp->if_capabilities = T4_CAP; 1086 #ifdef TCP_OFFLOAD 1087 if (is_offload(pi->adapter)) 1088 ifp->if_capabilities |= IFCAP_TOE; 1089 #endif 1090 ifp->if_capenable = T4_CAP_ENABLE; 1091 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1092 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1093 1094 ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1095 ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS; 1096 ifp->if_hw_tsomaxsegsize = 65536; 1097 1098 /* Initialize ifmedia for this port */ 1099 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 1100 cxgbe_media_status); 1101 build_medialist(pi, &pi->media); 1102 1103 pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 1104 EVENTHANDLER_PRI_ANY); 1105 1106 ether_ifattach(ifp, pi->hw_addr); 1107 1108 n = 128; 1109 s = malloc(n, M_CXGBE, M_WAITOK); 1110 o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq); 1111 MPASS(n > o); 1112 #ifdef TCP_OFFLOAD 1113 if (is_offload(pi->adapter)) { 1114 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)", 1115 pi->nofldtxq, pi->nofldrxq); 1116 MPASS(n > o); 1117 } 1118 #endif 1119 #ifdef DEV_NETMAP 1120 o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq, 1121 pi->nnmrxq); 1122 MPASS(n > o); 1123 #endif 1124 device_printf(dev, "%s\n", s); 1125 free(s, M_CXGBE); 1126 1127 #ifdef DEV_NETMAP 1128 /* nm_media handled here to keep implementation private to this file */ 1129 ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change, 1130 cxgbe_media_status); 1131 build_medialist(pi, &pi->nm_media); 1132 create_netmap_ifnet(pi); /* logs errors it something fails */ 1133 #endif 1134 cxgbe_sysctls(pi); 1135 1136 return (0); 1137 } 1138 1139 static int 1140 cxgbe_detach(device_t dev) 1141 { 1142 struct port_info *pi = device_get_softc(dev); 1143 struct adapter *sc = pi->adapter; 1144 struct ifnet *ifp = pi->ifp; 1145 1146 /* Tell if_ioctl and if_init that the port is going away */ 1147 ADAPTER_LOCK(sc); 1148 SET_DOOMED(pi); 1149 wakeup(&sc->flags); 1150 while (IS_BUSY(sc)) 1151 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 1152 SET_BUSY(sc); 1153 #ifdef INVARIANTS 1154 sc->last_op = "t4detach"; 1155 sc->last_op_thr = curthread; 1156 #endif 1157 ADAPTER_UNLOCK(sc); 1158 1159 if (pi->flags & HAS_TRACEQ) { 1160 sc->traceq = -1; /* cloner should not create ifnet */ 1161 t4_tracer_port_detach(sc); 1162 } 1163 1164 if (pi->vlan_c) 1165 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c); 1166 1167 PORT_LOCK(pi); 1168 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1169 callout_stop(&pi->tick); 1170 PORT_UNLOCK(pi); 1171 callout_drain(&pi->tick); 1172 1173 /* Let detach proceed even if these fail. */ 1174 cxgbe_uninit_synchronized(pi); 1175 port_full_uninit(pi); 1176 1177 ifmedia_removeall(&pi->media); 1178 ether_ifdetach(pi->ifp); 1179 if_free(pi->ifp); 1180 1181 #ifdef DEV_NETMAP 1182 /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */ 1183 destroy_netmap_ifnet(pi); 1184 #endif 1185 1186 ADAPTER_LOCK(sc); 1187 CLR_BUSY(sc); 1188 wakeup(&sc->flags); 1189 ADAPTER_UNLOCK(sc); 1190 1191 return (0); 1192 } 1193 1194 static void 1195 cxgbe_init(void *arg) 1196 { 1197 struct port_info *pi = arg; 1198 struct adapter *sc = pi->adapter; 1199 1200 if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0) 1201 return; 1202 cxgbe_init_synchronized(pi); 1203 end_synchronized_op(sc, 0); 1204 } 1205 1206 static int 1207 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1208 { 1209 int rc = 0, mtu, flags, can_sleep; 1210 struct port_info *pi = ifp->if_softc; 1211 struct adapter *sc = pi->adapter; 1212 struct ifreq *ifr = (struct ifreq *)data; 1213 uint32_t mask; 1214 1215 switch (cmd) { 1216 case SIOCSIFMTU: 1217 mtu = ifr->ifr_mtu; 1218 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1219 return (EINVAL); 1220 1221 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu"); 1222 if (rc) 1223 return (rc); 1224 ifp->if_mtu = mtu; 1225 if (pi->flags & PORT_INIT_DONE) { 1226 t4_update_fl_bufsize(ifp); 1227 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1228 rc = update_mac_settings(ifp, XGMAC_MTU); 1229 } 1230 end_synchronized_op(sc, 0); 1231 break; 1232 1233 case SIOCSIFFLAGS: 1234 can_sleep = 0; 1235 redo_sifflags: 1236 rc = begin_synchronized_op(sc, pi, 1237 can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg"); 1238 if (rc) 1239 return (rc); 1240 1241 if (ifp->if_flags & IFF_UP) { 1242 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1243 flags = pi->if_flags; 1244 if ((ifp->if_flags ^ flags) & 1245 (IFF_PROMISC | IFF_ALLMULTI)) { 1246 if (can_sleep == 1) { 1247 end_synchronized_op(sc, 0); 1248 can_sleep = 0; 1249 goto redo_sifflags; 1250 } 1251 rc = update_mac_settings(ifp, 1252 XGMAC_PROMISC | XGMAC_ALLMULTI); 1253 } 1254 } else { 1255 if (can_sleep == 0) { 1256 end_synchronized_op(sc, LOCK_HELD); 1257 can_sleep = 1; 1258 goto redo_sifflags; 1259 } 1260 rc = cxgbe_init_synchronized(pi); 1261 } 1262 pi->if_flags = ifp->if_flags; 1263 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1264 if (can_sleep == 0) { 1265 end_synchronized_op(sc, LOCK_HELD); 1266 can_sleep = 1; 1267 goto redo_sifflags; 1268 } 1269 rc = cxgbe_uninit_synchronized(pi); 1270 } 1271 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD); 1272 break; 1273 1274 case SIOCADDMULTI: 1275 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1276 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi"); 1277 if (rc) 1278 return (rc); 1279 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1280 rc = update_mac_settings(ifp, XGMAC_MCADDRS); 1281 end_synchronized_op(sc, LOCK_HELD); 1282 break; 1283 1284 case SIOCSIFCAP: 1285 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap"); 1286 if (rc) 1287 return (rc); 1288 1289 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1290 if (mask & IFCAP_TXCSUM) { 1291 ifp->if_capenable ^= IFCAP_TXCSUM; 1292 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1293 1294 if (IFCAP_TSO4 & ifp->if_capenable && 1295 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1296 ifp->if_capenable &= ~IFCAP_TSO4; 1297 if_printf(ifp, 1298 "tso4 disabled due to -txcsum.\n"); 1299 } 1300 } 1301 if (mask & IFCAP_TXCSUM_IPV6) { 1302 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1303 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1304 1305 if (IFCAP_TSO6 & ifp->if_capenable && 1306 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1307 ifp->if_capenable &= ~IFCAP_TSO6; 1308 if_printf(ifp, 1309 "tso6 disabled due to -txcsum6.\n"); 1310 } 1311 } 1312 if (mask & IFCAP_RXCSUM) 1313 ifp->if_capenable ^= IFCAP_RXCSUM; 1314 if (mask & IFCAP_RXCSUM_IPV6) 1315 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1316 1317 /* 1318 * Note that we leave CSUM_TSO alone (it is always set). The 1319 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1320 * sending a TSO request our way, so it's sufficient to toggle 1321 * IFCAP_TSOx only. 1322 */ 1323 if (mask & IFCAP_TSO4) { 1324 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1325 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1326 if_printf(ifp, "enable txcsum first.\n"); 1327 rc = EAGAIN; 1328 goto fail; 1329 } 1330 ifp->if_capenable ^= IFCAP_TSO4; 1331 } 1332 if (mask & IFCAP_TSO6) { 1333 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1334 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1335 if_printf(ifp, "enable txcsum6 first.\n"); 1336 rc = EAGAIN; 1337 goto fail; 1338 } 1339 ifp->if_capenable ^= IFCAP_TSO6; 1340 } 1341 if (mask & IFCAP_LRO) { 1342 #if defined(INET) || defined(INET6) 1343 int i; 1344 struct sge_rxq *rxq; 1345 1346 ifp->if_capenable ^= IFCAP_LRO; 1347 for_each_rxq(pi, i, rxq) { 1348 if (ifp->if_capenable & IFCAP_LRO) 1349 rxq->iq.flags |= IQ_LRO_ENABLED; 1350 else 1351 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1352 } 1353 #endif 1354 } 1355 #ifdef TCP_OFFLOAD 1356 if (mask & IFCAP_TOE) { 1357 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1358 1359 rc = toe_capability(pi, enable); 1360 if (rc != 0) 1361 goto fail; 1362 1363 ifp->if_capenable ^= mask; 1364 } 1365 #endif 1366 if (mask & IFCAP_VLAN_HWTAGGING) { 1367 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1368 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1369 rc = update_mac_settings(ifp, XGMAC_VLANEX); 1370 } 1371 if (mask & IFCAP_VLAN_MTU) { 1372 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1373 1374 /* Need to find out how to disable auto-mtu-inflation */ 1375 } 1376 if (mask & IFCAP_VLAN_HWTSO) 1377 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1378 if (mask & IFCAP_VLAN_HWCSUM) 1379 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1380 1381 #ifdef VLAN_CAPABILITIES 1382 VLAN_CAPABILITIES(ifp); 1383 #endif 1384 fail: 1385 end_synchronized_op(sc, 0); 1386 break; 1387 1388 case SIOCSIFMEDIA: 1389 case SIOCGIFMEDIA: 1390 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1391 break; 1392 1393 case SIOCGI2C: { 1394 struct ifi2creq i2c; 1395 1396 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); 1397 if (rc != 0) 1398 break; 1399 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) { 1400 rc = EPERM; 1401 break; 1402 } 1403 if (i2c.len > sizeof(i2c.data)) { 1404 rc = EINVAL; 1405 break; 1406 } 1407 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4i2c"); 1408 if (rc) 1409 return (rc); 1410 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr, 1411 i2c.offset, i2c.len, &i2c.data[0]); 1412 end_synchronized_op(sc, 0); 1413 if (rc == 0) 1414 rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); 1415 break; 1416 } 1417 1418 default: 1419 rc = ether_ioctl(ifp, cmd, data); 1420 } 1421 1422 return (rc); 1423 } 1424 1425 static int 1426 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1427 { 1428 struct port_info *pi = ifp->if_softc; 1429 struct adapter *sc = pi->adapter; 1430 struct sge_txq *txq = &sc->sge.txq[pi->first_txq]; 1431 struct buf_ring *br; 1432 int rc; 1433 1434 M_ASSERTPKTHDR(m); 1435 1436 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1437 m_freem(m); 1438 return (ENETDOWN); 1439 } 1440 1441 if (m->m_flags & M_FLOWID) 1442 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) 1443 + pi->rsrv_noflowq); 1444 br = txq->br; 1445 1446 if (TXQ_TRYLOCK(txq) == 0) { 1447 struct sge_eq *eq = &txq->eq; 1448 1449 /* 1450 * It is possible that t4_eth_tx finishes up and releases the 1451 * lock between the TRYLOCK above and the drbr_enqueue here. We 1452 * need to make sure that this mbuf doesn't just sit there in 1453 * the drbr. 1454 */ 1455 1456 rc = drbr_enqueue(ifp, br, m); 1457 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 && 1458 !(eq->flags & EQ_DOOMED)) 1459 callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq); 1460 return (rc); 1461 } 1462 1463 /* 1464 * txq->m is the mbuf that is held up due to a temporary shortage of 1465 * resources and it should be put on the wire first. Then what's in 1466 * drbr and finally the mbuf that was just passed in to us. 1467 * 1468 * Return code should indicate the fate of the mbuf that was passed in 1469 * this time. 1470 */ 1471 1472 TXQ_LOCK_ASSERT_OWNED(txq); 1473 if (drbr_needs_enqueue(ifp, br) || txq->m) { 1474 1475 /* Queued for transmission. */ 1476 1477 rc = drbr_enqueue(ifp, br, m); 1478 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 1479 (void) t4_eth_tx(ifp, txq, m); 1480 TXQ_UNLOCK(txq); 1481 return (rc); 1482 } 1483 1484 /* Direct transmission. */ 1485 rc = t4_eth_tx(ifp, txq, m); 1486 if (rc != 0 && txq->m) 1487 rc = 0; /* held, will be transmitted soon (hopefully) */ 1488 1489 TXQ_UNLOCK(txq); 1490 return (rc); 1491 } 1492 1493 static void 1494 cxgbe_qflush(struct ifnet *ifp) 1495 { 1496 struct port_info *pi = ifp->if_softc; 1497 struct sge_txq *txq; 1498 int i; 1499 struct mbuf *m; 1500 1501 /* queues do not exist if !PORT_INIT_DONE. */ 1502 if (pi->flags & PORT_INIT_DONE) { 1503 for_each_txq(pi, i, txq) { 1504 TXQ_LOCK(txq); 1505 m_freem(txq->m); 1506 txq->m = NULL; 1507 while ((m = buf_ring_dequeue_sc(txq->br)) != NULL) 1508 m_freem(m); 1509 TXQ_UNLOCK(txq); 1510 } 1511 } 1512 if_qflush(ifp); 1513 } 1514 1515 static uint64_t 1516 cxgbe_get_counter(struct ifnet *ifp, ift_counter c) 1517 { 1518 struct port_info *pi = ifp->if_softc; 1519 struct adapter *sc = pi->adapter; 1520 struct port_stats *s = &pi->stats; 1521 1522 cxgbe_refresh_stats(sc, pi); 1523 1524 switch (c) { 1525 case IFCOUNTER_IPACKETS: 1526 return (s->rx_frames - s->rx_pause); 1527 1528 case IFCOUNTER_IERRORS: 1529 return (s->rx_jabber + s->rx_runt + s->rx_too_long + 1530 s->rx_fcs_err + s->rx_len_err); 1531 1532 case IFCOUNTER_OPACKETS: 1533 return (s->tx_frames - s->tx_pause); 1534 1535 case IFCOUNTER_OERRORS: 1536 return (s->tx_error_frames); 1537 1538 case IFCOUNTER_IBYTES: 1539 return (s->rx_octets - s->rx_pause * 64); 1540 1541 case IFCOUNTER_OBYTES: 1542 return (s->tx_octets - s->tx_pause * 64); 1543 1544 case IFCOUNTER_IMCASTS: 1545 return (s->rx_mcast_frames - s->rx_pause); 1546 1547 case IFCOUNTER_OMCASTS: 1548 return (s->tx_mcast_frames - s->tx_pause); 1549 1550 case IFCOUNTER_IQDROPS: 1551 return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 1552 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 1553 s->rx_trunc3 + pi->tnl_cong_drops); 1554 1555 case IFCOUNTER_OQDROPS: { 1556 uint64_t drops; 1557 1558 drops = s->tx_drop; 1559 if (pi->flags & PORT_INIT_DONE) { 1560 int i; 1561 struct sge_txq *txq; 1562 1563 for_each_txq(pi, i, txq) 1564 drops += txq->br->br_drops; 1565 } 1566 1567 return (drops); 1568 1569 } 1570 1571 default: 1572 return (if_get_counter_default(ifp, c)); 1573 } 1574 } 1575 1576 static int 1577 cxgbe_media_change(struct ifnet *ifp) 1578 { 1579 struct port_info *pi = ifp->if_softc; 1580 1581 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1582 1583 return (EOPNOTSUPP); 1584 } 1585 1586 static void 1587 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1588 { 1589 struct port_info *pi = ifp->if_softc; 1590 struct ifmedia *media = NULL; 1591 struct ifmedia_entry *cur; 1592 int speed = pi->link_cfg.speed; 1593 int data = (pi->port_type << 8) | pi->mod_type; 1594 1595 if (ifp == pi->ifp) 1596 media = &pi->media; 1597 #ifdef DEV_NETMAP 1598 else if (ifp == pi->nm_ifp) 1599 media = &pi->nm_media; 1600 #endif 1601 MPASS(media != NULL); 1602 1603 cur = media->ifm_cur; 1604 if (cur->ifm_data != data) { 1605 build_medialist(pi, media); 1606 cur = media->ifm_cur; 1607 } 1608 1609 ifmr->ifm_status = IFM_AVALID; 1610 if (!pi->link_cfg.link_ok) 1611 return; 1612 1613 ifmr->ifm_status |= IFM_ACTIVE; 1614 1615 /* active and current will differ iff current media is autoselect. */ 1616 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1617 return; 1618 1619 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1620 if (speed == SPEED_10000) 1621 ifmr->ifm_active |= IFM_10G_T; 1622 else if (speed == SPEED_1000) 1623 ifmr->ifm_active |= IFM_1000_T; 1624 else if (speed == SPEED_100) 1625 ifmr->ifm_active |= IFM_100_TX; 1626 else if (speed == SPEED_10) 1627 ifmr->ifm_active |= IFM_10_T; 1628 else 1629 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1630 speed)); 1631 } 1632 1633 void 1634 t4_fatal_err(struct adapter *sc) 1635 { 1636 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1637 t4_intr_disable(sc); 1638 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1639 device_get_nameunit(sc->dev)); 1640 } 1641 1642 static int 1643 map_bars_0_and_4(struct adapter *sc) 1644 { 1645 sc->regs_rid = PCIR_BAR(0); 1646 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1647 &sc->regs_rid, RF_ACTIVE); 1648 if (sc->regs_res == NULL) { 1649 device_printf(sc->dev, "cannot map registers.\n"); 1650 return (ENXIO); 1651 } 1652 sc->bt = rman_get_bustag(sc->regs_res); 1653 sc->bh = rman_get_bushandle(sc->regs_res); 1654 sc->mmio_len = rman_get_size(sc->regs_res); 1655 setbit(&sc->doorbells, DOORBELL_KDB); 1656 1657 sc->msix_rid = PCIR_BAR(4); 1658 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1659 &sc->msix_rid, RF_ACTIVE); 1660 if (sc->msix_res == NULL) { 1661 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1662 return (ENXIO); 1663 } 1664 1665 return (0); 1666 } 1667 1668 static int 1669 map_bar_2(struct adapter *sc) 1670 { 1671 1672 /* 1673 * T4: only iWARP driver uses the userspace doorbells. There is no need 1674 * to map it if RDMA is disabled. 1675 */ 1676 if (is_t4(sc) && sc->rdmacaps == 0) 1677 return (0); 1678 1679 sc->udbs_rid = PCIR_BAR(2); 1680 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1681 &sc->udbs_rid, RF_ACTIVE); 1682 if (sc->udbs_res == NULL) { 1683 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1684 return (ENXIO); 1685 } 1686 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1687 1688 if (is_t5(sc)) { 1689 setbit(&sc->doorbells, DOORBELL_UDB); 1690 #if defined(__i386__) || defined(__amd64__) 1691 if (t5_write_combine) { 1692 int rc; 1693 1694 /* 1695 * Enable write combining on BAR2. This is the 1696 * userspace doorbell BAR and is split into 128B 1697 * (UDBS_SEG_SIZE) doorbell regions, each associated 1698 * with an egress queue. The first 64B has the doorbell 1699 * and the second 64B can be used to submit a tx work 1700 * request with an implicit doorbell. 1701 */ 1702 1703 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1704 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1705 if (rc == 0) { 1706 clrbit(&sc->doorbells, DOORBELL_UDB); 1707 setbit(&sc->doorbells, DOORBELL_WCWR); 1708 setbit(&sc->doorbells, DOORBELL_UDBWC); 1709 } else { 1710 device_printf(sc->dev, 1711 "couldn't enable write combining: %d\n", 1712 rc); 1713 } 1714 1715 t4_write_reg(sc, A_SGE_STAT_CFG, 1716 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1717 } 1718 #endif 1719 } 1720 1721 return (0); 1722 } 1723 1724 static const struct memwin t4_memwin[] = { 1725 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1726 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1727 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1728 }; 1729 1730 static const struct memwin t5_memwin[] = { 1731 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1732 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1733 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1734 }; 1735 1736 static void 1737 setup_memwin(struct adapter *sc) 1738 { 1739 const struct memwin *mw; 1740 int i, n; 1741 uint32_t bar0; 1742 1743 if (is_t4(sc)) { 1744 /* 1745 * Read low 32b of bar0 indirectly via the hardware backdoor 1746 * mechanism. Works from within PCI passthrough environments 1747 * too, where rman_get_start() can return a different value. We 1748 * need to program the T4 memory window decoders with the actual 1749 * addresses that will be coming across the PCIe link. 1750 */ 1751 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1752 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1753 1754 mw = &t4_memwin[0]; 1755 n = nitems(t4_memwin); 1756 } else { 1757 /* T5 uses the relative offset inside the PCIe BAR */ 1758 bar0 = 0; 1759 1760 mw = &t5_memwin[0]; 1761 n = nitems(t5_memwin); 1762 } 1763 1764 for (i = 0; i < n; i++, mw++) { 1765 t4_write_reg(sc, 1766 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 1767 (mw->base + bar0) | V_BIR(0) | 1768 V_WINDOW(ilog2(mw->aperture) - 10)); 1769 } 1770 1771 /* flush */ 1772 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1773 } 1774 1775 /* 1776 * Verify that the memory range specified by the addr/len pair is valid and lies 1777 * entirely within a single region (EDCx or MCx). 1778 */ 1779 static int 1780 validate_mem_range(struct adapter *sc, uint32_t addr, int len) 1781 { 1782 uint32_t em, addr_len, maddr, mlen; 1783 1784 /* Memory can only be accessed in naturally aligned 4 byte units */ 1785 if (addr & 3 || len & 3 || len == 0) 1786 return (EINVAL); 1787 1788 /* Enabled memories */ 1789 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1790 if (em & F_EDRAM0_ENABLE) { 1791 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1792 maddr = G_EDRAM0_BASE(addr_len) << 20; 1793 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1794 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1795 addr + len <= maddr + mlen) 1796 return (0); 1797 } 1798 if (em & F_EDRAM1_ENABLE) { 1799 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1800 maddr = G_EDRAM1_BASE(addr_len) << 20; 1801 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1802 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1803 addr + len <= maddr + mlen) 1804 return (0); 1805 } 1806 if (em & F_EXT_MEM_ENABLE) { 1807 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1808 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1809 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1810 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1811 addr + len <= maddr + mlen) 1812 return (0); 1813 } 1814 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 1815 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1816 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1817 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1818 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1819 addr + len <= maddr + mlen) 1820 return (0); 1821 } 1822 1823 return (EFAULT); 1824 } 1825 1826 static int 1827 fwmtype_to_hwmtype(int mtype) 1828 { 1829 1830 switch (mtype) { 1831 case FW_MEMTYPE_EDC0: 1832 return (MEM_EDC0); 1833 case FW_MEMTYPE_EDC1: 1834 return (MEM_EDC1); 1835 case FW_MEMTYPE_EXTMEM: 1836 return (MEM_MC0); 1837 case FW_MEMTYPE_EXTMEM1: 1838 return (MEM_MC1); 1839 default: 1840 panic("%s: cannot translate fw mtype %d.", __func__, mtype); 1841 } 1842 } 1843 1844 /* 1845 * Verify that the memory range specified by the memtype/offset/len pair is 1846 * valid and lies entirely within the memtype specified. The global address of 1847 * the start of the range is returned in addr. 1848 */ 1849 static int 1850 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 1851 uint32_t *addr) 1852 { 1853 uint32_t em, addr_len, maddr, mlen; 1854 1855 /* Memory can only be accessed in naturally aligned 4 byte units */ 1856 if (off & 3 || len & 3 || len == 0) 1857 return (EINVAL); 1858 1859 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1860 switch (fwmtype_to_hwmtype(mtype)) { 1861 case MEM_EDC0: 1862 if (!(em & F_EDRAM0_ENABLE)) 1863 return (EINVAL); 1864 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1865 maddr = G_EDRAM0_BASE(addr_len) << 20; 1866 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1867 break; 1868 case MEM_EDC1: 1869 if (!(em & F_EDRAM1_ENABLE)) 1870 return (EINVAL); 1871 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1872 maddr = G_EDRAM1_BASE(addr_len) << 20; 1873 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1874 break; 1875 case MEM_MC: 1876 if (!(em & F_EXT_MEM_ENABLE)) 1877 return (EINVAL); 1878 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1879 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1880 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1881 break; 1882 case MEM_MC1: 1883 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 1884 return (EINVAL); 1885 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1886 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1887 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1888 break; 1889 default: 1890 return (EINVAL); 1891 } 1892 1893 if (mlen > 0 && off < mlen && off + len <= mlen) { 1894 *addr = maddr + off; /* global address */ 1895 return (0); 1896 } 1897 1898 return (EFAULT); 1899 } 1900 1901 static void 1902 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 1903 { 1904 const struct memwin *mw; 1905 1906 if (is_t4(sc)) { 1907 KASSERT(win >= 0 && win < nitems(t4_memwin), 1908 ("%s: incorrect memwin# (%d)", __func__, win)); 1909 mw = &t4_memwin[win]; 1910 } else { 1911 KASSERT(win >= 0 && win < nitems(t5_memwin), 1912 ("%s: incorrect memwin# (%d)", __func__, win)); 1913 mw = &t5_memwin[win]; 1914 } 1915 1916 if (base != NULL) 1917 *base = mw->base; 1918 if (aperture != NULL) 1919 *aperture = mw->aperture; 1920 } 1921 1922 /* 1923 * Positions the memory window such that it can be used to access the specified 1924 * address in the chip's address space. The return value is the offset of addr 1925 * from the start of the window. 1926 */ 1927 static uint32_t 1928 position_memwin(struct adapter *sc, int n, uint32_t addr) 1929 { 1930 uint32_t start, pf; 1931 uint32_t reg; 1932 1933 KASSERT(n >= 0 && n <= 3, 1934 ("%s: invalid window %d.", __func__, n)); 1935 KASSERT((addr & 3) == 0, 1936 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 1937 1938 if (is_t4(sc)) { 1939 pf = 0; 1940 start = addr & ~0xf; /* start must be 16B aligned */ 1941 } else { 1942 pf = V_PFNUM(sc->pf); 1943 start = addr & ~0x7f; /* start must be 128B aligned */ 1944 } 1945 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 1946 1947 t4_write_reg(sc, reg, start | pf); 1948 t4_read_reg(sc, reg); 1949 1950 return (addr - start); 1951 } 1952 1953 static int 1954 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1955 struct intrs_and_queues *iaq) 1956 { 1957 int rc, itype, navail, nrxq10g, nrxq1g, n; 1958 int nofldrxq10g = 0, nofldrxq1g = 0; 1959 int nnmrxq10g = 0, nnmrxq1g = 0; 1960 1961 bzero(iaq, sizeof(*iaq)); 1962 1963 iaq->ntxq10g = t4_ntxq10g; 1964 iaq->ntxq1g = t4_ntxq1g; 1965 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1966 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1967 iaq->rsrv_noflowq = t4_rsrv_noflowq; 1968 #ifdef TCP_OFFLOAD 1969 if (is_offload(sc)) { 1970 iaq->nofldtxq10g = t4_nofldtxq10g; 1971 iaq->nofldtxq1g = t4_nofldtxq1g; 1972 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1973 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1974 } 1975 #endif 1976 #ifdef DEV_NETMAP 1977 iaq->nnmtxq10g = t4_nnmtxq10g; 1978 iaq->nnmtxq1g = t4_nnmtxq1g; 1979 iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g; 1980 iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g; 1981 #endif 1982 1983 for (itype = INTR_MSIX; itype; itype >>= 1) { 1984 1985 if ((itype & t4_intr_types) == 0) 1986 continue; /* not allowed */ 1987 1988 if (itype == INTR_MSIX) 1989 navail = pci_msix_count(sc->dev); 1990 else if (itype == INTR_MSI) 1991 navail = pci_msi_count(sc->dev); 1992 else 1993 navail = 1; 1994 restart: 1995 if (navail == 0) 1996 continue; 1997 1998 iaq->intr_type = itype; 1999 iaq->intr_flags_10g = 0; 2000 iaq->intr_flags_1g = 0; 2001 2002 /* 2003 * Best option: an interrupt vector for errors, one for the 2004 * firmware event queue, and one for every rxq (NIC, TOE, and 2005 * netmap). 2006 */ 2007 iaq->nirq = T4_EXTRA_INTR; 2008 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g); 2009 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g); 2010 if (iaq->nirq <= navail && 2011 (itype != INTR_MSI || powerof2(iaq->nirq))) { 2012 iaq->intr_flags_10g = INTR_ALL; 2013 iaq->intr_flags_1g = INTR_ALL; 2014 goto allocate; 2015 } 2016 2017 /* 2018 * Second best option: a vector for errors, one for the firmware 2019 * event queue, and vectors for either all the NIC rx queues or 2020 * all the TOE rx queues. The queues that don't get vectors 2021 * will forward their interrupts to those that do. 2022 * 2023 * Note: netmap rx queues cannot be created early and so they 2024 * can't be setup to receive forwarded interrupts for others. 2025 */ 2026 iaq->nirq = T4_EXTRA_INTR; 2027 if (nrxq10g >= nofldrxq10g) { 2028 iaq->intr_flags_10g = INTR_RXQ; 2029 iaq->nirq += n10g * nrxq10g; 2030 #ifdef DEV_NETMAP 2031 iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g); 2032 #endif 2033 } else { 2034 iaq->intr_flags_10g = INTR_OFLD_RXQ; 2035 iaq->nirq += n10g * nofldrxq10g; 2036 #ifdef DEV_NETMAP 2037 iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g); 2038 #endif 2039 } 2040 if (nrxq1g >= nofldrxq1g) { 2041 iaq->intr_flags_1g = INTR_RXQ; 2042 iaq->nirq += n1g * nrxq1g; 2043 #ifdef DEV_NETMAP 2044 iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g); 2045 #endif 2046 } else { 2047 iaq->intr_flags_1g = INTR_OFLD_RXQ; 2048 iaq->nirq += n1g * nofldrxq1g; 2049 #ifdef DEV_NETMAP 2050 iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g); 2051 #endif 2052 } 2053 if (iaq->nirq <= navail && 2054 (itype != INTR_MSI || powerof2(iaq->nirq))) 2055 goto allocate; 2056 2057 /* 2058 * Next best option: an interrupt vector for errors, one for the 2059 * firmware event queue, and at least one per port. At this 2060 * point we know we'll have to downsize nrxq and/or nofldrxq 2061 * and/or nnmrxq to fit what's available to us. 2062 */ 2063 iaq->nirq = T4_EXTRA_INTR; 2064 iaq->nirq += n10g + n1g; 2065 if (iaq->nirq <= navail) { 2066 int leftover = navail - iaq->nirq; 2067 2068 if (n10g > 0) { 2069 int target = max(nrxq10g, nofldrxq10g); 2070 2071 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ? 2072 INTR_RXQ : INTR_OFLD_RXQ; 2073 2074 n = 1; 2075 while (n < target && leftover >= n10g) { 2076 leftover -= n10g; 2077 iaq->nirq += n10g; 2078 n++; 2079 } 2080 iaq->nrxq10g = min(n, nrxq10g); 2081 #ifdef TCP_OFFLOAD 2082 iaq->nofldrxq10g = min(n, nofldrxq10g); 2083 #endif 2084 #ifdef DEV_NETMAP 2085 iaq->nnmrxq10g = min(n, nnmrxq10g); 2086 #endif 2087 } 2088 2089 if (n1g > 0) { 2090 int target = max(nrxq1g, nofldrxq1g); 2091 2092 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ? 2093 INTR_RXQ : INTR_OFLD_RXQ; 2094 2095 n = 1; 2096 while (n < target && leftover >= n1g) { 2097 leftover -= n1g; 2098 iaq->nirq += n1g; 2099 n++; 2100 } 2101 iaq->nrxq1g = min(n, nrxq1g); 2102 #ifdef TCP_OFFLOAD 2103 iaq->nofldrxq1g = min(n, nofldrxq1g); 2104 #endif 2105 #ifdef DEV_NETMAP 2106 iaq->nnmrxq1g = min(n, nnmrxq1g); 2107 #endif 2108 } 2109 2110 if (itype != INTR_MSI || powerof2(iaq->nirq)) 2111 goto allocate; 2112 } 2113 2114 /* 2115 * Least desirable option: one interrupt vector for everything. 2116 */ 2117 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 2118 iaq->intr_flags_10g = iaq->intr_flags_1g = 0; 2119 #ifdef TCP_OFFLOAD 2120 if (is_offload(sc)) 2121 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 2122 #endif 2123 #ifdef DEV_NETMAP 2124 iaq->nnmrxq10g = iaq->nnmrxq1g = 1; 2125 #endif 2126 2127 allocate: 2128 navail = iaq->nirq; 2129 rc = 0; 2130 if (itype == INTR_MSIX) 2131 rc = pci_alloc_msix(sc->dev, &navail); 2132 else if (itype == INTR_MSI) 2133 rc = pci_alloc_msi(sc->dev, &navail); 2134 2135 if (rc == 0) { 2136 if (navail == iaq->nirq) 2137 return (0); 2138 2139 /* 2140 * Didn't get the number requested. Use whatever number 2141 * the kernel is willing to allocate (it's in navail). 2142 */ 2143 device_printf(sc->dev, "fewer vectors than requested, " 2144 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 2145 itype, iaq->nirq, navail); 2146 pci_release_msi(sc->dev); 2147 goto restart; 2148 } 2149 2150 device_printf(sc->dev, 2151 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 2152 itype, rc, iaq->nirq, navail); 2153 } 2154 2155 device_printf(sc->dev, 2156 "failed to find a usable interrupt type. " 2157 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 2158 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 2159 2160 return (ENXIO); 2161 } 2162 2163 #define FW_VERSION(chip) ( \ 2164 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ 2165 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ 2166 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ 2167 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) 2168 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) 2169 2170 struct fw_info { 2171 uint8_t chip; 2172 char *kld_name; 2173 char *fw_mod_name; 2174 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 2175 } fw_info[] = { 2176 { 2177 .chip = CHELSIO_T4, 2178 .kld_name = "t4fw_cfg", 2179 .fw_mod_name = "t4fw", 2180 .fw_hdr = { 2181 .chip = FW_HDR_CHIP_T4, 2182 .fw_ver = htobe32_const(FW_VERSION(T4)), 2183 .intfver_nic = FW_INTFVER(T4, NIC), 2184 .intfver_vnic = FW_INTFVER(T4, VNIC), 2185 .intfver_ofld = FW_INTFVER(T4, OFLD), 2186 .intfver_ri = FW_INTFVER(T4, RI), 2187 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 2188 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 2189 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 2190 .intfver_fcoe = FW_INTFVER(T4, FCOE), 2191 }, 2192 }, { 2193 .chip = CHELSIO_T5, 2194 .kld_name = "t5fw_cfg", 2195 .fw_mod_name = "t5fw", 2196 .fw_hdr = { 2197 .chip = FW_HDR_CHIP_T5, 2198 .fw_ver = htobe32_const(FW_VERSION(T5)), 2199 .intfver_nic = FW_INTFVER(T5, NIC), 2200 .intfver_vnic = FW_INTFVER(T5, VNIC), 2201 .intfver_ofld = FW_INTFVER(T5, OFLD), 2202 .intfver_ri = FW_INTFVER(T5, RI), 2203 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 2204 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 2205 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 2206 .intfver_fcoe = FW_INTFVER(T5, FCOE), 2207 }, 2208 } 2209 }; 2210 2211 static struct fw_info * 2212 find_fw_info(int chip) 2213 { 2214 int i; 2215 2216 for (i = 0; i < nitems(fw_info); i++) { 2217 if (fw_info[i].chip == chip) 2218 return (&fw_info[i]); 2219 } 2220 return (NULL); 2221 } 2222 2223 /* 2224 * Is the given firmware API compatible with the one the driver was compiled 2225 * with? 2226 */ 2227 static int 2228 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 2229 { 2230 2231 /* short circuit if it's the exact same firmware version */ 2232 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 2233 return (1); 2234 2235 /* 2236 * XXX: Is this too conservative? Perhaps I should limit this to the 2237 * features that are supported in the driver. 2238 */ 2239 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 2240 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 2241 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 2242 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 2243 return (1); 2244 #undef SAME_INTF 2245 2246 return (0); 2247 } 2248 2249 /* 2250 * The firmware in the KLD is usable, but should it be installed? This routine 2251 * explains itself in detail if it indicates the KLD firmware should be 2252 * installed. 2253 */ 2254 static int 2255 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 2256 { 2257 const char *reason; 2258 2259 if (!card_fw_usable) { 2260 reason = "incompatible or unusable"; 2261 goto install; 2262 } 2263 2264 if (k > c) { 2265 reason = "older than the version bundled with this driver"; 2266 goto install; 2267 } 2268 2269 if (t4_fw_install == 2 && k != c) { 2270 reason = "different than the version bundled with this driver"; 2271 goto install; 2272 } 2273 2274 return (0); 2275 2276 install: 2277 if (t4_fw_install == 0) { 2278 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2279 "but the driver is prohibited from installing a different " 2280 "firmware on the card.\n", 2281 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2282 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 2283 2284 return (0); 2285 } 2286 2287 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2288 "installing firmware %u.%u.%u.%u on card.\n", 2289 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2290 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 2291 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2292 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2293 2294 return (1); 2295 } 2296 /* 2297 * Establish contact with the firmware and determine if we are the master driver 2298 * or not, and whether we are responsible for chip initialization. 2299 */ 2300 static int 2301 prep_firmware(struct adapter *sc) 2302 { 2303 const struct firmware *fw = NULL, *default_cfg; 2304 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 2305 enum dev_state state; 2306 struct fw_info *fw_info; 2307 struct fw_hdr *card_fw; /* fw on the card */ 2308 const struct fw_hdr *kld_fw; /* fw in the KLD */ 2309 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 2310 against */ 2311 2312 /* Contact firmware. */ 2313 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 2314 if (rc < 0 || state == DEV_STATE_ERR) { 2315 rc = -rc; 2316 device_printf(sc->dev, 2317 "failed to connect to the firmware: %d, %d.\n", rc, state); 2318 return (rc); 2319 } 2320 pf = rc; 2321 if (pf == sc->mbox) 2322 sc->flags |= MASTER_PF; 2323 else if (state == DEV_STATE_UNINIT) { 2324 /* 2325 * We didn't get to be the master so we definitely won't be 2326 * configuring the chip. It's a bug if someone else hasn't 2327 * configured it already. 2328 */ 2329 device_printf(sc->dev, "couldn't be master(%d), " 2330 "device not already initialized either(%d).\n", rc, state); 2331 return (EDOOFUS); 2332 } 2333 2334 /* This is the firmware whose headers the driver was compiled against */ 2335 fw_info = find_fw_info(chip_id(sc)); 2336 if (fw_info == NULL) { 2337 device_printf(sc->dev, 2338 "unable to look up firmware information for chip %d.\n", 2339 chip_id(sc)); 2340 return (EINVAL); 2341 } 2342 drv_fw = &fw_info->fw_hdr; 2343 2344 /* 2345 * The firmware KLD contains many modules. The KLD name is also the 2346 * name of the module that contains the default config file. 2347 */ 2348 default_cfg = firmware_get(fw_info->kld_name); 2349 2350 /* Read the header of the firmware on the card */ 2351 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 2352 rc = -t4_read_flash(sc, FLASH_FW_START, 2353 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 2354 if (rc == 0) 2355 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 2356 else { 2357 device_printf(sc->dev, 2358 "Unable to read card's firmware header: %d\n", rc); 2359 card_fw_usable = 0; 2360 } 2361 2362 /* This is the firmware in the KLD */ 2363 fw = firmware_get(fw_info->fw_mod_name); 2364 if (fw != NULL) { 2365 kld_fw = (const void *)fw->data; 2366 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 2367 } else { 2368 kld_fw = NULL; 2369 kld_fw_usable = 0; 2370 } 2371 2372 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2373 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) { 2374 /* 2375 * Common case: the firmware on the card is an exact match and 2376 * the KLD is an exact match too, or the KLD is 2377 * absent/incompatible. Note that t4_fw_install = 2 is ignored 2378 * here -- use cxgbetool loadfw if you want to reinstall the 2379 * same firmware as the one on the card. 2380 */ 2381 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2382 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2383 be32toh(card_fw->fw_ver))) { 2384 2385 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 2386 if (rc != 0) { 2387 device_printf(sc->dev, 2388 "failed to install firmware: %d\n", rc); 2389 goto done; 2390 } 2391 2392 /* Installed successfully, update the cached header too. */ 2393 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2394 card_fw_usable = 1; 2395 need_fw_reset = 0; /* already reset as part of load_fw */ 2396 } 2397 2398 if (!card_fw_usable) { 2399 uint32_t d, c, k; 2400 2401 d = ntohl(drv_fw->fw_ver); 2402 c = ntohl(card_fw->fw_ver); 2403 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2404 2405 device_printf(sc->dev, "Cannot find a usable firmware: " 2406 "fw_install %d, chip state %d, " 2407 "driver compiled with %d.%d.%d.%d, " 2408 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2409 t4_fw_install, state, 2410 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2411 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2412 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2413 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2414 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2415 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2416 rc = EINVAL; 2417 goto done; 2418 } 2419 2420 /* We're using whatever's on the card and it's known to be good. */ 2421 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2422 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2423 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2424 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2425 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2426 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2427 t4_get_tp_version(sc, &sc->params.tp_vers); 2428 2429 /* Reset device */ 2430 if (need_fw_reset && 2431 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2432 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2433 if (rc != ETIMEDOUT && rc != EIO) 2434 t4_fw_bye(sc, sc->mbox); 2435 goto done; 2436 } 2437 sc->flags |= FW_OK; 2438 2439 rc = get_params__pre_init(sc); 2440 if (rc != 0) 2441 goto done; /* error message displayed already */ 2442 2443 /* Partition adapter resources as specified in the config file. */ 2444 if (state == DEV_STATE_UNINIT) { 2445 2446 KASSERT(sc->flags & MASTER_PF, 2447 ("%s: trying to change chip settings when not master.", 2448 __func__)); 2449 2450 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2451 if (rc != 0) 2452 goto done; /* error message displayed already */ 2453 2454 t4_tweak_chip_settings(sc); 2455 2456 /* get basic stuff going */ 2457 rc = -t4_fw_initialize(sc, sc->mbox); 2458 if (rc != 0) { 2459 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2460 goto done; 2461 } 2462 } else { 2463 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2464 sc->cfcsum = 0; 2465 } 2466 2467 done: 2468 free(card_fw, M_CXGBE); 2469 if (fw != NULL) 2470 firmware_put(fw, FIRMWARE_UNLOAD); 2471 if (default_cfg != NULL) 2472 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2473 2474 return (rc); 2475 } 2476 2477 #define FW_PARAM_DEV(param) \ 2478 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2479 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2480 #define FW_PARAM_PFVF(param) \ 2481 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2482 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2483 2484 /* 2485 * Partition chip resources for use between various PFs, VFs, etc. 2486 */ 2487 static int 2488 partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2489 const char *name_prefix) 2490 { 2491 const struct firmware *cfg = NULL; 2492 int rc = 0; 2493 struct fw_caps_config_cmd caps; 2494 uint32_t mtype, moff, finicsum, cfcsum; 2495 2496 /* 2497 * Figure out what configuration file to use. Pick the default config 2498 * file for the card if the user hasn't specified one explicitly. 2499 */ 2500 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2501 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2502 /* Card specific overrides go here. */ 2503 if (pci_get_device(sc->dev) == 0x440a) 2504 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2505 if (is_fpga(sc)) 2506 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2507 } 2508 2509 /* 2510 * We need to load another module if the profile is anything except 2511 * "default" or "flash". 2512 */ 2513 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2514 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2515 char s[32]; 2516 2517 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2518 cfg = firmware_get(s); 2519 if (cfg == NULL) { 2520 if (default_cfg != NULL) { 2521 device_printf(sc->dev, 2522 "unable to load module \"%s\" for " 2523 "configuration profile \"%s\", will use " 2524 "the default config file instead.\n", 2525 s, sc->cfg_file); 2526 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2527 "%s", DEFAULT_CF); 2528 } else { 2529 device_printf(sc->dev, 2530 "unable to load module \"%s\" for " 2531 "configuration profile \"%s\", will use " 2532 "the config file on the card's flash " 2533 "instead.\n", s, sc->cfg_file); 2534 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2535 "%s", FLASH_CF); 2536 } 2537 } 2538 } 2539 2540 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2541 default_cfg == NULL) { 2542 device_printf(sc->dev, 2543 "default config file not available, will use the config " 2544 "file on the card's flash instead.\n"); 2545 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2546 } 2547 2548 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2549 u_int cflen, i, n; 2550 const uint32_t *cfdata; 2551 uint32_t param, val, addr, off, mw_base, mw_aperture; 2552 2553 KASSERT(cfg != NULL || default_cfg != NULL, 2554 ("%s: no config to upload", __func__)); 2555 2556 /* 2557 * Ask the firmware where it wants us to upload the config file. 2558 */ 2559 param = FW_PARAM_DEV(CF); 2560 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2561 if (rc != 0) { 2562 /* No support for config file? Shouldn't happen. */ 2563 device_printf(sc->dev, 2564 "failed to query config file location: %d.\n", rc); 2565 goto done; 2566 } 2567 mtype = G_FW_PARAMS_PARAM_Y(val); 2568 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2569 2570 /* 2571 * XXX: sheer laziness. We deliberately added 4 bytes of 2572 * useless stuffing/comments at the end of the config file so 2573 * it's ok to simply throw away the last remaining bytes when 2574 * the config file is not an exact multiple of 4. This also 2575 * helps with the validate_mt_off_len check. 2576 */ 2577 if (cfg != NULL) { 2578 cflen = cfg->datasize & ~3; 2579 cfdata = cfg->data; 2580 } else { 2581 cflen = default_cfg->datasize & ~3; 2582 cfdata = default_cfg->data; 2583 } 2584 2585 if (cflen > FLASH_CFG_MAX_SIZE) { 2586 device_printf(sc->dev, 2587 "config file too long (%d, max allowed is %d). " 2588 "Will try to use the config on the card, if any.\n", 2589 cflen, FLASH_CFG_MAX_SIZE); 2590 goto use_config_on_flash; 2591 } 2592 2593 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2594 if (rc != 0) { 2595 device_printf(sc->dev, 2596 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2597 "Will try to use the config on the card, if any.\n", 2598 __func__, mtype, moff, cflen, rc); 2599 goto use_config_on_flash; 2600 } 2601 2602 memwin_info(sc, 2, &mw_base, &mw_aperture); 2603 while (cflen) { 2604 off = position_memwin(sc, 2, addr); 2605 n = min(cflen, mw_aperture - off); 2606 for (i = 0; i < n; i += 4) 2607 t4_write_reg(sc, mw_base + off + i, *cfdata++); 2608 cflen -= n; 2609 addr += n; 2610 } 2611 } else { 2612 use_config_on_flash: 2613 mtype = FW_MEMTYPE_FLASH; 2614 moff = t4_flash_cfg_addr(sc); 2615 } 2616 2617 bzero(&caps, sizeof(caps)); 2618 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2619 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2620 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2621 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2622 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2623 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2624 if (rc != 0) { 2625 device_printf(sc->dev, 2626 "failed to pre-process config file: %d " 2627 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2628 goto done; 2629 } 2630 2631 finicsum = be32toh(caps.finicsum); 2632 cfcsum = be32toh(caps.cfcsum); 2633 if (finicsum != cfcsum) { 2634 device_printf(sc->dev, 2635 "WARNING: config file checksum mismatch: %08x %08x\n", 2636 finicsum, cfcsum); 2637 } 2638 sc->cfcsum = cfcsum; 2639 2640 #define LIMIT_CAPS(x) do { \ 2641 caps.x &= htobe16(t4_##x##_allowed); \ 2642 } while (0) 2643 2644 /* 2645 * Let the firmware know what features will (not) be used so it can tune 2646 * things accordingly. 2647 */ 2648 LIMIT_CAPS(linkcaps); 2649 LIMIT_CAPS(niccaps); 2650 LIMIT_CAPS(toecaps); 2651 LIMIT_CAPS(rdmacaps); 2652 LIMIT_CAPS(iscsicaps); 2653 LIMIT_CAPS(fcoecaps); 2654 #undef LIMIT_CAPS 2655 2656 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2657 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2658 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2659 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2660 if (rc != 0) { 2661 device_printf(sc->dev, 2662 "failed to process config file: %d.\n", rc); 2663 } 2664 done: 2665 if (cfg != NULL) 2666 firmware_put(cfg, FIRMWARE_UNLOAD); 2667 return (rc); 2668 } 2669 2670 /* 2671 * Retrieve parameters that are needed (or nice to have) very early. 2672 */ 2673 static int 2674 get_params__pre_init(struct adapter *sc) 2675 { 2676 int rc; 2677 uint32_t param[2], val[2]; 2678 struct fw_devlog_cmd cmd; 2679 struct devlog_params *dlog = &sc->params.devlog; 2680 2681 param[0] = FW_PARAM_DEV(PORTVEC); 2682 param[1] = FW_PARAM_DEV(CCLK); 2683 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2684 if (rc != 0) { 2685 device_printf(sc->dev, 2686 "failed to query parameters (pre_init): %d.\n", rc); 2687 return (rc); 2688 } 2689 2690 sc->params.portvec = val[0]; 2691 sc->params.nports = bitcount32(val[0]); 2692 sc->params.vpd.cclk = val[1]; 2693 2694 /* Read device log parameters. */ 2695 bzero(&cmd, sizeof(cmd)); 2696 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2697 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2698 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2699 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2700 if (rc != 0) { 2701 device_printf(sc->dev, 2702 "failed to get devlog parameters: %d.\n", rc); 2703 bzero(dlog, sizeof (*dlog)); 2704 rc = 0; /* devlog isn't critical for device operation */ 2705 } else { 2706 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2707 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2708 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2709 dlog->size = be32toh(cmd.memsize_devlog); 2710 } 2711 2712 return (rc); 2713 } 2714 2715 /* 2716 * Retrieve various parameters that are of interest to the driver. The device 2717 * has been initialized by the firmware at this point. 2718 */ 2719 static int 2720 get_params__post_init(struct adapter *sc) 2721 { 2722 int rc; 2723 uint32_t param[7], val[7]; 2724 struct fw_caps_config_cmd caps; 2725 2726 param[0] = FW_PARAM_PFVF(IQFLINT_START); 2727 param[1] = FW_PARAM_PFVF(EQ_START); 2728 param[2] = FW_PARAM_PFVF(FILTER_START); 2729 param[3] = FW_PARAM_PFVF(FILTER_END); 2730 param[4] = FW_PARAM_PFVF(L2T_START); 2731 param[5] = FW_PARAM_PFVF(L2T_END); 2732 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2733 if (rc != 0) { 2734 device_printf(sc->dev, 2735 "failed to query parameters (post_init): %d.\n", rc); 2736 return (rc); 2737 } 2738 2739 sc->sge.iq_start = val[0]; 2740 sc->sge.eq_start = val[1]; 2741 sc->tids.ftid_base = val[2]; 2742 sc->tids.nftids = val[3] - val[2] + 1; 2743 sc->params.ftid_min = val[2]; 2744 sc->params.ftid_max = val[3]; 2745 sc->vres.l2t.start = val[4]; 2746 sc->vres.l2t.size = val[5] - val[4] + 1; 2747 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2748 ("%s: L2 table size (%u) larger than expected (%u)", 2749 __func__, sc->vres.l2t.size, L2T_SIZE)); 2750 2751 /* get capabilites */ 2752 bzero(&caps, sizeof(caps)); 2753 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2754 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2755 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2756 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2757 if (rc != 0) { 2758 device_printf(sc->dev, 2759 "failed to get card capabilities: %d.\n", rc); 2760 return (rc); 2761 } 2762 2763 #define READ_CAPS(x) do { \ 2764 sc->x = htobe16(caps.x); \ 2765 } while (0) 2766 READ_CAPS(linkcaps); 2767 READ_CAPS(niccaps); 2768 READ_CAPS(toecaps); 2769 READ_CAPS(rdmacaps); 2770 READ_CAPS(iscsicaps); 2771 READ_CAPS(fcoecaps); 2772 2773 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { 2774 param[0] = FW_PARAM_PFVF(ETHOFLD_START); 2775 param[1] = FW_PARAM_PFVF(ETHOFLD_END); 2776 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2777 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); 2778 if (rc != 0) { 2779 device_printf(sc->dev, 2780 "failed to query NIC parameters: %d.\n", rc); 2781 return (rc); 2782 } 2783 sc->tids.etid_base = val[0]; 2784 sc->params.etid_min = val[0]; 2785 sc->tids.netids = val[1] - val[0] + 1; 2786 sc->params.netids = sc->tids.netids; 2787 sc->params.eo_wr_cred = val[2]; 2788 sc->params.ethoffload = 1; 2789 } 2790 2791 if (sc->toecaps) { 2792 /* query offload-related parameters */ 2793 param[0] = FW_PARAM_DEV(NTID); 2794 param[1] = FW_PARAM_PFVF(SERVER_START); 2795 param[2] = FW_PARAM_PFVF(SERVER_END); 2796 param[3] = FW_PARAM_PFVF(TDDP_START); 2797 param[4] = FW_PARAM_PFVF(TDDP_END); 2798 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2799 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2800 if (rc != 0) { 2801 device_printf(sc->dev, 2802 "failed to query TOE parameters: %d.\n", rc); 2803 return (rc); 2804 } 2805 sc->tids.ntids = val[0]; 2806 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 2807 sc->tids.stid_base = val[1]; 2808 sc->tids.nstids = val[2] - val[1] + 1; 2809 sc->vres.ddp.start = val[3]; 2810 sc->vres.ddp.size = val[4] - val[3] + 1; 2811 sc->params.ofldq_wr_cred = val[5]; 2812 sc->params.offload = 1; 2813 } 2814 if (sc->rdmacaps) { 2815 param[0] = FW_PARAM_PFVF(STAG_START); 2816 param[1] = FW_PARAM_PFVF(STAG_END); 2817 param[2] = FW_PARAM_PFVF(RQ_START); 2818 param[3] = FW_PARAM_PFVF(RQ_END); 2819 param[4] = FW_PARAM_PFVF(PBL_START); 2820 param[5] = FW_PARAM_PFVF(PBL_END); 2821 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2822 if (rc != 0) { 2823 device_printf(sc->dev, 2824 "failed to query RDMA parameters(1): %d.\n", rc); 2825 return (rc); 2826 } 2827 sc->vres.stag.start = val[0]; 2828 sc->vres.stag.size = val[1] - val[0] + 1; 2829 sc->vres.rq.start = val[2]; 2830 sc->vres.rq.size = val[3] - val[2] + 1; 2831 sc->vres.pbl.start = val[4]; 2832 sc->vres.pbl.size = val[5] - val[4] + 1; 2833 2834 param[0] = FW_PARAM_PFVF(SQRQ_START); 2835 param[1] = FW_PARAM_PFVF(SQRQ_END); 2836 param[2] = FW_PARAM_PFVF(CQ_START); 2837 param[3] = FW_PARAM_PFVF(CQ_END); 2838 param[4] = FW_PARAM_PFVF(OCQ_START); 2839 param[5] = FW_PARAM_PFVF(OCQ_END); 2840 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2841 if (rc != 0) { 2842 device_printf(sc->dev, 2843 "failed to query RDMA parameters(2): %d.\n", rc); 2844 return (rc); 2845 } 2846 sc->vres.qp.start = val[0]; 2847 sc->vres.qp.size = val[1] - val[0] + 1; 2848 sc->vres.cq.start = val[2]; 2849 sc->vres.cq.size = val[3] - val[2] + 1; 2850 sc->vres.ocq.start = val[4]; 2851 sc->vres.ocq.size = val[5] - val[4] + 1; 2852 } 2853 if (sc->iscsicaps) { 2854 param[0] = FW_PARAM_PFVF(ISCSI_START); 2855 param[1] = FW_PARAM_PFVF(ISCSI_END); 2856 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2857 if (rc != 0) { 2858 device_printf(sc->dev, 2859 "failed to query iSCSI parameters: %d.\n", rc); 2860 return (rc); 2861 } 2862 sc->vres.iscsi.start = val[0]; 2863 sc->vres.iscsi.size = val[1] - val[0] + 1; 2864 } 2865 2866 /* 2867 * We've got the params we wanted to query via the firmware. Now grab 2868 * some others directly from the chip. 2869 */ 2870 rc = t4_read_chip_settings(sc); 2871 2872 return (rc); 2873 } 2874 2875 static int 2876 set_params__post_init(struct adapter *sc) 2877 { 2878 uint32_t param, val; 2879 2880 /* ask for encapsulated CPLs */ 2881 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 2882 val = 1; 2883 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2884 2885 return (0); 2886 } 2887 2888 #undef FW_PARAM_PFVF 2889 #undef FW_PARAM_DEV 2890 2891 static void 2892 t4_set_desc(struct adapter *sc) 2893 { 2894 char buf[128]; 2895 struct adapter_params *p = &sc->params; 2896 2897 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, " 2898 "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "", 2899 chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec); 2900 2901 device_set_desc_copy(sc->dev, buf); 2902 } 2903 2904 static void 2905 build_medialist(struct port_info *pi, struct ifmedia *media) 2906 { 2907 int data, m; 2908 2909 PORT_LOCK(pi); 2910 2911 ifmedia_removeall(media); 2912 2913 m = IFM_ETHER | IFM_FDX; 2914 data = (pi->port_type << 8) | pi->mod_type; 2915 2916 switch(pi->port_type) { 2917 case FW_PORT_TYPE_BT_XFI: 2918 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2919 break; 2920 2921 case FW_PORT_TYPE_BT_XAUI: 2922 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2923 /* fall through */ 2924 2925 case FW_PORT_TYPE_BT_SGMII: 2926 ifmedia_add(media, m | IFM_1000_T, data, NULL); 2927 ifmedia_add(media, m | IFM_100_TX, data, NULL); 2928 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL); 2929 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2930 break; 2931 2932 case FW_PORT_TYPE_CX4: 2933 ifmedia_add(media, m | IFM_10G_CX4, data, NULL); 2934 ifmedia_set(media, m | IFM_10G_CX4); 2935 break; 2936 2937 case FW_PORT_TYPE_QSFP_10G: 2938 case FW_PORT_TYPE_SFP: 2939 case FW_PORT_TYPE_FIBER_XFI: 2940 case FW_PORT_TYPE_FIBER_XAUI: 2941 switch (pi->mod_type) { 2942 2943 case FW_PORT_MOD_TYPE_LR: 2944 ifmedia_add(media, m | IFM_10G_LR, data, NULL); 2945 ifmedia_set(media, m | IFM_10G_LR); 2946 break; 2947 2948 case FW_PORT_MOD_TYPE_SR: 2949 ifmedia_add(media, m | IFM_10G_SR, data, NULL); 2950 ifmedia_set(media, m | IFM_10G_SR); 2951 break; 2952 2953 case FW_PORT_MOD_TYPE_LRM: 2954 ifmedia_add(media, m | IFM_10G_LRM, data, NULL); 2955 ifmedia_set(media, m | IFM_10G_LRM); 2956 break; 2957 2958 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2959 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2960 ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL); 2961 ifmedia_set(media, m | IFM_10G_TWINAX); 2962 break; 2963 2964 case FW_PORT_MOD_TYPE_NONE: 2965 m &= ~IFM_FDX; 2966 ifmedia_add(media, m | IFM_NONE, data, NULL); 2967 ifmedia_set(media, m | IFM_NONE); 2968 break; 2969 2970 case FW_PORT_MOD_TYPE_NA: 2971 case FW_PORT_MOD_TYPE_ER: 2972 default: 2973 device_printf(pi->dev, 2974 "unknown port_type (%d), mod_type (%d)\n", 2975 pi->port_type, pi->mod_type); 2976 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2977 ifmedia_set(media, m | IFM_UNKNOWN); 2978 break; 2979 } 2980 break; 2981 2982 case FW_PORT_TYPE_QSFP: 2983 switch (pi->mod_type) { 2984 2985 case FW_PORT_MOD_TYPE_LR: 2986 ifmedia_add(media, m | IFM_40G_LR4, data, NULL); 2987 ifmedia_set(media, m | IFM_40G_LR4); 2988 break; 2989 2990 case FW_PORT_MOD_TYPE_SR: 2991 ifmedia_add(media, m | IFM_40G_SR4, data, NULL); 2992 ifmedia_set(media, m | IFM_40G_SR4); 2993 break; 2994 2995 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2996 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2997 ifmedia_add(media, m | IFM_40G_CR4, data, NULL); 2998 ifmedia_set(media, m | IFM_40G_CR4); 2999 break; 3000 3001 case FW_PORT_MOD_TYPE_NONE: 3002 m &= ~IFM_FDX; 3003 ifmedia_add(media, m | IFM_NONE, data, NULL); 3004 ifmedia_set(media, m | IFM_NONE); 3005 break; 3006 3007 default: 3008 device_printf(pi->dev, 3009 "unknown port_type (%d), mod_type (%d)\n", 3010 pi->port_type, pi->mod_type); 3011 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 3012 ifmedia_set(media, m | IFM_UNKNOWN); 3013 break; 3014 } 3015 break; 3016 3017 default: 3018 device_printf(pi->dev, 3019 "unknown port_type (%d), mod_type (%d)\n", pi->port_type, 3020 pi->mod_type); 3021 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 3022 ifmedia_set(media, m | IFM_UNKNOWN); 3023 break; 3024 } 3025 3026 PORT_UNLOCK(pi); 3027 } 3028 3029 #define FW_MAC_EXACT_CHUNK 7 3030 3031 /* 3032 * Program the port's XGMAC based on parameters in ifnet. The caller also 3033 * indicates which parameters should be programmed (the rest are left alone). 3034 */ 3035 int 3036 update_mac_settings(struct ifnet *ifp, int flags) 3037 { 3038 int rc = 0; 3039 struct port_info *pi = ifp->if_softc; 3040 struct adapter *sc = pi->adapter; 3041 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 3042 uint16_t viid = 0xffff; 3043 int16_t *xact_addr_filt = NULL; 3044 3045 ASSERT_SYNCHRONIZED_OP(sc); 3046 KASSERT(flags, ("%s: not told what to update.", __func__)); 3047 3048 if (ifp == pi->ifp) { 3049 viid = pi->viid; 3050 xact_addr_filt = &pi->xact_addr_filt; 3051 } 3052 #ifdef DEV_NETMAP 3053 else if (ifp == pi->nm_ifp) { 3054 viid = pi->nm_viid; 3055 xact_addr_filt = &pi->nm_xact_addr_filt; 3056 } 3057 #endif 3058 if (flags & XGMAC_MTU) 3059 mtu = ifp->if_mtu; 3060 3061 if (flags & XGMAC_PROMISC) 3062 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 3063 3064 if (flags & XGMAC_ALLMULTI) 3065 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 3066 3067 if (flags & XGMAC_VLANEX) 3068 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 3069 3070 if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) { 3071 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti, 3072 1, vlanex, false); 3073 if (rc) { 3074 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, 3075 rc); 3076 return (rc); 3077 } 3078 } 3079 3080 if (flags & XGMAC_UCADDR) { 3081 uint8_t ucaddr[ETHER_ADDR_LEN]; 3082 3083 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 3084 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr, 3085 true, true); 3086 if (rc < 0) { 3087 rc = -rc; 3088 if_printf(ifp, "change_mac failed: %d\n", rc); 3089 return (rc); 3090 } else { 3091 *xact_addr_filt = rc; 3092 rc = 0; 3093 } 3094 } 3095 3096 if (flags & XGMAC_MCADDRS) { 3097 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 3098 int del = 1; 3099 uint64_t hash = 0; 3100 struct ifmultiaddr *ifma; 3101 int i = 0, j; 3102 3103 if_maddr_rlock(ifp); 3104 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3105 if (ifma->ifma_addr->sa_family != AF_LINK) 3106 continue; 3107 mcaddr[i] = 3108 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 3109 MPASS(ETHER_IS_MULTICAST(mcaddr[i])); 3110 i++; 3111 3112 if (i == FW_MAC_EXACT_CHUNK) { 3113 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, 3114 i, mcaddr, NULL, &hash, 0); 3115 if (rc < 0) { 3116 rc = -rc; 3117 for (j = 0; j < i; j++) { 3118 if_printf(ifp, 3119 "failed to add mc address" 3120 " %02x:%02x:%02x:" 3121 "%02x:%02x:%02x rc=%d\n", 3122 mcaddr[j][0], mcaddr[j][1], 3123 mcaddr[j][2], mcaddr[j][3], 3124 mcaddr[j][4], mcaddr[j][5], 3125 rc); 3126 } 3127 goto mcfail; 3128 } 3129 del = 0; 3130 i = 0; 3131 } 3132 } 3133 if (i > 0) { 3134 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i, 3135 mcaddr, NULL, &hash, 0); 3136 if (rc < 0) { 3137 rc = -rc; 3138 for (j = 0; j < i; j++) { 3139 if_printf(ifp, 3140 "failed to add mc address" 3141 " %02x:%02x:%02x:" 3142 "%02x:%02x:%02x rc=%d\n", 3143 mcaddr[j][0], mcaddr[j][1], 3144 mcaddr[j][2], mcaddr[j][3], 3145 mcaddr[j][4], mcaddr[j][5], 3146 rc); 3147 } 3148 goto mcfail; 3149 } 3150 } 3151 3152 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0); 3153 if (rc != 0) 3154 if_printf(ifp, "failed to set mc address hash: %d", rc); 3155 mcfail: 3156 if_maddr_runlock(ifp); 3157 } 3158 3159 return (rc); 3160 } 3161 3162 int 3163 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags, 3164 char *wmesg) 3165 { 3166 int rc, pri; 3167 3168 #ifdef WITNESS 3169 /* the caller thinks it's ok to sleep, but is it really? */ 3170 if (flags & SLEEP_OK) 3171 pause("t4slptst", 1); 3172 #endif 3173 3174 if (INTR_OK) 3175 pri = PCATCH; 3176 else 3177 pri = 0; 3178 3179 ADAPTER_LOCK(sc); 3180 for (;;) { 3181 3182 if (pi && IS_DOOMED(pi)) { 3183 rc = ENXIO; 3184 goto done; 3185 } 3186 3187 if (!IS_BUSY(sc)) { 3188 rc = 0; 3189 break; 3190 } 3191 3192 if (!(flags & SLEEP_OK)) { 3193 rc = EBUSY; 3194 goto done; 3195 } 3196 3197 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 3198 rc = EINTR; 3199 goto done; 3200 } 3201 } 3202 3203 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 3204 SET_BUSY(sc); 3205 #ifdef INVARIANTS 3206 sc->last_op = wmesg; 3207 sc->last_op_thr = curthread; 3208 #endif 3209 3210 done: 3211 if (!(flags & HOLD_LOCK) || rc) 3212 ADAPTER_UNLOCK(sc); 3213 3214 return (rc); 3215 } 3216 3217 void 3218 end_synchronized_op(struct adapter *sc, int flags) 3219 { 3220 3221 if (flags & LOCK_HELD) 3222 ADAPTER_LOCK_ASSERT_OWNED(sc); 3223 else 3224 ADAPTER_LOCK(sc); 3225 3226 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 3227 CLR_BUSY(sc); 3228 wakeup(&sc->flags); 3229 ADAPTER_UNLOCK(sc); 3230 } 3231 3232 static int 3233 cxgbe_init_synchronized(struct port_info *pi) 3234 { 3235 struct adapter *sc = pi->adapter; 3236 struct ifnet *ifp = pi->ifp; 3237 int rc = 0; 3238 3239 ASSERT_SYNCHRONIZED_OP(sc); 3240 3241 if (isset(&sc->open_device_map, pi->port_id)) { 3242 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 3243 ("mismatch between open_device_map and if_drv_flags")); 3244 return (0); /* already running */ 3245 } 3246 3247 if (!(sc->flags & FULL_INIT_DONE) && 3248 ((rc = adapter_full_init(sc)) != 0)) 3249 return (rc); /* error message displayed already */ 3250 3251 if (!(pi->flags & PORT_INIT_DONE) && 3252 ((rc = port_full_init(pi)) != 0)) 3253 return (rc); /* error message displayed already */ 3254 3255 rc = update_mac_settings(ifp, XGMAC_ALL); 3256 if (rc) 3257 goto done; /* error message displayed already */ 3258 3259 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 3260 if (rc != 0) { 3261 if_printf(ifp, "enable_vi failed: %d\n", rc); 3262 goto done; 3263 } 3264 3265 /* 3266 * The first iq of the first port to come up is used for tracing. 3267 */ 3268 if (sc->traceq < 0) { 3269 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id; 3270 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : 3271 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | 3272 V_QUEUENUMBER(sc->traceq)); 3273 pi->flags |= HAS_TRACEQ; 3274 } 3275 3276 /* all ok */ 3277 setbit(&sc->open_device_map, pi->port_id); 3278 PORT_LOCK(pi); 3279 ifp->if_drv_flags |= IFF_DRV_RUNNING; 3280 PORT_UNLOCK(pi); 3281 3282 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 3283 done: 3284 if (rc != 0) 3285 cxgbe_uninit_synchronized(pi); 3286 3287 return (rc); 3288 } 3289 3290 /* 3291 * Idempotent. 3292 */ 3293 static int 3294 cxgbe_uninit_synchronized(struct port_info *pi) 3295 { 3296 struct adapter *sc = pi->adapter; 3297 struct ifnet *ifp = pi->ifp; 3298 int rc; 3299 3300 ASSERT_SYNCHRONIZED_OP(sc); 3301 3302 /* 3303 * Disable the VI so that all its data in either direction is discarded 3304 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 3305 * tick) intact as the TP can deliver negative advice or data that it's 3306 * holding in its RAM (for an offloaded connection) even after the VI is 3307 * disabled. 3308 */ 3309 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 3310 if (rc) { 3311 if_printf(ifp, "disable_vi failed: %d\n", rc); 3312 return (rc); 3313 } 3314 3315 clrbit(&sc->open_device_map, pi->port_id); 3316 PORT_LOCK(pi); 3317 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3318 PORT_UNLOCK(pi); 3319 3320 pi->link_cfg.link_ok = 0; 3321 pi->link_cfg.speed = 0; 3322 pi->linkdnrc = -1; 3323 t4_os_link_changed(sc, pi->port_id, 0, -1); 3324 3325 return (0); 3326 } 3327 3328 /* 3329 * It is ok for this function to fail midway and return right away. t4_detach 3330 * will walk the entire sc->irq list and clean up whatever is valid. 3331 */ 3332 static int 3333 setup_intr_handlers(struct adapter *sc) 3334 { 3335 int rc, rid, p, q; 3336 char s[8]; 3337 struct irq *irq; 3338 struct port_info *pi; 3339 struct sge_rxq *rxq; 3340 #ifdef TCP_OFFLOAD 3341 struct sge_ofld_rxq *ofld_rxq; 3342 #endif 3343 #ifdef DEV_NETMAP 3344 struct sge_nm_rxq *nm_rxq; 3345 #endif 3346 3347 /* 3348 * Setup interrupts. 3349 */ 3350 irq = &sc->irq[0]; 3351 rid = sc->intr_type == INTR_INTX ? 0 : 1; 3352 if (sc->intr_count == 1) 3353 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all")); 3354 3355 /* Multiple interrupts. */ 3356 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 3357 ("%s: too few intr.", __func__)); 3358 3359 /* The first one is always error intr */ 3360 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 3361 if (rc != 0) 3362 return (rc); 3363 irq++; 3364 rid++; 3365 3366 /* The second one is always the firmware event queue */ 3367 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt"); 3368 if (rc != 0) 3369 return (rc); 3370 irq++; 3371 rid++; 3372 3373 for_each_port(sc, p) { 3374 pi = sc->port[p]; 3375 3376 if (pi->flags & INTR_RXQ) { 3377 for_each_rxq(pi, q, rxq) { 3378 snprintf(s, sizeof(s), "%d.%d", p, q); 3379 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq, 3380 s); 3381 if (rc != 0) 3382 return (rc); 3383 irq++; 3384 rid++; 3385 } 3386 } 3387 #ifdef TCP_OFFLOAD 3388 if (pi->flags & INTR_OFLD_RXQ) { 3389 for_each_ofld_rxq(pi, q, ofld_rxq) { 3390 snprintf(s, sizeof(s), "%d,%d", p, q); 3391 rc = t4_alloc_irq(sc, irq, rid, t4_intr, 3392 ofld_rxq, s); 3393 if (rc != 0) 3394 return (rc); 3395 irq++; 3396 rid++; 3397 } 3398 } 3399 #endif 3400 #ifdef DEV_NETMAP 3401 if (pi->flags & INTR_NM_RXQ) { 3402 for_each_nm_rxq(pi, q, nm_rxq) { 3403 snprintf(s, sizeof(s), "%d-%d", p, q); 3404 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr, 3405 nm_rxq, s); 3406 if (rc != 0) 3407 return (rc); 3408 irq++; 3409 rid++; 3410 } 3411 } 3412 #endif 3413 } 3414 MPASS(irq == &sc->irq[sc->intr_count]); 3415 3416 return (0); 3417 } 3418 3419 int 3420 adapter_full_init(struct adapter *sc) 3421 { 3422 int rc, i; 3423 3424 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3425 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 3426 ("%s: FULL_INIT_DONE already", __func__)); 3427 3428 /* 3429 * queues that belong to the adapter (not any particular port). 3430 */ 3431 rc = t4_setup_adapter_queues(sc); 3432 if (rc != 0) 3433 goto done; 3434 3435 for (i = 0; i < nitems(sc->tq); i++) { 3436 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 3437 taskqueue_thread_enqueue, &sc->tq[i]); 3438 if (sc->tq[i] == NULL) { 3439 device_printf(sc->dev, 3440 "failed to allocate task queue %d\n", i); 3441 rc = ENOMEM; 3442 goto done; 3443 } 3444 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3445 device_get_nameunit(sc->dev), i); 3446 } 3447 3448 t4_intr_enable(sc); 3449 sc->flags |= FULL_INIT_DONE; 3450 done: 3451 if (rc != 0) 3452 adapter_full_uninit(sc); 3453 3454 return (rc); 3455 } 3456 3457 int 3458 adapter_full_uninit(struct adapter *sc) 3459 { 3460 int i; 3461 3462 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3463 3464 t4_teardown_adapter_queues(sc); 3465 3466 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3467 taskqueue_free(sc->tq[i]); 3468 sc->tq[i] = NULL; 3469 } 3470 3471 sc->flags &= ~FULL_INIT_DONE; 3472 3473 return (0); 3474 } 3475 3476 int 3477 port_full_init(struct port_info *pi) 3478 { 3479 struct adapter *sc = pi->adapter; 3480 struct ifnet *ifp = pi->ifp; 3481 uint16_t *rss; 3482 struct sge_rxq *rxq; 3483 int rc, i, j; 3484 3485 ASSERT_SYNCHRONIZED_OP(sc); 3486 KASSERT((pi->flags & PORT_INIT_DONE) == 0, 3487 ("%s: PORT_INIT_DONE already", __func__)); 3488 3489 sysctl_ctx_init(&pi->ctx); 3490 pi->flags |= PORT_SYSCTL_CTX; 3491 3492 /* 3493 * Allocate tx/rx/fl queues for this port. 3494 */ 3495 rc = t4_setup_port_queues(pi); 3496 if (rc != 0) 3497 goto done; /* error message displayed already */ 3498 3499 /* 3500 * Setup RSS for this port. Save a copy of the RSS table for later use. 3501 */ 3502 rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); 3503 for (i = 0; i < pi->rss_size;) { 3504 for_each_rxq(pi, j, rxq) { 3505 rss[i++] = rxq->iq.abs_id; 3506 if (i == pi->rss_size) 3507 break; 3508 } 3509 } 3510 3511 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss, 3512 pi->rss_size); 3513 if (rc != 0) { 3514 if_printf(ifp, "rss_config failed: %d\n", rc); 3515 goto done; 3516 } 3517 3518 pi->rss = rss; 3519 pi->flags |= PORT_INIT_DONE; 3520 done: 3521 if (rc != 0) 3522 port_full_uninit(pi); 3523 3524 return (rc); 3525 } 3526 3527 /* 3528 * Idempotent. 3529 */ 3530 int 3531 port_full_uninit(struct port_info *pi) 3532 { 3533 struct adapter *sc = pi->adapter; 3534 int i; 3535 struct sge_rxq *rxq; 3536 struct sge_txq *txq; 3537 #ifdef TCP_OFFLOAD 3538 struct sge_ofld_rxq *ofld_rxq; 3539 struct sge_wrq *ofld_txq; 3540 #endif 3541 3542 if (pi->flags & PORT_INIT_DONE) { 3543 3544 /* Need to quiesce queues. XXX: ctrl queues? */ 3545 3546 for_each_txq(pi, i, txq) { 3547 quiesce_eq(sc, &txq->eq); 3548 } 3549 3550 #ifdef TCP_OFFLOAD 3551 for_each_ofld_txq(pi, i, ofld_txq) { 3552 quiesce_eq(sc, &ofld_txq->eq); 3553 } 3554 #endif 3555 3556 for_each_rxq(pi, i, rxq) { 3557 quiesce_iq(sc, &rxq->iq); 3558 quiesce_fl(sc, &rxq->fl); 3559 } 3560 3561 #ifdef TCP_OFFLOAD 3562 for_each_ofld_rxq(pi, i, ofld_rxq) { 3563 quiesce_iq(sc, &ofld_rxq->iq); 3564 quiesce_fl(sc, &ofld_rxq->fl); 3565 } 3566 #endif 3567 free(pi->rss, M_CXGBE); 3568 } 3569 3570 t4_teardown_port_queues(pi); 3571 pi->flags &= ~PORT_INIT_DONE; 3572 3573 return (0); 3574 } 3575 3576 static void 3577 quiesce_eq(struct adapter *sc, struct sge_eq *eq) 3578 { 3579 EQ_LOCK(eq); 3580 eq->flags |= EQ_DOOMED; 3581 3582 /* 3583 * Wait for the response to a credit flush if one's 3584 * pending. 3585 */ 3586 while (eq->flags & EQ_CRFLUSHED) 3587 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0); 3588 EQ_UNLOCK(eq); 3589 3590 callout_drain(&eq->tx_callout); /* XXX: iffy */ 3591 pause("callout", 10); /* Still iffy */ 3592 3593 taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task); 3594 } 3595 3596 static void 3597 quiesce_iq(struct adapter *sc, struct sge_iq *iq) 3598 { 3599 (void) sc; /* unused */ 3600 3601 /* Synchronize with the interrupt handler */ 3602 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 3603 pause("iqfree", 1); 3604 } 3605 3606 static void 3607 quiesce_fl(struct adapter *sc, struct sge_fl *fl) 3608 { 3609 mtx_lock(&sc->sfl_lock); 3610 FL_LOCK(fl); 3611 fl->flags |= FL_DOOMED; 3612 FL_UNLOCK(fl); 3613 mtx_unlock(&sc->sfl_lock); 3614 3615 callout_drain(&sc->sfl_callout); 3616 KASSERT((fl->flags & FL_STARVING) == 0, 3617 ("%s: still starving", __func__)); 3618 } 3619 3620 static int 3621 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 3622 driver_intr_t *handler, void *arg, char *name) 3623 { 3624 int rc; 3625 3626 irq->rid = rid; 3627 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 3628 RF_SHAREABLE | RF_ACTIVE); 3629 if (irq->res == NULL) { 3630 device_printf(sc->dev, 3631 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 3632 return (ENOMEM); 3633 } 3634 3635 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 3636 NULL, handler, arg, &irq->tag); 3637 if (rc != 0) { 3638 device_printf(sc->dev, 3639 "failed to setup interrupt for rid %d, name %s: %d\n", 3640 rid, name, rc); 3641 } else if (name) 3642 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 3643 3644 return (rc); 3645 } 3646 3647 static int 3648 t4_free_irq(struct adapter *sc, struct irq *irq) 3649 { 3650 if (irq->tag) 3651 bus_teardown_intr(sc->dev, irq->res, irq->tag); 3652 if (irq->res) 3653 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 3654 3655 bzero(irq, sizeof(*irq)); 3656 3657 return (0); 3658 } 3659 3660 static void 3661 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 3662 unsigned int end) 3663 { 3664 uint32_t *p = (uint32_t *)(buf + start); 3665 3666 for ( ; start <= end; start += sizeof(uint32_t)) 3667 *p++ = t4_read_reg(sc, start); 3668 } 3669 3670 static void 3671 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 3672 { 3673 int i, n; 3674 const unsigned int *reg_ranges; 3675 static const unsigned int t4_reg_ranges[] = { 3676 0x1008, 0x1108, 3677 0x1180, 0x11b4, 3678 0x11fc, 0x123c, 3679 0x1300, 0x173c, 3680 0x1800, 0x18fc, 3681 0x3000, 0x30d8, 3682 0x30e0, 0x5924, 3683 0x5960, 0x59d4, 3684 0x5a00, 0x5af8, 3685 0x6000, 0x6098, 3686 0x6100, 0x6150, 3687 0x6200, 0x6208, 3688 0x6240, 0x6248, 3689 0x6280, 0x6338, 3690 0x6370, 0x638c, 3691 0x6400, 0x643c, 3692 0x6500, 0x6524, 3693 0x6a00, 0x6a38, 3694 0x6a60, 0x6a78, 3695 0x6b00, 0x6b84, 3696 0x6bf0, 0x6c84, 3697 0x6cf0, 0x6d84, 3698 0x6df0, 0x6e84, 3699 0x6ef0, 0x6f84, 3700 0x6ff0, 0x7084, 3701 0x70f0, 0x7184, 3702 0x71f0, 0x7284, 3703 0x72f0, 0x7384, 3704 0x73f0, 0x7450, 3705 0x7500, 0x7530, 3706 0x7600, 0x761c, 3707 0x7680, 0x76cc, 3708 0x7700, 0x7798, 3709 0x77c0, 0x77fc, 3710 0x7900, 0x79fc, 3711 0x7b00, 0x7c38, 3712 0x7d00, 0x7efc, 3713 0x8dc0, 0x8e1c, 3714 0x8e30, 0x8e78, 3715 0x8ea0, 0x8f6c, 3716 0x8fc0, 0x9074, 3717 0x90fc, 0x90fc, 3718 0x9400, 0x9458, 3719 0x9600, 0x96bc, 3720 0x9800, 0x9808, 3721 0x9820, 0x983c, 3722 0x9850, 0x9864, 3723 0x9c00, 0x9c6c, 3724 0x9c80, 0x9cec, 3725 0x9d00, 0x9d6c, 3726 0x9d80, 0x9dec, 3727 0x9e00, 0x9e6c, 3728 0x9e80, 0x9eec, 3729 0x9f00, 0x9f6c, 3730 0x9f80, 0x9fec, 3731 0xd004, 0xd03c, 3732 0xdfc0, 0xdfe0, 3733 0xe000, 0xea7c, 3734 0xf000, 0x11110, 3735 0x11118, 0x11190, 3736 0x19040, 0x1906c, 3737 0x19078, 0x19080, 3738 0x1908c, 0x19124, 3739 0x19150, 0x191b0, 3740 0x191d0, 0x191e8, 3741 0x19238, 0x1924c, 3742 0x193f8, 0x19474, 3743 0x19490, 0x194f8, 3744 0x19800, 0x19f30, 3745 0x1a000, 0x1a06c, 3746 0x1a0b0, 0x1a120, 3747 0x1a128, 0x1a138, 3748 0x1a190, 0x1a1c4, 3749 0x1a1fc, 0x1a1fc, 3750 0x1e040, 0x1e04c, 3751 0x1e284, 0x1e28c, 3752 0x1e2c0, 0x1e2c0, 3753 0x1e2e0, 0x1e2e0, 3754 0x1e300, 0x1e384, 3755 0x1e3c0, 0x1e3c8, 3756 0x1e440, 0x1e44c, 3757 0x1e684, 0x1e68c, 3758 0x1e6c0, 0x1e6c0, 3759 0x1e6e0, 0x1e6e0, 3760 0x1e700, 0x1e784, 3761 0x1e7c0, 0x1e7c8, 3762 0x1e840, 0x1e84c, 3763 0x1ea84, 0x1ea8c, 3764 0x1eac0, 0x1eac0, 3765 0x1eae0, 0x1eae0, 3766 0x1eb00, 0x1eb84, 3767 0x1ebc0, 0x1ebc8, 3768 0x1ec40, 0x1ec4c, 3769 0x1ee84, 0x1ee8c, 3770 0x1eec0, 0x1eec0, 3771 0x1eee0, 0x1eee0, 3772 0x1ef00, 0x1ef84, 3773 0x1efc0, 0x1efc8, 3774 0x1f040, 0x1f04c, 3775 0x1f284, 0x1f28c, 3776 0x1f2c0, 0x1f2c0, 3777 0x1f2e0, 0x1f2e0, 3778 0x1f300, 0x1f384, 3779 0x1f3c0, 0x1f3c8, 3780 0x1f440, 0x1f44c, 3781 0x1f684, 0x1f68c, 3782 0x1f6c0, 0x1f6c0, 3783 0x1f6e0, 0x1f6e0, 3784 0x1f700, 0x1f784, 3785 0x1f7c0, 0x1f7c8, 3786 0x1f840, 0x1f84c, 3787 0x1fa84, 0x1fa8c, 3788 0x1fac0, 0x1fac0, 3789 0x1fae0, 0x1fae0, 3790 0x1fb00, 0x1fb84, 3791 0x1fbc0, 0x1fbc8, 3792 0x1fc40, 0x1fc4c, 3793 0x1fe84, 0x1fe8c, 3794 0x1fec0, 0x1fec0, 3795 0x1fee0, 0x1fee0, 3796 0x1ff00, 0x1ff84, 3797 0x1ffc0, 0x1ffc8, 3798 0x20000, 0x2002c, 3799 0x20100, 0x2013c, 3800 0x20190, 0x201c8, 3801 0x20200, 0x20318, 3802 0x20400, 0x20528, 3803 0x20540, 0x20614, 3804 0x21000, 0x21040, 3805 0x2104c, 0x21060, 3806 0x210c0, 0x210ec, 3807 0x21200, 0x21268, 3808 0x21270, 0x21284, 3809 0x212fc, 0x21388, 3810 0x21400, 0x21404, 3811 0x21500, 0x21518, 3812 0x2152c, 0x2153c, 3813 0x21550, 0x21554, 3814 0x21600, 0x21600, 3815 0x21608, 0x21628, 3816 0x21630, 0x2163c, 3817 0x21700, 0x2171c, 3818 0x21780, 0x2178c, 3819 0x21800, 0x21c38, 3820 0x21c80, 0x21d7c, 3821 0x21e00, 0x21e04, 3822 0x22000, 0x2202c, 3823 0x22100, 0x2213c, 3824 0x22190, 0x221c8, 3825 0x22200, 0x22318, 3826 0x22400, 0x22528, 3827 0x22540, 0x22614, 3828 0x23000, 0x23040, 3829 0x2304c, 0x23060, 3830 0x230c0, 0x230ec, 3831 0x23200, 0x23268, 3832 0x23270, 0x23284, 3833 0x232fc, 0x23388, 3834 0x23400, 0x23404, 3835 0x23500, 0x23518, 3836 0x2352c, 0x2353c, 3837 0x23550, 0x23554, 3838 0x23600, 0x23600, 3839 0x23608, 0x23628, 3840 0x23630, 0x2363c, 3841 0x23700, 0x2371c, 3842 0x23780, 0x2378c, 3843 0x23800, 0x23c38, 3844 0x23c80, 0x23d7c, 3845 0x23e00, 0x23e04, 3846 0x24000, 0x2402c, 3847 0x24100, 0x2413c, 3848 0x24190, 0x241c8, 3849 0x24200, 0x24318, 3850 0x24400, 0x24528, 3851 0x24540, 0x24614, 3852 0x25000, 0x25040, 3853 0x2504c, 0x25060, 3854 0x250c0, 0x250ec, 3855 0x25200, 0x25268, 3856 0x25270, 0x25284, 3857 0x252fc, 0x25388, 3858 0x25400, 0x25404, 3859 0x25500, 0x25518, 3860 0x2552c, 0x2553c, 3861 0x25550, 0x25554, 3862 0x25600, 0x25600, 3863 0x25608, 0x25628, 3864 0x25630, 0x2563c, 3865 0x25700, 0x2571c, 3866 0x25780, 0x2578c, 3867 0x25800, 0x25c38, 3868 0x25c80, 0x25d7c, 3869 0x25e00, 0x25e04, 3870 0x26000, 0x2602c, 3871 0x26100, 0x2613c, 3872 0x26190, 0x261c8, 3873 0x26200, 0x26318, 3874 0x26400, 0x26528, 3875 0x26540, 0x26614, 3876 0x27000, 0x27040, 3877 0x2704c, 0x27060, 3878 0x270c0, 0x270ec, 3879 0x27200, 0x27268, 3880 0x27270, 0x27284, 3881 0x272fc, 0x27388, 3882 0x27400, 0x27404, 3883 0x27500, 0x27518, 3884 0x2752c, 0x2753c, 3885 0x27550, 0x27554, 3886 0x27600, 0x27600, 3887 0x27608, 0x27628, 3888 0x27630, 0x2763c, 3889 0x27700, 0x2771c, 3890 0x27780, 0x2778c, 3891 0x27800, 0x27c38, 3892 0x27c80, 0x27d7c, 3893 0x27e00, 0x27e04 3894 }; 3895 static const unsigned int t5_reg_ranges[] = { 3896 0x1008, 0x1148, 3897 0x1180, 0x11b4, 3898 0x11fc, 0x123c, 3899 0x1280, 0x173c, 3900 0x1800, 0x18fc, 3901 0x3000, 0x3028, 3902 0x3060, 0x30d8, 3903 0x30e0, 0x30fc, 3904 0x3140, 0x357c, 3905 0x35a8, 0x35cc, 3906 0x35ec, 0x35ec, 3907 0x3600, 0x5624, 3908 0x56cc, 0x575c, 3909 0x580c, 0x5814, 3910 0x5890, 0x58bc, 3911 0x5940, 0x59dc, 3912 0x59fc, 0x5a18, 3913 0x5a60, 0x5a9c, 3914 0x5b94, 0x5bfc, 3915 0x6000, 0x6040, 3916 0x6058, 0x614c, 3917 0x7700, 0x7798, 3918 0x77c0, 0x78fc, 3919 0x7b00, 0x7c54, 3920 0x7d00, 0x7efc, 3921 0x8dc0, 0x8de0, 3922 0x8df8, 0x8e84, 3923 0x8ea0, 0x8f84, 3924 0x8fc0, 0x90f8, 3925 0x9400, 0x9470, 3926 0x9600, 0x96f4, 3927 0x9800, 0x9808, 3928 0x9820, 0x983c, 3929 0x9850, 0x9864, 3930 0x9c00, 0x9c6c, 3931 0x9c80, 0x9cec, 3932 0x9d00, 0x9d6c, 3933 0x9d80, 0x9dec, 3934 0x9e00, 0x9e6c, 3935 0x9e80, 0x9eec, 3936 0x9f00, 0x9f6c, 3937 0x9f80, 0xa020, 3938 0xd004, 0xd03c, 3939 0xdfc0, 0xdfe0, 3940 0xe000, 0x11088, 3941 0x1109c, 0x11110, 3942 0x11118, 0x1117c, 3943 0x11190, 0x11204, 3944 0x19040, 0x1906c, 3945 0x19078, 0x19080, 3946 0x1908c, 0x19124, 3947 0x19150, 0x191b0, 3948 0x191d0, 0x191e8, 3949 0x19238, 0x19290, 3950 0x193f8, 0x19474, 3951 0x19490, 0x194cc, 3952 0x194f0, 0x194f8, 3953 0x19c00, 0x19c60, 3954 0x19c94, 0x19e10, 3955 0x19e50, 0x19f34, 3956 0x19f40, 0x19f50, 3957 0x19f90, 0x19fe4, 3958 0x1a000, 0x1a06c, 3959 0x1a0b0, 0x1a120, 3960 0x1a128, 0x1a138, 3961 0x1a190, 0x1a1c4, 3962 0x1a1fc, 0x1a1fc, 3963 0x1e008, 0x1e00c, 3964 0x1e040, 0x1e04c, 3965 0x1e284, 0x1e290, 3966 0x1e2c0, 0x1e2c0, 3967 0x1e2e0, 0x1e2e0, 3968 0x1e300, 0x1e384, 3969 0x1e3c0, 0x1e3c8, 3970 0x1e408, 0x1e40c, 3971 0x1e440, 0x1e44c, 3972 0x1e684, 0x1e690, 3973 0x1e6c0, 0x1e6c0, 3974 0x1e6e0, 0x1e6e0, 3975 0x1e700, 0x1e784, 3976 0x1e7c0, 0x1e7c8, 3977 0x1e808, 0x1e80c, 3978 0x1e840, 0x1e84c, 3979 0x1ea84, 0x1ea90, 3980 0x1eac0, 0x1eac0, 3981 0x1eae0, 0x1eae0, 3982 0x1eb00, 0x1eb84, 3983 0x1ebc0, 0x1ebc8, 3984 0x1ec08, 0x1ec0c, 3985 0x1ec40, 0x1ec4c, 3986 0x1ee84, 0x1ee90, 3987 0x1eec0, 0x1eec0, 3988 0x1eee0, 0x1eee0, 3989 0x1ef00, 0x1ef84, 3990 0x1efc0, 0x1efc8, 3991 0x1f008, 0x1f00c, 3992 0x1f040, 0x1f04c, 3993 0x1f284, 0x1f290, 3994 0x1f2c0, 0x1f2c0, 3995 0x1f2e0, 0x1f2e0, 3996 0x1f300, 0x1f384, 3997 0x1f3c0, 0x1f3c8, 3998 0x1f408, 0x1f40c, 3999 0x1f440, 0x1f44c, 4000 0x1f684, 0x1f690, 4001 0x1f6c0, 0x1f6c0, 4002 0x1f6e0, 0x1f6e0, 4003 0x1f700, 0x1f784, 4004 0x1f7c0, 0x1f7c8, 4005 0x1f808, 0x1f80c, 4006 0x1f840, 0x1f84c, 4007 0x1fa84, 0x1fa90, 4008 0x1fac0, 0x1fac0, 4009 0x1fae0, 0x1fae0, 4010 0x1fb00, 0x1fb84, 4011 0x1fbc0, 0x1fbc8, 4012 0x1fc08, 0x1fc0c, 4013 0x1fc40, 0x1fc4c, 4014 0x1fe84, 0x1fe90, 4015 0x1fec0, 0x1fec0, 4016 0x1fee0, 0x1fee0, 4017 0x1ff00, 0x1ff84, 4018 0x1ffc0, 0x1ffc8, 4019 0x30000, 0x30030, 4020 0x30100, 0x30144, 4021 0x30190, 0x301d0, 4022 0x30200, 0x30318, 4023 0x30400, 0x3052c, 4024 0x30540, 0x3061c, 4025 0x30800, 0x30834, 4026 0x308c0, 0x30908, 4027 0x30910, 0x309ac, 4028 0x30a00, 0x30a2c, 4029 0x30a44, 0x30a50, 4030 0x30a74, 0x30c24, 4031 0x30d00, 0x30d00, 4032 0x30d08, 0x30d14, 4033 0x30d1c, 0x30d20, 4034 0x30d3c, 0x30d50, 4035 0x31200, 0x3120c, 4036 0x31220, 0x31220, 4037 0x31240, 0x31240, 4038 0x31600, 0x3160c, 4039 0x31a00, 0x31a1c, 4040 0x31e00, 0x31e20, 4041 0x31e38, 0x31e3c, 4042 0x31e80, 0x31e80, 4043 0x31e88, 0x31ea8, 4044 0x31eb0, 0x31eb4, 4045 0x31ec8, 0x31ed4, 4046 0x31fb8, 0x32004, 4047 0x32200, 0x32200, 4048 0x32208, 0x32240, 4049 0x32248, 0x32280, 4050 0x32288, 0x322c0, 4051 0x322c8, 0x322fc, 4052 0x32600, 0x32630, 4053 0x32a00, 0x32abc, 4054 0x32b00, 0x32b70, 4055 0x33000, 0x33048, 4056 0x33060, 0x3309c, 4057 0x330f0, 0x33148, 4058 0x33160, 0x3319c, 4059 0x331f0, 0x332e4, 4060 0x332f8, 0x333e4, 4061 0x333f8, 0x33448, 4062 0x33460, 0x3349c, 4063 0x334f0, 0x33548, 4064 0x33560, 0x3359c, 4065 0x335f0, 0x336e4, 4066 0x336f8, 0x337e4, 4067 0x337f8, 0x337fc, 4068 0x33814, 0x33814, 4069 0x3382c, 0x3382c, 4070 0x33880, 0x3388c, 4071 0x338e8, 0x338ec, 4072 0x33900, 0x33948, 4073 0x33960, 0x3399c, 4074 0x339f0, 0x33ae4, 4075 0x33af8, 0x33b10, 4076 0x33b28, 0x33b28, 4077 0x33b3c, 0x33b50, 4078 0x33bf0, 0x33c10, 4079 0x33c28, 0x33c28, 4080 0x33c3c, 0x33c50, 4081 0x33cf0, 0x33cfc, 4082 0x34000, 0x34030, 4083 0x34100, 0x34144, 4084 0x34190, 0x341d0, 4085 0x34200, 0x34318, 4086 0x34400, 0x3452c, 4087 0x34540, 0x3461c, 4088 0x34800, 0x34834, 4089 0x348c0, 0x34908, 4090 0x34910, 0x349ac, 4091 0x34a00, 0x34a2c, 4092 0x34a44, 0x34a50, 4093 0x34a74, 0x34c24, 4094 0x34d00, 0x34d00, 4095 0x34d08, 0x34d14, 4096 0x34d1c, 0x34d20, 4097 0x34d3c, 0x34d50, 4098 0x35200, 0x3520c, 4099 0x35220, 0x35220, 4100 0x35240, 0x35240, 4101 0x35600, 0x3560c, 4102 0x35a00, 0x35a1c, 4103 0x35e00, 0x35e20, 4104 0x35e38, 0x35e3c, 4105 0x35e80, 0x35e80, 4106 0x35e88, 0x35ea8, 4107 0x35eb0, 0x35eb4, 4108 0x35ec8, 0x35ed4, 4109 0x35fb8, 0x36004, 4110 0x36200, 0x36200, 4111 0x36208, 0x36240, 4112 0x36248, 0x36280, 4113 0x36288, 0x362c0, 4114 0x362c8, 0x362fc, 4115 0x36600, 0x36630, 4116 0x36a00, 0x36abc, 4117 0x36b00, 0x36b70, 4118 0x37000, 0x37048, 4119 0x37060, 0x3709c, 4120 0x370f0, 0x37148, 4121 0x37160, 0x3719c, 4122 0x371f0, 0x372e4, 4123 0x372f8, 0x373e4, 4124 0x373f8, 0x37448, 4125 0x37460, 0x3749c, 4126 0x374f0, 0x37548, 4127 0x37560, 0x3759c, 4128 0x375f0, 0x376e4, 4129 0x376f8, 0x377e4, 4130 0x377f8, 0x377fc, 4131 0x37814, 0x37814, 4132 0x3782c, 0x3782c, 4133 0x37880, 0x3788c, 4134 0x378e8, 0x378ec, 4135 0x37900, 0x37948, 4136 0x37960, 0x3799c, 4137 0x379f0, 0x37ae4, 4138 0x37af8, 0x37b10, 4139 0x37b28, 0x37b28, 4140 0x37b3c, 0x37b50, 4141 0x37bf0, 0x37c10, 4142 0x37c28, 0x37c28, 4143 0x37c3c, 0x37c50, 4144 0x37cf0, 0x37cfc, 4145 0x38000, 0x38030, 4146 0x38100, 0x38144, 4147 0x38190, 0x381d0, 4148 0x38200, 0x38318, 4149 0x38400, 0x3852c, 4150 0x38540, 0x3861c, 4151 0x38800, 0x38834, 4152 0x388c0, 0x38908, 4153 0x38910, 0x389ac, 4154 0x38a00, 0x38a2c, 4155 0x38a44, 0x38a50, 4156 0x38a74, 0x38c24, 4157 0x38d00, 0x38d00, 4158 0x38d08, 0x38d14, 4159 0x38d1c, 0x38d20, 4160 0x38d3c, 0x38d50, 4161 0x39200, 0x3920c, 4162 0x39220, 0x39220, 4163 0x39240, 0x39240, 4164 0x39600, 0x3960c, 4165 0x39a00, 0x39a1c, 4166 0x39e00, 0x39e20, 4167 0x39e38, 0x39e3c, 4168 0x39e80, 0x39e80, 4169 0x39e88, 0x39ea8, 4170 0x39eb0, 0x39eb4, 4171 0x39ec8, 0x39ed4, 4172 0x39fb8, 0x3a004, 4173 0x3a200, 0x3a200, 4174 0x3a208, 0x3a240, 4175 0x3a248, 0x3a280, 4176 0x3a288, 0x3a2c0, 4177 0x3a2c8, 0x3a2fc, 4178 0x3a600, 0x3a630, 4179 0x3aa00, 0x3aabc, 4180 0x3ab00, 0x3ab70, 4181 0x3b000, 0x3b048, 4182 0x3b060, 0x3b09c, 4183 0x3b0f0, 0x3b148, 4184 0x3b160, 0x3b19c, 4185 0x3b1f0, 0x3b2e4, 4186 0x3b2f8, 0x3b3e4, 4187 0x3b3f8, 0x3b448, 4188 0x3b460, 0x3b49c, 4189 0x3b4f0, 0x3b548, 4190 0x3b560, 0x3b59c, 4191 0x3b5f0, 0x3b6e4, 4192 0x3b6f8, 0x3b7e4, 4193 0x3b7f8, 0x3b7fc, 4194 0x3b814, 0x3b814, 4195 0x3b82c, 0x3b82c, 4196 0x3b880, 0x3b88c, 4197 0x3b8e8, 0x3b8ec, 4198 0x3b900, 0x3b948, 4199 0x3b960, 0x3b99c, 4200 0x3b9f0, 0x3bae4, 4201 0x3baf8, 0x3bb10, 4202 0x3bb28, 0x3bb28, 4203 0x3bb3c, 0x3bb50, 4204 0x3bbf0, 0x3bc10, 4205 0x3bc28, 0x3bc28, 4206 0x3bc3c, 0x3bc50, 4207 0x3bcf0, 0x3bcfc, 4208 0x3c000, 0x3c030, 4209 0x3c100, 0x3c144, 4210 0x3c190, 0x3c1d0, 4211 0x3c200, 0x3c318, 4212 0x3c400, 0x3c52c, 4213 0x3c540, 0x3c61c, 4214 0x3c800, 0x3c834, 4215 0x3c8c0, 0x3c908, 4216 0x3c910, 0x3c9ac, 4217 0x3ca00, 0x3ca2c, 4218 0x3ca44, 0x3ca50, 4219 0x3ca74, 0x3cc24, 4220 0x3cd00, 0x3cd00, 4221 0x3cd08, 0x3cd14, 4222 0x3cd1c, 0x3cd20, 4223 0x3cd3c, 0x3cd50, 4224 0x3d200, 0x3d20c, 4225 0x3d220, 0x3d220, 4226 0x3d240, 0x3d240, 4227 0x3d600, 0x3d60c, 4228 0x3da00, 0x3da1c, 4229 0x3de00, 0x3de20, 4230 0x3de38, 0x3de3c, 4231 0x3de80, 0x3de80, 4232 0x3de88, 0x3dea8, 4233 0x3deb0, 0x3deb4, 4234 0x3dec8, 0x3ded4, 4235 0x3dfb8, 0x3e004, 4236 0x3e200, 0x3e200, 4237 0x3e208, 0x3e240, 4238 0x3e248, 0x3e280, 4239 0x3e288, 0x3e2c0, 4240 0x3e2c8, 0x3e2fc, 4241 0x3e600, 0x3e630, 4242 0x3ea00, 0x3eabc, 4243 0x3eb00, 0x3eb70, 4244 0x3f000, 0x3f048, 4245 0x3f060, 0x3f09c, 4246 0x3f0f0, 0x3f148, 4247 0x3f160, 0x3f19c, 4248 0x3f1f0, 0x3f2e4, 4249 0x3f2f8, 0x3f3e4, 4250 0x3f3f8, 0x3f448, 4251 0x3f460, 0x3f49c, 4252 0x3f4f0, 0x3f548, 4253 0x3f560, 0x3f59c, 4254 0x3f5f0, 0x3f6e4, 4255 0x3f6f8, 0x3f7e4, 4256 0x3f7f8, 0x3f7fc, 4257 0x3f814, 0x3f814, 4258 0x3f82c, 0x3f82c, 4259 0x3f880, 0x3f88c, 4260 0x3f8e8, 0x3f8ec, 4261 0x3f900, 0x3f948, 4262 0x3f960, 0x3f99c, 4263 0x3f9f0, 0x3fae4, 4264 0x3faf8, 0x3fb10, 4265 0x3fb28, 0x3fb28, 4266 0x3fb3c, 0x3fb50, 4267 0x3fbf0, 0x3fc10, 4268 0x3fc28, 0x3fc28, 4269 0x3fc3c, 0x3fc50, 4270 0x3fcf0, 0x3fcfc, 4271 0x40000, 0x4000c, 4272 0x40040, 0x40068, 4273 0x4007c, 0x40144, 4274 0x40180, 0x4018c, 4275 0x40200, 0x40298, 4276 0x402ac, 0x4033c, 4277 0x403f8, 0x403fc, 4278 0x41304, 0x413c4, 4279 0x41400, 0x4141c, 4280 0x41480, 0x414d0, 4281 0x44000, 0x44078, 4282 0x440c0, 0x44278, 4283 0x442c0, 0x44478, 4284 0x444c0, 0x44678, 4285 0x446c0, 0x44878, 4286 0x448c0, 0x449fc, 4287 0x45000, 0x45068, 4288 0x45080, 0x45084, 4289 0x450a0, 0x450b0, 4290 0x45200, 0x45268, 4291 0x45280, 0x45284, 4292 0x452a0, 0x452b0, 4293 0x460c0, 0x460e4, 4294 0x47000, 0x4708c, 4295 0x47200, 0x47250, 4296 0x47400, 0x47420, 4297 0x47600, 0x47618, 4298 0x47800, 0x47814, 4299 0x48000, 0x4800c, 4300 0x48040, 0x48068, 4301 0x4807c, 0x48144, 4302 0x48180, 0x4818c, 4303 0x48200, 0x48298, 4304 0x482ac, 0x4833c, 4305 0x483f8, 0x483fc, 4306 0x49304, 0x493c4, 4307 0x49400, 0x4941c, 4308 0x49480, 0x494d0, 4309 0x4c000, 0x4c078, 4310 0x4c0c0, 0x4c278, 4311 0x4c2c0, 0x4c478, 4312 0x4c4c0, 0x4c678, 4313 0x4c6c0, 0x4c878, 4314 0x4c8c0, 0x4c9fc, 4315 0x4d000, 0x4d068, 4316 0x4d080, 0x4d084, 4317 0x4d0a0, 0x4d0b0, 4318 0x4d200, 0x4d268, 4319 0x4d280, 0x4d284, 4320 0x4d2a0, 0x4d2b0, 4321 0x4e0c0, 0x4e0e4, 4322 0x4f000, 0x4f08c, 4323 0x4f200, 0x4f250, 4324 0x4f400, 0x4f420, 4325 0x4f600, 0x4f618, 4326 0x4f800, 0x4f814, 4327 0x50000, 0x500cc, 4328 0x50400, 0x50400, 4329 0x50800, 0x508cc, 4330 0x50c00, 0x50c00, 4331 0x51000, 0x5101c, 4332 0x51300, 0x51308, 4333 }; 4334 4335 if (is_t4(sc)) { 4336 reg_ranges = &t4_reg_ranges[0]; 4337 n = nitems(t4_reg_ranges); 4338 } else { 4339 reg_ranges = &t5_reg_ranges[0]; 4340 n = nitems(t5_reg_ranges); 4341 } 4342 4343 regs->version = chip_id(sc) | chip_rev(sc) << 10; 4344 for (i = 0; i < n; i += 2) 4345 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 4346 } 4347 4348 static void 4349 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi) 4350 { 4351 int i; 4352 u_int v, tnl_cong_drops; 4353 struct timeval tv; 4354 const struct timeval interval = {0, 250000}; /* 250ms */ 4355 4356 getmicrotime(&tv); 4357 timevalsub(&tv, &interval); 4358 if (timevalcmp(&tv, &pi->last_refreshed, <)) 4359 return; 4360 4361 tnl_cong_drops = 0; 4362 t4_get_port_stats(sc, pi->tx_chan, &pi->stats); 4363 for (i = 0; i < NCHAN; i++) { 4364 if (pi->rx_chan_map & (1 << i)) { 4365 mtx_lock(&sc->regwin_lock); 4366 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 4367 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 4368 mtx_unlock(&sc->regwin_lock); 4369 tnl_cong_drops += v; 4370 } 4371 } 4372 pi->tnl_cong_drops = tnl_cong_drops; 4373 getmicrotime(&pi->last_refreshed); 4374 } 4375 4376 static void 4377 cxgbe_tick(void *arg) 4378 { 4379 struct port_info *pi = arg; 4380 struct adapter *sc = pi->adapter; 4381 struct ifnet *ifp = pi->ifp; 4382 4383 PORT_LOCK(pi); 4384 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 4385 PORT_UNLOCK(pi); 4386 return; /* without scheduling another callout */ 4387 } 4388 4389 cxgbe_refresh_stats(sc, pi); 4390 4391 callout_schedule(&pi->tick, hz); 4392 PORT_UNLOCK(pi); 4393 } 4394 4395 static void 4396 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 4397 { 4398 struct ifnet *vlan; 4399 4400 if (arg != ifp || ifp->if_type != IFT_ETHER) 4401 return; 4402 4403 vlan = VLAN_DEVAT(ifp, vid); 4404 VLAN_SETCOOKIE(vlan, ifp); 4405 } 4406 4407 static int 4408 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 4409 { 4410 4411 #ifdef INVARIANTS 4412 panic("%s: opcode 0x%02x on iq %p with payload %p", 4413 __func__, rss->opcode, iq, m); 4414 #else 4415 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 4416 __func__, rss->opcode, iq, m); 4417 m_freem(m); 4418 #endif 4419 return (EDOOFUS); 4420 } 4421 4422 int 4423 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 4424 { 4425 uintptr_t *loc, new; 4426 4427 if (opcode >= nitems(sc->cpl_handler)) 4428 return (EINVAL); 4429 4430 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 4431 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 4432 atomic_store_rel_ptr(loc, new); 4433 4434 return (0); 4435 } 4436 4437 static int 4438 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 4439 { 4440 4441 #ifdef INVARIANTS 4442 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 4443 #else 4444 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 4445 __func__, iq, ctrl); 4446 #endif 4447 return (EDOOFUS); 4448 } 4449 4450 int 4451 t4_register_an_handler(struct adapter *sc, an_handler_t h) 4452 { 4453 uintptr_t *loc, new; 4454 4455 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 4456 loc = (uintptr_t *) &sc->an_handler; 4457 atomic_store_rel_ptr(loc, new); 4458 4459 return (0); 4460 } 4461 4462 static int 4463 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 4464 { 4465 const struct cpl_fw6_msg *cpl = 4466 __containerof(rpl, struct cpl_fw6_msg, data[0]); 4467 4468 #ifdef INVARIANTS 4469 panic("%s: fw_msg type %d", __func__, cpl->type); 4470 #else 4471 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 4472 #endif 4473 return (EDOOFUS); 4474 } 4475 4476 int 4477 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 4478 { 4479 uintptr_t *loc, new; 4480 4481 if (type >= nitems(sc->fw_msg_handler)) 4482 return (EINVAL); 4483 4484 /* 4485 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 4486 * handler dispatch table. Reject any attempt to install a handler for 4487 * this subtype. 4488 */ 4489 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 4490 return (EINVAL); 4491 4492 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 4493 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 4494 atomic_store_rel_ptr(loc, new); 4495 4496 return (0); 4497 } 4498 4499 static int 4500 t4_sysctls(struct adapter *sc) 4501 { 4502 struct sysctl_ctx_list *ctx; 4503 struct sysctl_oid *oid; 4504 struct sysctl_oid_list *children, *c0; 4505 static char *caps[] = { 4506 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 4507 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL" /* caps[1] niccaps */ 4508 "\6HASHFILTER\7ETHOFLD", 4509 "\20\1TOE", /* caps[2] toecaps */ 4510 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 4511 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 4512 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 4513 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 4514 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 4515 "\4PO_INITIAOR\5PO_TARGET" 4516 }; 4517 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 4518 4519 ctx = device_get_sysctl_ctx(sc->dev); 4520 4521 /* 4522 * dev.t4nex.X. 4523 */ 4524 oid = device_get_sysctl_tree(sc->dev); 4525 c0 = children = SYSCTL_CHILDREN(oid); 4526 4527 sc->sc_do_rxcopy = 1; 4528 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, 4529 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); 4530 4531 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 4532 sc->params.nports, "# of ports"); 4533 4534 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 4535 NULL, chip_rev(sc), "chip hardware revision"); 4536 4537 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 4538 CTLFLAG_RD, sc->fw_version, 0, "firmware version"); 4539 4540 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 4541 CTLFLAG_RD, sc->cfg_file, 0, "configuration file"); 4542 4543 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 4544 sc->cfcsum, "config file checksum"); 4545 4546 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 4547 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 4548 sysctl_bitfield, "A", "available doorbells"); 4549 4550 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 4551 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 4552 sysctl_bitfield, "A", "available link capabilities"); 4553 4554 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 4555 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 4556 sysctl_bitfield, "A", "available NIC capabilities"); 4557 4558 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 4559 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 4560 sysctl_bitfield, "A", "available TCP offload capabilities"); 4561 4562 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 4563 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 4564 sysctl_bitfield, "A", "available RDMA capabilities"); 4565 4566 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 4567 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 4568 sysctl_bitfield, "A", "available iSCSI capabilities"); 4569 4570 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 4571 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 4572 sysctl_bitfield, "A", "available FCoE capabilities"); 4573 4574 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 4575 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 4576 4577 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 4578 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 4579 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 4580 "interrupt holdoff timer values (us)"); 4581 4582 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 4583 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 4584 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 4585 "interrupt holdoff packet counter values"); 4586 4587 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, 4588 NULL, sc->tids.nftids, "number of filters"); 4589 4590 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | 4591 CTLFLAG_RD, sc, 0, sysctl_temperature, "I", 4592 "chip temperature (in Celsius)"); 4593 4594 t4_sge_sysctls(sc, ctx, children); 4595 4596 sc->lro_timeout = 100; 4597 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, 4598 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); 4599 4600 #ifdef SBUF_DRAIN 4601 /* 4602 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 4603 */ 4604 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 4605 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 4606 "logs and miscellaneous information"); 4607 children = SYSCTL_CHILDREN(oid); 4608 4609 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 4610 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4611 sysctl_cctrl, "A", "congestion control"); 4612 4613 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 4614 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4615 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 4616 4617 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 4618 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 4619 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 4620 4621 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 4622 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 4623 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 4624 4625 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 4626 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 4627 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 4628 4629 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 4630 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 4631 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 4632 4633 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 4634 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 4635 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 4636 4637 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 4638 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4639 sysctl_cim_la, "A", "CIM logic analyzer"); 4640 4641 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", 4642 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4643 sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); 4644 4645 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 4646 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 4647 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 4648 4649 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 4650 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 4651 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 4652 4653 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 4654 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 4655 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 4656 4657 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 4658 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 4659 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 4660 4661 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 4662 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 4663 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 4664 4665 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 4666 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 4667 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 4668 4669 if (is_t5(sc)) { 4670 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 4671 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 4672 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 4673 4674 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 4675 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 4676 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 4677 } 4678 4679 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", 4680 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4681 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); 4682 4683 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 4684 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4685 sysctl_cim_qcfg, "A", "CIM queue configuration"); 4686 4687 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 4688 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4689 sysctl_cpl_stats, "A", "CPL statistics"); 4690 4691 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 4692 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4693 sysctl_ddp_stats, "A", "DDP statistics"); 4694 4695 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 4696 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4697 sysctl_devlog, "A", "firmware's device log"); 4698 4699 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 4700 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4701 sysctl_fcoe_stats, "A", "FCoE statistics"); 4702 4703 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 4704 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4705 sysctl_hw_sched, "A", "hardware scheduler "); 4706 4707 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 4708 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4709 sysctl_l2t, "A", "hardware L2 table"); 4710 4711 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 4712 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4713 sysctl_lb_stats, "A", "loopback statistics"); 4714 4715 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 4716 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4717 sysctl_meminfo, "A", "memory regions"); 4718 4719 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", 4720 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4721 sysctl_mps_tcam, "A", "MPS TCAM entries"); 4722 4723 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 4724 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4725 sysctl_path_mtus, "A", "path MTUs"); 4726 4727 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 4728 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4729 sysctl_pm_stats, "A", "PM statistics"); 4730 4731 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 4732 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4733 sysctl_rdma_stats, "A", "RDMA statistics"); 4734 4735 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 4736 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4737 sysctl_tcp_stats, "A", "TCP statistics"); 4738 4739 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 4740 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4741 sysctl_tids, "A", "TID information"); 4742 4743 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 4744 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4745 sysctl_tp_err_stats, "A", "TP error statistics"); 4746 4747 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", 4748 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4749 sysctl_tp_la, "A", "TP logic analyzer"); 4750 4751 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 4752 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4753 sysctl_tx_rate, "A", "Tx rate"); 4754 4755 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", 4756 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4757 sysctl_ulprx_la, "A", "ULPRX logic analyzer"); 4758 4759 if (is_t5(sc)) { 4760 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 4761 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4762 sysctl_wcwr_stats, "A", "write combined work requests"); 4763 } 4764 #endif 4765 4766 #ifdef TCP_OFFLOAD 4767 if (is_offload(sc)) { 4768 /* 4769 * dev.t4nex.X.toe. 4770 */ 4771 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 4772 NULL, "TOE parameters"); 4773 children = SYSCTL_CHILDREN(oid); 4774 4775 sc->tt.sndbuf = 256 * 1024; 4776 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 4777 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 4778 4779 sc->tt.ddp = 0; 4780 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 4781 &sc->tt.ddp, 0, "DDP allowed"); 4782 4783 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 4784 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 4785 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 4786 4787 sc->tt.ddp_thres = 4788 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 4789 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 4790 &sc->tt.ddp_thres, 0, "DDP threshold"); 4791 4792 sc->tt.rx_coalesce = 1; 4793 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", 4794 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); 4795 } 4796 #endif 4797 4798 4799 return (0); 4800 } 4801 4802 static int 4803 cxgbe_sysctls(struct port_info *pi) 4804 { 4805 struct sysctl_ctx_list *ctx; 4806 struct sysctl_oid *oid; 4807 struct sysctl_oid_list *children; 4808 struct adapter *sc = pi->adapter; 4809 4810 ctx = device_get_sysctl_ctx(pi->dev); 4811 4812 /* 4813 * dev.cxgbe.X. 4814 */ 4815 oid = device_get_sysctl_tree(pi->dev); 4816 children = SYSCTL_CHILDREN(oid); 4817 4818 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | 4819 CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); 4820 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { 4821 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 4822 CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I", 4823 "PHY temperature (in Celsius)"); 4824 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", 4825 CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I", 4826 "PHY firmware version"); 4827 } 4828 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 4829 &pi->nrxq, 0, "# of rx queues"); 4830 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 4831 &pi->ntxq, 0, "# of tx queues"); 4832 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 4833 &pi->first_rxq, 0, "index of first rx queue"); 4834 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 4835 &pi->first_txq, 0, "index of first tx queue"); 4836 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT | 4837 CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU", 4838 "Reserve queue 0 for non-flowid packets"); 4839 4840 #ifdef TCP_OFFLOAD 4841 if (is_offload(sc)) { 4842 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 4843 &pi->nofldrxq, 0, 4844 "# of rx queues for offloaded TCP connections"); 4845 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 4846 &pi->nofldtxq, 0, 4847 "# of tx queues for offloaded TCP connections"); 4848 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 4849 CTLFLAG_RD, &pi->first_ofld_rxq, 0, 4850 "index of first TOE rx queue"); 4851 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 4852 CTLFLAG_RD, &pi->first_ofld_txq, 0, 4853 "index of first TOE tx queue"); 4854 } 4855 #endif 4856 #ifdef DEV_NETMAP 4857 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD, 4858 &pi->nnmrxq, 0, "# of rx queues for netmap"); 4859 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD, 4860 &pi->nnmtxq, 0, "# of tx queues for netmap"); 4861 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq", 4862 CTLFLAG_RD, &pi->first_nm_rxq, 0, 4863 "index of first netmap rx queue"); 4864 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq", 4865 CTLFLAG_RD, &pi->first_nm_txq, 0, 4866 "index of first netmap tx queue"); 4867 #endif 4868 4869 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 4870 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 4871 "holdoff timer index"); 4872 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 4873 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 4874 "holdoff packet counter index"); 4875 4876 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 4877 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 4878 "rx queue size"); 4879 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 4880 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 4881 "tx queue size"); 4882 4883 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings", 4884 CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings, 4885 "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)"); 4886 4887 /* 4888 * dev.cxgbe.X.stats. 4889 */ 4890 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 4891 NULL, "port statistics"); 4892 children = SYSCTL_CHILDREN(oid); 4893 4894 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 4895 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 4896 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ 4897 sysctl_handle_t4_reg64, "QU", desc) 4898 4899 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 4900 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 4901 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 4902 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 4903 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 4904 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 4905 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 4906 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 4907 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 4908 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 4909 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 4910 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 4911 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 4912 "# of tx frames in this range", 4913 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 4914 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 4915 "# of tx frames in this range", 4916 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 4917 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 4918 "# of tx frames in this range", 4919 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 4920 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 4921 "# of tx frames in this range", 4922 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 4923 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 4924 "# of tx frames in this range", 4925 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 4926 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 4927 "# of tx frames in this range", 4928 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 4929 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 4930 "# of tx frames in this range", 4931 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 4932 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 4933 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 4934 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 4935 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 4936 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 4937 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 4938 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 4939 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 4940 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 4941 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 4942 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 4943 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 4944 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 4945 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 4946 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 4947 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 4948 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 4949 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 4950 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 4951 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 4952 4953 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 4954 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 4955 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 4956 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 4957 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 4958 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 4959 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 4960 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 4961 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 4962 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 4963 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 4964 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 4965 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 4966 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 4967 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 4968 "# of frames received with bad FCS", 4969 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 4970 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 4971 "# of frames received with length error", 4972 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 4973 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 4974 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 4975 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 4976 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 4977 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 4978 "# of rx frames in this range", 4979 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 4980 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 4981 "# of rx frames in this range", 4982 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 4983 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 4984 "# of rx frames in this range", 4985 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 4986 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 4987 "# of rx frames in this range", 4988 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 4989 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 4990 "# of rx frames in this range", 4991 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 4992 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 4993 "# of rx frames in this range", 4994 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 4995 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 4996 "# of rx frames in this range", 4997 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 4998 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 4999 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 5000 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 5001 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 5002 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 5003 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 5004 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 5005 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 5006 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 5007 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 5008 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 5009 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 5010 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 5011 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 5012 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 5013 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 5014 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 5015 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 5016 5017 #undef SYSCTL_ADD_T4_REG64 5018 5019 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 5020 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 5021 &pi->stats.name, desc) 5022 5023 /* We get these from port_stats and they may be stale by upto 1s */ 5024 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 5025 "# drops due to buffer-group 0 overflows"); 5026 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 5027 "# drops due to buffer-group 1 overflows"); 5028 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 5029 "# drops due to buffer-group 2 overflows"); 5030 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 5031 "# drops due to buffer-group 3 overflows"); 5032 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 5033 "# of buffer-group 0 truncated packets"); 5034 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 5035 "# of buffer-group 1 truncated packets"); 5036 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 5037 "# of buffer-group 2 truncated packets"); 5038 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 5039 "# of buffer-group 3 truncated packets"); 5040 5041 #undef SYSCTL_ADD_T4_PORTSTAT 5042 5043 return (0); 5044 } 5045 5046 static int 5047 sysctl_int_array(SYSCTL_HANDLER_ARGS) 5048 { 5049 int rc, *i; 5050 struct sbuf sb; 5051 5052 sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND); 5053 for (i = arg1; arg2; arg2 -= sizeof(int), i++) 5054 sbuf_printf(&sb, "%d ", *i); 5055 sbuf_trim(&sb); 5056 sbuf_finish(&sb); 5057 rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 5058 sbuf_delete(&sb); 5059 return (rc); 5060 } 5061 5062 static int 5063 sysctl_bitfield(SYSCTL_HANDLER_ARGS) 5064 { 5065 int rc; 5066 struct sbuf *sb; 5067 5068 rc = sysctl_wire_old_buffer(req, 0); 5069 if (rc != 0) 5070 return(rc); 5071 5072 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5073 if (sb == NULL) 5074 return (ENOMEM); 5075 5076 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 5077 rc = sbuf_finish(sb); 5078 sbuf_delete(sb); 5079 5080 return (rc); 5081 } 5082 5083 static int 5084 sysctl_btphy(SYSCTL_HANDLER_ARGS) 5085 { 5086 struct port_info *pi = arg1; 5087 int op = arg2; 5088 struct adapter *sc = pi->adapter; 5089 u_int v; 5090 int rc; 5091 5092 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt"); 5093 if (rc) 5094 return (rc); 5095 /* XXX: magic numbers */ 5096 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, 5097 &v); 5098 end_synchronized_op(sc, 0); 5099 if (rc) 5100 return (rc); 5101 if (op == 0) 5102 v /= 256; 5103 5104 rc = sysctl_handle_int(oidp, &v, 0, req); 5105 return (rc); 5106 } 5107 5108 static int 5109 sysctl_noflowq(SYSCTL_HANDLER_ARGS) 5110 { 5111 struct port_info *pi = arg1; 5112 int rc, val; 5113 5114 val = pi->rsrv_noflowq; 5115 rc = sysctl_handle_int(oidp, &val, 0, req); 5116 if (rc != 0 || req->newptr == NULL) 5117 return (rc); 5118 5119 if ((val >= 1) && (pi->ntxq > 1)) 5120 pi->rsrv_noflowq = 1; 5121 else 5122 pi->rsrv_noflowq = 0; 5123 5124 return (rc); 5125 } 5126 5127 static int 5128 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 5129 { 5130 struct port_info *pi = arg1; 5131 struct adapter *sc = pi->adapter; 5132 int idx, rc, i; 5133 struct sge_rxq *rxq; 5134 #ifdef TCP_OFFLOAD 5135 struct sge_ofld_rxq *ofld_rxq; 5136 #endif 5137 uint8_t v; 5138 5139 idx = pi->tmr_idx; 5140 5141 rc = sysctl_handle_int(oidp, &idx, 0, req); 5142 if (rc != 0 || req->newptr == NULL) 5143 return (rc); 5144 5145 if (idx < 0 || idx >= SGE_NTIMERS) 5146 return (EINVAL); 5147 5148 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5149 "t4tmr"); 5150 if (rc) 5151 return (rc); 5152 5153 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 5154 for_each_rxq(pi, i, rxq) { 5155 #ifdef atomic_store_rel_8 5156 atomic_store_rel_8(&rxq->iq.intr_params, v); 5157 #else 5158 rxq->iq.intr_params = v; 5159 #endif 5160 } 5161 #ifdef TCP_OFFLOAD 5162 for_each_ofld_rxq(pi, i, ofld_rxq) { 5163 #ifdef atomic_store_rel_8 5164 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); 5165 #else 5166 ofld_rxq->iq.intr_params = v; 5167 #endif 5168 } 5169 #endif 5170 pi->tmr_idx = idx; 5171 5172 end_synchronized_op(sc, LOCK_HELD); 5173 return (0); 5174 } 5175 5176 static int 5177 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 5178 { 5179 struct port_info *pi = arg1; 5180 struct adapter *sc = pi->adapter; 5181 int idx, rc; 5182 5183 idx = pi->pktc_idx; 5184 5185 rc = sysctl_handle_int(oidp, &idx, 0, req); 5186 if (rc != 0 || req->newptr == NULL) 5187 return (rc); 5188 5189 if (idx < -1 || idx >= SGE_NCOUNTERS) 5190 return (EINVAL); 5191 5192 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5193 "t4pktc"); 5194 if (rc) 5195 return (rc); 5196 5197 if (pi->flags & PORT_INIT_DONE) 5198 rc = EBUSY; /* cannot be changed once the queues are created */ 5199 else 5200 pi->pktc_idx = idx; 5201 5202 end_synchronized_op(sc, LOCK_HELD); 5203 return (rc); 5204 } 5205 5206 static int 5207 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 5208 { 5209 struct port_info *pi = arg1; 5210 struct adapter *sc = pi->adapter; 5211 int qsize, rc; 5212 5213 qsize = pi->qsize_rxq; 5214 5215 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5216 if (rc != 0 || req->newptr == NULL) 5217 return (rc); 5218 5219 if (qsize < 128 || (qsize & 7)) 5220 return (EINVAL); 5221 5222 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5223 "t4rxqs"); 5224 if (rc) 5225 return (rc); 5226 5227 if (pi->flags & PORT_INIT_DONE) 5228 rc = EBUSY; /* cannot be changed once the queues are created */ 5229 else 5230 pi->qsize_rxq = qsize; 5231 5232 end_synchronized_op(sc, LOCK_HELD); 5233 return (rc); 5234 } 5235 5236 static int 5237 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 5238 { 5239 struct port_info *pi = arg1; 5240 struct adapter *sc = pi->adapter; 5241 int qsize, rc; 5242 5243 qsize = pi->qsize_txq; 5244 5245 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5246 if (rc != 0 || req->newptr == NULL) 5247 return (rc); 5248 5249 /* bufring size must be powerof2 */ 5250 if (qsize < 128 || !powerof2(qsize)) 5251 return (EINVAL); 5252 5253 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5254 "t4txqs"); 5255 if (rc) 5256 return (rc); 5257 5258 if (pi->flags & PORT_INIT_DONE) 5259 rc = EBUSY; /* cannot be changed once the queues are created */ 5260 else 5261 pi->qsize_txq = qsize; 5262 5263 end_synchronized_op(sc, LOCK_HELD); 5264 return (rc); 5265 } 5266 5267 static int 5268 sysctl_pause_settings(SYSCTL_HANDLER_ARGS) 5269 { 5270 struct port_info *pi = arg1; 5271 struct adapter *sc = pi->adapter; 5272 struct link_config *lc = &pi->link_cfg; 5273 int rc; 5274 5275 if (req->newptr == NULL) { 5276 struct sbuf *sb; 5277 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX"; 5278 5279 rc = sysctl_wire_old_buffer(req, 0); 5280 if (rc != 0) 5281 return(rc); 5282 5283 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5284 if (sb == NULL) 5285 return (ENOMEM); 5286 5287 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits); 5288 rc = sbuf_finish(sb); 5289 sbuf_delete(sb); 5290 } else { 5291 char s[2]; 5292 int n; 5293 5294 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX)); 5295 s[1] = 0; 5296 5297 rc = sysctl_handle_string(oidp, s, sizeof(s), req); 5298 if (rc != 0) 5299 return(rc); 5300 5301 if (s[1] != 0) 5302 return (EINVAL); 5303 if (s[0] < '0' || s[0] > '9') 5304 return (EINVAL); /* not a number */ 5305 n = s[0] - '0'; 5306 if (n & ~(PAUSE_TX | PAUSE_RX)) 5307 return (EINVAL); /* some other bit is set too */ 5308 5309 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE"); 5310 if (rc) 5311 return (rc); 5312 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) { 5313 int link_ok = lc->link_ok; 5314 5315 lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX); 5316 lc->requested_fc |= n; 5317 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc); 5318 lc->link_ok = link_ok; /* restore */ 5319 } 5320 end_synchronized_op(sc, 0); 5321 } 5322 5323 return (rc); 5324 } 5325 5326 static int 5327 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 5328 { 5329 struct adapter *sc = arg1; 5330 int reg = arg2; 5331 uint64_t val; 5332 5333 val = t4_read_reg64(sc, reg); 5334 5335 return (sysctl_handle_64(oidp, &val, 0, req)); 5336 } 5337 5338 static int 5339 sysctl_temperature(SYSCTL_HANDLER_ARGS) 5340 { 5341 struct adapter *sc = arg1; 5342 int rc, t; 5343 uint32_t param, val; 5344 5345 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); 5346 if (rc) 5347 return (rc); 5348 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 5349 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 5350 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); 5351 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 5352 end_synchronized_op(sc, 0); 5353 if (rc) 5354 return (rc); 5355 5356 /* unknown is returned as 0 but we display -1 in that case */ 5357 t = val == 0 ? -1 : val; 5358 5359 rc = sysctl_handle_int(oidp, &t, 0, req); 5360 return (rc); 5361 } 5362 5363 #ifdef SBUF_DRAIN 5364 static int 5365 sysctl_cctrl(SYSCTL_HANDLER_ARGS) 5366 { 5367 struct adapter *sc = arg1; 5368 struct sbuf *sb; 5369 int rc, i; 5370 uint16_t incr[NMTUS][NCCTRL_WIN]; 5371 static const char *dec_fac[] = { 5372 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 5373 "0.9375" 5374 }; 5375 5376 rc = sysctl_wire_old_buffer(req, 0); 5377 if (rc != 0) 5378 return (rc); 5379 5380 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5381 if (sb == NULL) 5382 return (ENOMEM); 5383 5384 t4_read_cong_tbl(sc, incr); 5385 5386 for (i = 0; i < NCCTRL_WIN; ++i) { 5387 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 5388 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 5389 incr[5][i], incr[6][i], incr[7][i]); 5390 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 5391 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 5392 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 5393 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 5394 } 5395 5396 rc = sbuf_finish(sb); 5397 sbuf_delete(sb); 5398 5399 return (rc); 5400 } 5401 5402 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 5403 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 5404 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 5405 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 5406 }; 5407 5408 static int 5409 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 5410 { 5411 struct adapter *sc = arg1; 5412 struct sbuf *sb; 5413 int rc, i, n, qid = arg2; 5414 uint32_t *buf, *p; 5415 char *qtype; 5416 u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5; 5417 5418 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 5419 ("%s: bad qid %d\n", __func__, qid)); 5420 5421 if (qid < CIM_NUM_IBQ) { 5422 /* inbound queue */ 5423 qtype = "IBQ"; 5424 n = 4 * CIM_IBQ_SIZE; 5425 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5426 rc = t4_read_cim_ibq(sc, qid, buf, n); 5427 } else { 5428 /* outbound queue */ 5429 qtype = "OBQ"; 5430 qid -= CIM_NUM_IBQ; 5431 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 5432 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5433 rc = t4_read_cim_obq(sc, qid, buf, n); 5434 } 5435 5436 if (rc < 0) { 5437 rc = -rc; 5438 goto done; 5439 } 5440 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 5441 5442 rc = sysctl_wire_old_buffer(req, 0); 5443 if (rc != 0) 5444 goto done; 5445 5446 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5447 if (sb == NULL) { 5448 rc = ENOMEM; 5449 goto done; 5450 } 5451 5452 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 5453 for (i = 0, p = buf; i < n; i += 16, p += 4) 5454 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 5455 p[2], p[3]); 5456 5457 rc = sbuf_finish(sb); 5458 sbuf_delete(sb); 5459 done: 5460 free(buf, M_CXGBE); 5461 return (rc); 5462 } 5463 5464 static int 5465 sysctl_cim_la(SYSCTL_HANDLER_ARGS) 5466 { 5467 struct adapter *sc = arg1; 5468 u_int cfg; 5469 struct sbuf *sb; 5470 uint32_t *buf, *p; 5471 int rc; 5472 5473 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 5474 if (rc != 0) 5475 return (rc); 5476 5477 rc = sysctl_wire_old_buffer(req, 0); 5478 if (rc != 0) 5479 return (rc); 5480 5481 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5482 if (sb == NULL) 5483 return (ENOMEM); 5484 5485 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 5486 M_ZERO | M_WAITOK); 5487 5488 rc = -t4_cim_read_la(sc, buf, NULL); 5489 if (rc != 0) 5490 goto done; 5491 5492 sbuf_printf(sb, "Status Data PC%s", 5493 cfg & F_UPDBGLACAPTPCONLY ? "" : 5494 " LS0Stat LS0Addr LS0Data"); 5495 5496 KASSERT((sc->params.cim_la_size & 7) == 0, 5497 ("%s: p will walk off the end of buf", __func__)); 5498 5499 for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) { 5500 if (cfg & F_UPDBGLACAPTPCONLY) { 5501 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 5502 p[6], p[7]); 5503 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 5504 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 5505 p[4] & 0xff, p[5] >> 8); 5506 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 5507 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5508 p[1] & 0xf, p[2] >> 4); 5509 } else { 5510 sbuf_printf(sb, 5511 "\n %02x %x%07x %x%07x %08x %08x " 5512 "%08x%08x%08x%08x", 5513 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5514 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 5515 p[6], p[7]); 5516 } 5517 } 5518 5519 rc = sbuf_finish(sb); 5520 sbuf_delete(sb); 5521 done: 5522 free(buf, M_CXGBE); 5523 return (rc); 5524 } 5525 5526 static int 5527 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) 5528 { 5529 struct adapter *sc = arg1; 5530 u_int i; 5531 struct sbuf *sb; 5532 uint32_t *buf, *p; 5533 int rc; 5534 5535 rc = sysctl_wire_old_buffer(req, 0); 5536 if (rc != 0) 5537 return (rc); 5538 5539 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5540 if (sb == NULL) 5541 return (ENOMEM); 5542 5543 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, 5544 M_ZERO | M_WAITOK); 5545 5546 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); 5547 p = buf; 5548 5549 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5550 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], 5551 p[1], p[0]); 5552 } 5553 5554 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); 5555 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5556 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", 5557 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, 5558 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, 5559 (p[1] >> 2) | ((p[2] & 3) << 30), 5560 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, 5561 p[0] & 1); 5562 } 5563 5564 rc = sbuf_finish(sb); 5565 sbuf_delete(sb); 5566 free(buf, M_CXGBE); 5567 return (rc); 5568 } 5569 5570 static int 5571 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) 5572 { 5573 struct adapter *sc = arg1; 5574 u_int i; 5575 struct sbuf *sb; 5576 uint32_t *buf, *p; 5577 int rc; 5578 5579 rc = sysctl_wire_old_buffer(req, 0); 5580 if (rc != 0) 5581 return (rc); 5582 5583 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5584 if (sb == NULL) 5585 return (ENOMEM); 5586 5587 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, 5588 M_ZERO | M_WAITOK); 5589 5590 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); 5591 p = buf; 5592 5593 sbuf_printf(sb, "Cntl ID DataBE Addr Data"); 5594 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5595 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", 5596 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, 5597 p[4], p[3], p[2], p[1], p[0]); 5598 } 5599 5600 sbuf_printf(sb, "\n\nCntl ID Data"); 5601 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5602 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", 5603 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); 5604 } 5605 5606 rc = sbuf_finish(sb); 5607 sbuf_delete(sb); 5608 free(buf, M_CXGBE); 5609 return (rc); 5610 } 5611 5612 static int 5613 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 5614 { 5615 struct adapter *sc = arg1; 5616 struct sbuf *sb; 5617 int rc, i; 5618 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5619 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5620 uint16_t thres[CIM_NUM_IBQ]; 5621 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 5622 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 5623 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 5624 5625 if (is_t4(sc)) { 5626 cim_num_obq = CIM_NUM_OBQ; 5627 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 5628 obq_rdaddr = A_UP_OBQ_0_REALADDR; 5629 } else { 5630 cim_num_obq = CIM_NUM_OBQ_T5; 5631 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 5632 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 5633 } 5634 nq = CIM_NUM_IBQ + cim_num_obq; 5635 5636 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 5637 if (rc == 0) 5638 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 5639 if (rc != 0) 5640 return (rc); 5641 5642 t4_read_cimq_cfg(sc, base, size, thres); 5643 5644 rc = sysctl_wire_old_buffer(req, 0); 5645 if (rc != 0) 5646 return (rc); 5647 5648 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5649 if (sb == NULL) 5650 return (ENOMEM); 5651 5652 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 5653 5654 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 5655 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 5656 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 5657 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5658 G_QUEREMFLITS(p[2]) * 16); 5659 for ( ; i < nq; i++, p += 4, wr += 2) 5660 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 5661 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 5662 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5663 G_QUEREMFLITS(p[2]) * 16); 5664 5665 rc = sbuf_finish(sb); 5666 sbuf_delete(sb); 5667 5668 return (rc); 5669 } 5670 5671 static int 5672 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 5673 { 5674 struct adapter *sc = arg1; 5675 struct sbuf *sb; 5676 int rc; 5677 struct tp_cpl_stats stats; 5678 5679 rc = sysctl_wire_old_buffer(req, 0); 5680 if (rc != 0) 5681 return (rc); 5682 5683 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5684 if (sb == NULL) 5685 return (ENOMEM); 5686 5687 t4_tp_get_cpl_stats(sc, &stats); 5688 5689 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 5690 "channel 3\n"); 5691 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 5692 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 5693 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 5694 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 5695 5696 rc = sbuf_finish(sb); 5697 sbuf_delete(sb); 5698 5699 return (rc); 5700 } 5701 5702 static int 5703 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 5704 { 5705 struct adapter *sc = arg1; 5706 struct sbuf *sb; 5707 int rc; 5708 struct tp_usm_stats stats; 5709 5710 rc = sysctl_wire_old_buffer(req, 0); 5711 if (rc != 0) 5712 return(rc); 5713 5714 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5715 if (sb == NULL) 5716 return (ENOMEM); 5717 5718 t4_get_usm_stats(sc, &stats); 5719 5720 sbuf_printf(sb, "Frames: %u\n", stats.frames); 5721 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 5722 sbuf_printf(sb, "Drops: %u", stats.drops); 5723 5724 rc = sbuf_finish(sb); 5725 sbuf_delete(sb); 5726 5727 return (rc); 5728 } 5729 5730 const char *devlog_level_strings[] = { 5731 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 5732 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 5733 [FW_DEVLOG_LEVEL_ERR] = "ERR", 5734 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 5735 [FW_DEVLOG_LEVEL_INFO] = "INFO", 5736 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 5737 }; 5738 5739 const char *devlog_facility_strings[] = { 5740 [FW_DEVLOG_FACILITY_CORE] = "CORE", 5741 [FW_DEVLOG_FACILITY_CF] = "CF", 5742 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 5743 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 5744 [FW_DEVLOG_FACILITY_RES] = "RES", 5745 [FW_DEVLOG_FACILITY_HW] = "HW", 5746 [FW_DEVLOG_FACILITY_FLR] = "FLR", 5747 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 5748 [FW_DEVLOG_FACILITY_PHY] = "PHY", 5749 [FW_DEVLOG_FACILITY_MAC] = "MAC", 5750 [FW_DEVLOG_FACILITY_PORT] = "PORT", 5751 [FW_DEVLOG_FACILITY_VI] = "VI", 5752 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 5753 [FW_DEVLOG_FACILITY_ACL] = "ACL", 5754 [FW_DEVLOG_FACILITY_TM] = "TM", 5755 [FW_DEVLOG_FACILITY_QFC] = "QFC", 5756 [FW_DEVLOG_FACILITY_DCB] = "DCB", 5757 [FW_DEVLOG_FACILITY_ETH] = "ETH", 5758 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 5759 [FW_DEVLOG_FACILITY_RI] = "RI", 5760 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 5761 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 5762 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 5763 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 5764 }; 5765 5766 static int 5767 sysctl_devlog(SYSCTL_HANDLER_ARGS) 5768 { 5769 struct adapter *sc = arg1; 5770 struct devlog_params *dparams = &sc->params.devlog; 5771 struct fw_devlog_e *buf, *e; 5772 int i, j, rc, nentries, first = 0, m; 5773 struct sbuf *sb; 5774 uint64_t ftstamp = UINT64_MAX; 5775 5776 if (dparams->start == 0) { 5777 dparams->memtype = FW_MEMTYPE_EDC0; 5778 dparams->start = 0x84000; 5779 dparams->size = 32768; 5780 } 5781 5782 nentries = dparams->size / sizeof(struct fw_devlog_e); 5783 5784 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 5785 if (buf == NULL) 5786 return (ENOMEM); 5787 5788 m = fwmtype_to_hwmtype(dparams->memtype); 5789 rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf); 5790 if (rc != 0) 5791 goto done; 5792 5793 for (i = 0; i < nentries; i++) { 5794 e = &buf[i]; 5795 5796 if (e->timestamp == 0) 5797 break; /* end */ 5798 5799 e->timestamp = be64toh(e->timestamp); 5800 e->seqno = be32toh(e->seqno); 5801 for (j = 0; j < 8; j++) 5802 e->params[j] = be32toh(e->params[j]); 5803 5804 if (e->timestamp < ftstamp) { 5805 ftstamp = e->timestamp; 5806 first = i; 5807 } 5808 } 5809 5810 if (buf[first].timestamp == 0) 5811 goto done; /* nothing in the log */ 5812 5813 rc = sysctl_wire_old_buffer(req, 0); 5814 if (rc != 0) 5815 goto done; 5816 5817 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5818 if (sb == NULL) { 5819 rc = ENOMEM; 5820 goto done; 5821 } 5822 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 5823 "Seq#", "Tstamp", "Level", "Facility", "Message"); 5824 5825 i = first; 5826 do { 5827 e = &buf[i]; 5828 if (e->timestamp == 0) 5829 break; /* end */ 5830 5831 sbuf_printf(sb, "%10d %15ju %8s %8s ", 5832 e->seqno, e->timestamp, 5833 (e->level < nitems(devlog_level_strings) ? 5834 devlog_level_strings[e->level] : "UNKNOWN"), 5835 (e->facility < nitems(devlog_facility_strings) ? 5836 devlog_facility_strings[e->facility] : "UNKNOWN")); 5837 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 5838 e->params[2], e->params[3], e->params[4], 5839 e->params[5], e->params[6], e->params[7]); 5840 5841 if (++i == nentries) 5842 i = 0; 5843 } while (i != first); 5844 5845 rc = sbuf_finish(sb); 5846 sbuf_delete(sb); 5847 done: 5848 free(buf, M_CXGBE); 5849 return (rc); 5850 } 5851 5852 static int 5853 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 5854 { 5855 struct adapter *sc = arg1; 5856 struct sbuf *sb; 5857 int rc; 5858 struct tp_fcoe_stats stats[4]; 5859 5860 rc = sysctl_wire_old_buffer(req, 0); 5861 if (rc != 0) 5862 return (rc); 5863 5864 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5865 if (sb == NULL) 5866 return (ENOMEM); 5867 5868 t4_get_fcoe_stats(sc, 0, &stats[0]); 5869 t4_get_fcoe_stats(sc, 1, &stats[1]); 5870 t4_get_fcoe_stats(sc, 2, &stats[2]); 5871 t4_get_fcoe_stats(sc, 3, &stats[3]); 5872 5873 sbuf_printf(sb, " channel 0 channel 1 " 5874 "channel 2 channel 3\n"); 5875 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 5876 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 5877 stats[3].octetsDDP); 5878 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 5879 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 5880 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 5881 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 5882 stats[3].framesDrop); 5883 5884 rc = sbuf_finish(sb); 5885 sbuf_delete(sb); 5886 5887 return (rc); 5888 } 5889 5890 static int 5891 sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 5892 { 5893 struct adapter *sc = arg1; 5894 struct sbuf *sb; 5895 int rc, i; 5896 unsigned int map, kbps, ipg, mode; 5897 unsigned int pace_tab[NTX_SCHED]; 5898 5899 rc = sysctl_wire_old_buffer(req, 0); 5900 if (rc != 0) 5901 return (rc); 5902 5903 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5904 if (sb == NULL) 5905 return (ENOMEM); 5906 5907 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 5908 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 5909 t4_read_pace_tbl(sc, pace_tab); 5910 5911 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 5912 "Class IPG (0.1 ns) Flow IPG (us)"); 5913 5914 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 5915 t4_get_tx_sched(sc, i, &kbps, &ipg); 5916 sbuf_printf(sb, "\n %u %-5s %u ", i, 5917 (mode & (1 << i)) ? "flow" : "class", map & 3); 5918 if (kbps) 5919 sbuf_printf(sb, "%9u ", kbps); 5920 else 5921 sbuf_printf(sb, " disabled "); 5922 5923 if (ipg) 5924 sbuf_printf(sb, "%13u ", ipg); 5925 else 5926 sbuf_printf(sb, " disabled "); 5927 5928 if (pace_tab[i]) 5929 sbuf_printf(sb, "%10u", pace_tab[i]); 5930 else 5931 sbuf_printf(sb, " disabled"); 5932 } 5933 5934 rc = sbuf_finish(sb); 5935 sbuf_delete(sb); 5936 5937 return (rc); 5938 } 5939 5940 static int 5941 sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 5942 { 5943 struct adapter *sc = arg1; 5944 struct sbuf *sb; 5945 int rc, i, j; 5946 uint64_t *p0, *p1; 5947 struct lb_port_stats s[2]; 5948 static const char *stat_name[] = { 5949 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 5950 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 5951 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 5952 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 5953 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 5954 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 5955 "BG2FramesTrunc:", "BG3FramesTrunc:" 5956 }; 5957 5958 rc = sysctl_wire_old_buffer(req, 0); 5959 if (rc != 0) 5960 return (rc); 5961 5962 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5963 if (sb == NULL) 5964 return (ENOMEM); 5965 5966 memset(s, 0, sizeof(s)); 5967 5968 for (i = 0; i < 4; i += 2) { 5969 t4_get_lb_stats(sc, i, &s[0]); 5970 t4_get_lb_stats(sc, i + 1, &s[1]); 5971 5972 p0 = &s[0].octets; 5973 p1 = &s[1].octets; 5974 sbuf_printf(sb, "%s Loopback %u" 5975 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 5976 5977 for (j = 0; j < nitems(stat_name); j++) 5978 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 5979 *p0++, *p1++); 5980 } 5981 5982 rc = sbuf_finish(sb); 5983 sbuf_delete(sb); 5984 5985 return (rc); 5986 } 5987 5988 static int 5989 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) 5990 { 5991 int rc = 0; 5992 struct port_info *pi = arg1; 5993 struct sbuf *sb; 5994 static const char *linkdnreasons[] = { 5995 "non-specific", "remote fault", "autoneg failed", "reserved3", 5996 "PHY overheated", "unknown", "rx los", "reserved7" 5997 }; 5998 5999 rc = sysctl_wire_old_buffer(req, 0); 6000 if (rc != 0) 6001 return(rc); 6002 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); 6003 if (sb == NULL) 6004 return (ENOMEM); 6005 6006 if (pi->linkdnrc < 0) 6007 sbuf_printf(sb, "n/a"); 6008 else if (pi->linkdnrc < nitems(linkdnreasons)) 6009 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]); 6010 else 6011 sbuf_printf(sb, "%d", pi->linkdnrc); 6012 6013 rc = sbuf_finish(sb); 6014 sbuf_delete(sb); 6015 6016 return (rc); 6017 } 6018 6019 struct mem_desc { 6020 unsigned int base; 6021 unsigned int limit; 6022 unsigned int idx; 6023 }; 6024 6025 static int 6026 mem_desc_cmp(const void *a, const void *b) 6027 { 6028 return ((const struct mem_desc *)a)->base - 6029 ((const struct mem_desc *)b)->base; 6030 } 6031 6032 static void 6033 mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 6034 unsigned int to) 6035 { 6036 unsigned int size; 6037 6038 size = to - from + 1; 6039 if (size == 0) 6040 return; 6041 6042 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 6043 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 6044 } 6045 6046 static int 6047 sysctl_meminfo(SYSCTL_HANDLER_ARGS) 6048 { 6049 struct adapter *sc = arg1; 6050 struct sbuf *sb; 6051 int rc, i, n; 6052 uint32_t lo, hi, used, alloc; 6053 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 6054 static const char *region[] = { 6055 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 6056 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 6057 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 6058 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 6059 "RQUDP region:", "PBL region:", "TXPBL region:", 6060 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 6061 "On-chip queues:" 6062 }; 6063 struct mem_desc avail[4]; 6064 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 6065 struct mem_desc *md = mem; 6066 6067 rc = sysctl_wire_old_buffer(req, 0); 6068 if (rc != 0) 6069 return (rc); 6070 6071 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6072 if (sb == NULL) 6073 return (ENOMEM); 6074 6075 for (i = 0; i < nitems(mem); i++) { 6076 mem[i].limit = 0; 6077 mem[i].idx = i; 6078 } 6079 6080 /* Find and sort the populated memory ranges */ 6081 i = 0; 6082 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 6083 if (lo & F_EDRAM0_ENABLE) { 6084 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 6085 avail[i].base = G_EDRAM0_BASE(hi) << 20; 6086 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 6087 avail[i].idx = 0; 6088 i++; 6089 } 6090 if (lo & F_EDRAM1_ENABLE) { 6091 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 6092 avail[i].base = G_EDRAM1_BASE(hi) << 20; 6093 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 6094 avail[i].idx = 1; 6095 i++; 6096 } 6097 if (lo & F_EXT_MEM_ENABLE) { 6098 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 6099 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 6100 avail[i].limit = avail[i].base + 6101 (G_EXT_MEM_SIZE(hi) << 20); 6102 avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */ 6103 i++; 6104 } 6105 if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) { 6106 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 6107 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 6108 avail[i].limit = avail[i].base + 6109 (G_EXT_MEM1_SIZE(hi) << 20); 6110 avail[i].idx = 4; 6111 i++; 6112 } 6113 if (!i) /* no memory available */ 6114 return 0; 6115 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 6116 6117 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 6118 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 6119 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 6120 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 6121 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 6122 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 6123 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 6124 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 6125 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 6126 6127 /* the next few have explicit upper bounds */ 6128 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 6129 md->limit = md->base - 1 + 6130 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 6131 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 6132 md++; 6133 6134 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 6135 md->limit = md->base - 1 + 6136 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 6137 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 6138 md++; 6139 6140 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6141 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 6142 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 6143 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 6144 } else { 6145 md->base = 0; 6146 md->idx = nitems(region); /* hide it */ 6147 } 6148 md++; 6149 6150 #define ulp_region(reg) \ 6151 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 6152 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 6153 6154 ulp_region(RX_ISCSI); 6155 ulp_region(RX_TDDP); 6156 ulp_region(TX_TPT); 6157 ulp_region(RX_STAG); 6158 ulp_region(RX_RQ); 6159 ulp_region(RX_RQUDP); 6160 ulp_region(RX_PBL); 6161 ulp_region(TX_PBL); 6162 #undef ulp_region 6163 6164 md->base = 0; 6165 md->idx = nitems(region); 6166 if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) { 6167 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); 6168 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc, 6169 A_SGE_DBVFIFO_SIZE))) << 2) - 1; 6170 } 6171 md++; 6172 6173 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 6174 md->limit = md->base + sc->tids.ntids - 1; 6175 md++; 6176 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 6177 md->limit = md->base + sc->tids.ntids - 1; 6178 md++; 6179 6180 md->base = sc->vres.ocq.start; 6181 if (sc->vres.ocq.size) 6182 md->limit = md->base + sc->vres.ocq.size - 1; 6183 else 6184 md->idx = nitems(region); /* hide it */ 6185 md++; 6186 6187 /* add any address-space holes, there can be up to 3 */ 6188 for (n = 0; n < i - 1; n++) 6189 if (avail[n].limit < avail[n + 1].base) 6190 (md++)->base = avail[n].limit; 6191 if (avail[n].limit) 6192 (md++)->base = avail[n].limit; 6193 6194 n = md - mem; 6195 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 6196 6197 for (lo = 0; lo < i; lo++) 6198 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 6199 avail[lo].limit - 1); 6200 6201 sbuf_printf(sb, "\n"); 6202 for (i = 0; i < n; i++) { 6203 if (mem[i].idx >= nitems(region)) 6204 continue; /* skip holes */ 6205 if (!mem[i].limit) 6206 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 6207 mem_region_show(sb, region[mem[i].idx], mem[i].base, 6208 mem[i].limit); 6209 } 6210 6211 sbuf_printf(sb, "\n"); 6212 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 6213 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 6214 mem_region_show(sb, "uP RAM:", lo, hi); 6215 6216 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 6217 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 6218 mem_region_show(sb, "uP Extmem2:", lo, hi); 6219 6220 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 6221 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 6222 G_PMRXMAXPAGE(lo), 6223 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 6224 (lo & F_PMRXNUMCHN) ? 2 : 1); 6225 6226 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 6227 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 6228 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 6229 G_PMTXMAXPAGE(lo), 6230 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 6231 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 6232 sbuf_printf(sb, "%u p-structs\n", 6233 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 6234 6235 for (i = 0; i < 4; i++) { 6236 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 6237 if (is_t4(sc)) { 6238 used = G_USED(lo); 6239 alloc = G_ALLOC(lo); 6240 } else { 6241 used = G_T5_USED(lo); 6242 alloc = G_T5_ALLOC(lo); 6243 } 6244 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 6245 i, used, alloc); 6246 } 6247 for (i = 0; i < 4; i++) { 6248 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 6249 if (is_t4(sc)) { 6250 used = G_USED(lo); 6251 alloc = G_ALLOC(lo); 6252 } else { 6253 used = G_T5_USED(lo); 6254 alloc = G_T5_ALLOC(lo); 6255 } 6256 sbuf_printf(sb, 6257 "\nLoopback %d using %u pages out of %u allocated", 6258 i, used, alloc); 6259 } 6260 6261 rc = sbuf_finish(sb); 6262 sbuf_delete(sb); 6263 6264 return (rc); 6265 } 6266 6267 static inline void 6268 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) 6269 { 6270 *mask = x | y; 6271 y = htobe64(y); 6272 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); 6273 } 6274 6275 static int 6276 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) 6277 { 6278 struct adapter *sc = arg1; 6279 struct sbuf *sb; 6280 int rc, i, n; 6281 6282 rc = sysctl_wire_old_buffer(req, 0); 6283 if (rc != 0) 6284 return (rc); 6285 6286 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6287 if (sb == NULL) 6288 return (ENOMEM); 6289 6290 sbuf_printf(sb, 6291 "Idx Ethernet address Mask Vld Ports PF" 6292 " VF Replication P0 P1 P2 P3 ML"); 6293 n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES : 6294 NUM_MPS_T5_CLS_SRAM_L_INSTANCES; 6295 for (i = 0; i < n; i++) { 6296 uint64_t tcamx, tcamy, mask; 6297 uint32_t cls_lo, cls_hi; 6298 uint8_t addr[ETHER_ADDR_LEN]; 6299 6300 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); 6301 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); 6302 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 6303 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 6304 6305 if (tcamx & tcamy) 6306 continue; 6307 6308 tcamxy2valmask(tcamx, tcamy, addr, &mask); 6309 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" 6310 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], 6311 addr[3], addr[4], addr[5], (uintmax_t)mask, 6312 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', 6313 G_PORTMAP(cls_hi), G_PF(cls_lo), 6314 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); 6315 6316 if (cls_lo & F_REPLICATE) { 6317 struct fw_ldst_cmd ldst_cmd; 6318 6319 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 6320 ldst_cmd.op_to_addrspace = 6321 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 6322 F_FW_CMD_REQUEST | F_FW_CMD_READ | 6323 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 6324 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 6325 ldst_cmd.u.mps.fid_ctl = 6326 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 6327 V_FW_LDST_CMD_CTL(i)); 6328 6329 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 6330 "t4mps"); 6331 if (rc) 6332 break; 6333 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 6334 sizeof(ldst_cmd), &ldst_cmd); 6335 end_synchronized_op(sc, 0); 6336 6337 if (rc != 0) { 6338 sbuf_printf(sb, 6339 " ------------ error %3u ------------", rc); 6340 rc = 0; 6341 } else { 6342 sbuf_printf(sb, " %08x %08x %08x %08x", 6343 be32toh(ldst_cmd.u.mps.rplc127_96), 6344 be32toh(ldst_cmd.u.mps.rplc95_64), 6345 be32toh(ldst_cmd.u.mps.rplc63_32), 6346 be32toh(ldst_cmd.u.mps.rplc31_0)); 6347 } 6348 } else 6349 sbuf_printf(sb, "%36s", ""); 6350 6351 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), 6352 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), 6353 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); 6354 } 6355 6356 if (rc) 6357 (void) sbuf_finish(sb); 6358 else 6359 rc = sbuf_finish(sb); 6360 sbuf_delete(sb); 6361 6362 return (rc); 6363 } 6364 6365 static int 6366 sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 6367 { 6368 struct adapter *sc = arg1; 6369 struct sbuf *sb; 6370 int rc; 6371 uint16_t mtus[NMTUS]; 6372 6373 rc = sysctl_wire_old_buffer(req, 0); 6374 if (rc != 0) 6375 return (rc); 6376 6377 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6378 if (sb == NULL) 6379 return (ENOMEM); 6380 6381 t4_read_mtu_tbl(sc, mtus, NULL); 6382 6383 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 6384 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 6385 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 6386 mtus[14], mtus[15]); 6387 6388 rc = sbuf_finish(sb); 6389 sbuf_delete(sb); 6390 6391 return (rc); 6392 } 6393 6394 static int 6395 sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 6396 { 6397 struct adapter *sc = arg1; 6398 struct sbuf *sb; 6399 int rc, i; 6400 uint32_t cnt[PM_NSTATS]; 6401 uint64_t cyc[PM_NSTATS]; 6402 static const char *rx_stats[] = { 6403 "Read:", "Write bypass:", "Write mem:", "Flush:" 6404 }; 6405 static const char *tx_stats[] = { 6406 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:" 6407 }; 6408 6409 rc = sysctl_wire_old_buffer(req, 0); 6410 if (rc != 0) 6411 return (rc); 6412 6413 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6414 if (sb == NULL) 6415 return (ENOMEM); 6416 6417 t4_pmtx_get_stats(sc, cnt, cyc); 6418 sbuf_printf(sb, " Tx pcmds Tx bytes"); 6419 for (i = 0; i < ARRAY_SIZE(tx_stats); i++) 6420 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i], 6421 cyc[i]); 6422 6423 t4_pmrx_get_stats(sc, cnt, cyc); 6424 sbuf_printf(sb, "\n Rx pcmds Rx bytes"); 6425 for (i = 0; i < ARRAY_SIZE(rx_stats); i++) 6426 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i], 6427 cyc[i]); 6428 6429 rc = sbuf_finish(sb); 6430 sbuf_delete(sb); 6431 6432 return (rc); 6433 } 6434 6435 static int 6436 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 6437 { 6438 struct adapter *sc = arg1; 6439 struct sbuf *sb; 6440 int rc; 6441 struct tp_rdma_stats stats; 6442 6443 rc = sysctl_wire_old_buffer(req, 0); 6444 if (rc != 0) 6445 return (rc); 6446 6447 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6448 if (sb == NULL) 6449 return (ENOMEM); 6450 6451 t4_tp_get_rdma_stats(sc, &stats); 6452 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 6453 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 6454 6455 rc = sbuf_finish(sb); 6456 sbuf_delete(sb); 6457 6458 return (rc); 6459 } 6460 6461 static int 6462 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 6463 { 6464 struct adapter *sc = arg1; 6465 struct sbuf *sb; 6466 int rc; 6467 struct tp_tcp_stats v4, v6; 6468 6469 rc = sysctl_wire_old_buffer(req, 0); 6470 if (rc != 0) 6471 return (rc); 6472 6473 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6474 if (sb == NULL) 6475 return (ENOMEM); 6476 6477 t4_tp_get_tcp_stats(sc, &v4, &v6); 6478 sbuf_printf(sb, 6479 " IP IPv6\n"); 6480 sbuf_printf(sb, "OutRsts: %20u %20u\n", 6481 v4.tcpOutRsts, v6.tcpOutRsts); 6482 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 6483 v4.tcpInSegs, v6.tcpInSegs); 6484 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 6485 v4.tcpOutSegs, v6.tcpOutSegs); 6486 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 6487 v4.tcpRetransSegs, v6.tcpRetransSegs); 6488 6489 rc = sbuf_finish(sb); 6490 sbuf_delete(sb); 6491 6492 return (rc); 6493 } 6494 6495 static int 6496 sysctl_tids(SYSCTL_HANDLER_ARGS) 6497 { 6498 struct adapter *sc = arg1; 6499 struct sbuf *sb; 6500 int rc; 6501 struct tid_info *t = &sc->tids; 6502 6503 rc = sysctl_wire_old_buffer(req, 0); 6504 if (rc != 0) 6505 return (rc); 6506 6507 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6508 if (sb == NULL) 6509 return (ENOMEM); 6510 6511 if (t->natids) { 6512 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 6513 t->atids_in_use); 6514 } 6515 6516 if (t->ntids) { 6517 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6518 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 6519 6520 if (b) { 6521 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 6522 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6523 t->ntids - 1); 6524 } else { 6525 sbuf_printf(sb, "TID range: %u-%u", 6526 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6527 t->ntids - 1); 6528 } 6529 } else 6530 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 6531 sbuf_printf(sb, ", in use: %u\n", 6532 atomic_load_acq_int(&t->tids_in_use)); 6533 } 6534 6535 if (t->nstids) { 6536 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 6537 t->stid_base + t->nstids - 1, t->stids_in_use); 6538 } 6539 6540 if (t->nftids) { 6541 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 6542 t->ftid_base + t->nftids - 1); 6543 } 6544 6545 if (t->netids) { 6546 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base, 6547 t->etid_base + t->netids - 1); 6548 } 6549 6550 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 6551 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 6552 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 6553 6554 rc = sbuf_finish(sb); 6555 sbuf_delete(sb); 6556 6557 return (rc); 6558 } 6559 6560 static int 6561 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 6562 { 6563 struct adapter *sc = arg1; 6564 struct sbuf *sb; 6565 int rc; 6566 struct tp_err_stats stats; 6567 6568 rc = sysctl_wire_old_buffer(req, 0); 6569 if (rc != 0) 6570 return (rc); 6571 6572 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6573 if (sb == NULL) 6574 return (ENOMEM); 6575 6576 t4_tp_get_err_stats(sc, &stats); 6577 6578 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6579 "channel 3\n"); 6580 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 6581 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 6582 stats.macInErrs[3]); 6583 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 6584 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 6585 stats.hdrInErrs[3]); 6586 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 6587 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 6588 stats.tcpInErrs[3]); 6589 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 6590 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 6591 stats.tcp6InErrs[3]); 6592 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 6593 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 6594 stats.tnlCongDrops[3]); 6595 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 6596 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 6597 stats.tnlTxDrops[3]); 6598 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 6599 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 6600 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 6601 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 6602 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 6603 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 6604 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 6605 stats.ofldNoNeigh, stats.ofldCongDefer); 6606 6607 rc = sbuf_finish(sb); 6608 sbuf_delete(sb); 6609 6610 return (rc); 6611 } 6612 6613 struct field_desc { 6614 const char *name; 6615 u_int start; 6616 u_int width; 6617 }; 6618 6619 static void 6620 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) 6621 { 6622 char buf[32]; 6623 int line_size = 0; 6624 6625 while (f->name) { 6626 uint64_t mask = (1ULL << f->width) - 1; 6627 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, 6628 ((uintmax_t)v >> f->start) & mask); 6629 6630 if (line_size + len >= 79) { 6631 line_size = 8; 6632 sbuf_printf(sb, "\n "); 6633 } 6634 sbuf_printf(sb, "%s ", buf); 6635 line_size += len + 1; 6636 f++; 6637 } 6638 sbuf_printf(sb, "\n"); 6639 } 6640 6641 static struct field_desc tp_la0[] = { 6642 { "RcfOpCodeOut", 60, 4 }, 6643 { "State", 56, 4 }, 6644 { "WcfState", 52, 4 }, 6645 { "RcfOpcSrcOut", 50, 2 }, 6646 { "CRxError", 49, 1 }, 6647 { "ERxError", 48, 1 }, 6648 { "SanityFailed", 47, 1 }, 6649 { "SpuriousMsg", 46, 1 }, 6650 { "FlushInputMsg", 45, 1 }, 6651 { "FlushInputCpl", 44, 1 }, 6652 { "RssUpBit", 43, 1 }, 6653 { "RssFilterHit", 42, 1 }, 6654 { "Tid", 32, 10 }, 6655 { "InitTcb", 31, 1 }, 6656 { "LineNumber", 24, 7 }, 6657 { "Emsg", 23, 1 }, 6658 { "EdataOut", 22, 1 }, 6659 { "Cmsg", 21, 1 }, 6660 { "CdataOut", 20, 1 }, 6661 { "EreadPdu", 19, 1 }, 6662 { "CreadPdu", 18, 1 }, 6663 { "TunnelPkt", 17, 1 }, 6664 { "RcfPeerFin", 16, 1 }, 6665 { "RcfReasonOut", 12, 4 }, 6666 { "TxCchannel", 10, 2 }, 6667 { "RcfTxChannel", 8, 2 }, 6668 { "RxEchannel", 6, 2 }, 6669 { "RcfRxChannel", 5, 1 }, 6670 { "RcfDataOutSrdy", 4, 1 }, 6671 { "RxDvld", 3, 1 }, 6672 { "RxOoDvld", 2, 1 }, 6673 { "RxCongestion", 1, 1 }, 6674 { "TxCongestion", 0, 1 }, 6675 { NULL } 6676 }; 6677 6678 static struct field_desc tp_la1[] = { 6679 { "CplCmdIn", 56, 8 }, 6680 { "CplCmdOut", 48, 8 }, 6681 { "ESynOut", 47, 1 }, 6682 { "EAckOut", 46, 1 }, 6683 { "EFinOut", 45, 1 }, 6684 { "ERstOut", 44, 1 }, 6685 { "SynIn", 43, 1 }, 6686 { "AckIn", 42, 1 }, 6687 { "FinIn", 41, 1 }, 6688 { "RstIn", 40, 1 }, 6689 { "DataIn", 39, 1 }, 6690 { "DataInVld", 38, 1 }, 6691 { "PadIn", 37, 1 }, 6692 { "RxBufEmpty", 36, 1 }, 6693 { "RxDdp", 35, 1 }, 6694 { "RxFbCongestion", 34, 1 }, 6695 { "TxFbCongestion", 33, 1 }, 6696 { "TxPktSumSrdy", 32, 1 }, 6697 { "RcfUlpType", 28, 4 }, 6698 { "Eread", 27, 1 }, 6699 { "Ebypass", 26, 1 }, 6700 { "Esave", 25, 1 }, 6701 { "Static0", 24, 1 }, 6702 { "Cread", 23, 1 }, 6703 { "Cbypass", 22, 1 }, 6704 { "Csave", 21, 1 }, 6705 { "CPktOut", 20, 1 }, 6706 { "RxPagePoolFull", 18, 2 }, 6707 { "RxLpbkPkt", 17, 1 }, 6708 { "TxLpbkPkt", 16, 1 }, 6709 { "RxVfValid", 15, 1 }, 6710 { "SynLearned", 14, 1 }, 6711 { "SetDelEntry", 13, 1 }, 6712 { "SetInvEntry", 12, 1 }, 6713 { "CpcmdDvld", 11, 1 }, 6714 { "CpcmdSave", 10, 1 }, 6715 { "RxPstructsFull", 8, 2 }, 6716 { "EpcmdDvld", 7, 1 }, 6717 { "EpcmdFlush", 6, 1 }, 6718 { "EpcmdTrimPrefix", 5, 1 }, 6719 { "EpcmdTrimPostfix", 4, 1 }, 6720 { "ERssIp4Pkt", 3, 1 }, 6721 { "ERssIp6Pkt", 2, 1 }, 6722 { "ERssTcpUdpPkt", 1, 1 }, 6723 { "ERssFceFipPkt", 0, 1 }, 6724 { NULL } 6725 }; 6726 6727 static struct field_desc tp_la2[] = { 6728 { "CplCmdIn", 56, 8 }, 6729 { "MpsVfVld", 55, 1 }, 6730 { "MpsPf", 52, 3 }, 6731 { "MpsVf", 44, 8 }, 6732 { "SynIn", 43, 1 }, 6733 { "AckIn", 42, 1 }, 6734 { "FinIn", 41, 1 }, 6735 { "RstIn", 40, 1 }, 6736 { "DataIn", 39, 1 }, 6737 { "DataInVld", 38, 1 }, 6738 { "PadIn", 37, 1 }, 6739 { "RxBufEmpty", 36, 1 }, 6740 { "RxDdp", 35, 1 }, 6741 { "RxFbCongestion", 34, 1 }, 6742 { "TxFbCongestion", 33, 1 }, 6743 { "TxPktSumSrdy", 32, 1 }, 6744 { "RcfUlpType", 28, 4 }, 6745 { "Eread", 27, 1 }, 6746 { "Ebypass", 26, 1 }, 6747 { "Esave", 25, 1 }, 6748 { "Static0", 24, 1 }, 6749 { "Cread", 23, 1 }, 6750 { "Cbypass", 22, 1 }, 6751 { "Csave", 21, 1 }, 6752 { "CPktOut", 20, 1 }, 6753 { "RxPagePoolFull", 18, 2 }, 6754 { "RxLpbkPkt", 17, 1 }, 6755 { "TxLpbkPkt", 16, 1 }, 6756 { "RxVfValid", 15, 1 }, 6757 { "SynLearned", 14, 1 }, 6758 { "SetDelEntry", 13, 1 }, 6759 { "SetInvEntry", 12, 1 }, 6760 { "CpcmdDvld", 11, 1 }, 6761 { "CpcmdSave", 10, 1 }, 6762 { "RxPstructsFull", 8, 2 }, 6763 { "EpcmdDvld", 7, 1 }, 6764 { "EpcmdFlush", 6, 1 }, 6765 { "EpcmdTrimPrefix", 5, 1 }, 6766 { "EpcmdTrimPostfix", 4, 1 }, 6767 { "ERssIp4Pkt", 3, 1 }, 6768 { "ERssIp6Pkt", 2, 1 }, 6769 { "ERssTcpUdpPkt", 1, 1 }, 6770 { "ERssFceFipPkt", 0, 1 }, 6771 { NULL } 6772 }; 6773 6774 static void 6775 tp_la_show(struct sbuf *sb, uint64_t *p, int idx) 6776 { 6777 6778 field_desc_show(sb, *p, tp_la0); 6779 } 6780 6781 static void 6782 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) 6783 { 6784 6785 if (idx) 6786 sbuf_printf(sb, "\n"); 6787 field_desc_show(sb, p[0], tp_la0); 6788 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6789 field_desc_show(sb, p[1], tp_la0); 6790 } 6791 6792 static void 6793 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) 6794 { 6795 6796 if (idx) 6797 sbuf_printf(sb, "\n"); 6798 field_desc_show(sb, p[0], tp_la0); 6799 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6800 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); 6801 } 6802 6803 static int 6804 sysctl_tp_la(SYSCTL_HANDLER_ARGS) 6805 { 6806 struct adapter *sc = arg1; 6807 struct sbuf *sb; 6808 uint64_t *buf, *p; 6809 int rc; 6810 u_int i, inc; 6811 void (*show_func)(struct sbuf *, uint64_t *, int); 6812 6813 rc = sysctl_wire_old_buffer(req, 0); 6814 if (rc != 0) 6815 return (rc); 6816 6817 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6818 if (sb == NULL) 6819 return (ENOMEM); 6820 6821 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); 6822 6823 t4_tp_read_la(sc, buf, NULL); 6824 p = buf; 6825 6826 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { 6827 case 2: 6828 inc = 2; 6829 show_func = tp_la_show2; 6830 break; 6831 case 3: 6832 inc = 2; 6833 show_func = tp_la_show3; 6834 break; 6835 default: 6836 inc = 1; 6837 show_func = tp_la_show; 6838 } 6839 6840 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) 6841 (*show_func)(sb, p, i); 6842 6843 rc = sbuf_finish(sb); 6844 sbuf_delete(sb); 6845 free(buf, M_CXGBE); 6846 return (rc); 6847 } 6848 6849 static int 6850 sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 6851 { 6852 struct adapter *sc = arg1; 6853 struct sbuf *sb; 6854 int rc; 6855 u64 nrate[NCHAN], orate[NCHAN]; 6856 6857 rc = sysctl_wire_old_buffer(req, 0); 6858 if (rc != 0) 6859 return (rc); 6860 6861 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6862 if (sb == NULL) 6863 return (ENOMEM); 6864 6865 t4_get_chan_txrate(sc, nrate, orate); 6866 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6867 "channel 3\n"); 6868 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 6869 nrate[0], nrate[1], nrate[2], nrate[3]); 6870 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 6871 orate[0], orate[1], orate[2], orate[3]); 6872 6873 rc = sbuf_finish(sb); 6874 sbuf_delete(sb); 6875 6876 return (rc); 6877 } 6878 6879 static int 6880 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) 6881 { 6882 struct adapter *sc = arg1; 6883 struct sbuf *sb; 6884 uint32_t *buf, *p; 6885 int rc, i; 6886 6887 rc = sysctl_wire_old_buffer(req, 0); 6888 if (rc != 0) 6889 return (rc); 6890 6891 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6892 if (sb == NULL) 6893 return (ENOMEM); 6894 6895 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, 6896 M_ZERO | M_WAITOK); 6897 6898 t4_ulprx_read_la(sc, buf); 6899 p = buf; 6900 6901 sbuf_printf(sb, " Pcmd Type Message" 6902 " Data"); 6903 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { 6904 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", 6905 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); 6906 } 6907 6908 rc = sbuf_finish(sb); 6909 sbuf_delete(sb); 6910 free(buf, M_CXGBE); 6911 return (rc); 6912 } 6913 6914 static int 6915 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 6916 { 6917 struct adapter *sc = arg1; 6918 struct sbuf *sb; 6919 int rc, v; 6920 6921 rc = sysctl_wire_old_buffer(req, 0); 6922 if (rc != 0) 6923 return (rc); 6924 6925 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6926 if (sb == NULL) 6927 return (ENOMEM); 6928 6929 v = t4_read_reg(sc, A_SGE_STAT_CFG); 6930 if (G_STATSOURCE_T5(v) == 7) { 6931 if (G_STATMODE(v) == 0) { 6932 sbuf_printf(sb, "total %d, incomplete %d", 6933 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6934 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6935 } else if (G_STATMODE(v) == 1) { 6936 sbuf_printf(sb, "total %d, data overflow %d", 6937 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6938 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6939 } 6940 } 6941 rc = sbuf_finish(sb); 6942 sbuf_delete(sb); 6943 6944 return (rc); 6945 } 6946 #endif 6947 6948 static inline void 6949 txq_start(struct ifnet *ifp, struct sge_txq *txq) 6950 { 6951 struct buf_ring *br; 6952 struct mbuf *m; 6953 6954 TXQ_LOCK_ASSERT_OWNED(txq); 6955 6956 br = txq->br; 6957 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 6958 if (m) 6959 t4_eth_tx(ifp, txq, m); 6960 } 6961 6962 void 6963 t4_tx_callout(void *arg) 6964 { 6965 struct sge_eq *eq = arg; 6966 struct adapter *sc; 6967 6968 if (EQ_TRYLOCK(eq) == 0) 6969 goto reschedule; 6970 6971 if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) { 6972 EQ_UNLOCK(eq); 6973 reschedule: 6974 if (__predict_true(!(eq->flags && EQ_DOOMED))) 6975 callout_schedule(&eq->tx_callout, 1); 6976 return; 6977 } 6978 6979 EQ_LOCK_ASSERT_OWNED(eq); 6980 6981 if (__predict_true((eq->flags & EQ_DOOMED) == 0)) { 6982 6983 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 6984 struct sge_txq *txq = arg; 6985 struct port_info *pi = txq->ifp->if_softc; 6986 6987 sc = pi->adapter; 6988 } else { 6989 struct sge_wrq *wrq = arg; 6990 6991 sc = wrq->adapter; 6992 } 6993 6994 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task); 6995 } 6996 6997 EQ_UNLOCK(eq); 6998 } 6999 7000 void 7001 t4_tx_task(void *arg, int count) 7002 { 7003 struct sge_eq *eq = arg; 7004 7005 EQ_LOCK(eq); 7006 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 7007 struct sge_txq *txq = arg; 7008 txq_start(txq->ifp, txq); 7009 } else { 7010 struct sge_wrq *wrq = arg; 7011 t4_wrq_tx_locked(wrq->adapter, wrq, NULL); 7012 } 7013 EQ_UNLOCK(eq); 7014 } 7015 7016 static uint32_t 7017 fconf_to_mode(uint32_t fconf) 7018 { 7019 uint32_t mode; 7020 7021 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 7022 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 7023 7024 if (fconf & F_FRAGMENTATION) 7025 mode |= T4_FILTER_IP_FRAGMENT; 7026 7027 if (fconf & F_MPSHITTYPE) 7028 mode |= T4_FILTER_MPS_HIT_TYPE; 7029 7030 if (fconf & F_MACMATCH) 7031 mode |= T4_FILTER_MAC_IDX; 7032 7033 if (fconf & F_ETHERTYPE) 7034 mode |= T4_FILTER_ETH_TYPE; 7035 7036 if (fconf & F_PROTOCOL) 7037 mode |= T4_FILTER_IP_PROTO; 7038 7039 if (fconf & F_TOS) 7040 mode |= T4_FILTER_IP_TOS; 7041 7042 if (fconf & F_VLAN) 7043 mode |= T4_FILTER_VLAN; 7044 7045 if (fconf & F_VNIC_ID) 7046 mode |= T4_FILTER_VNIC; 7047 7048 if (fconf & F_PORT) 7049 mode |= T4_FILTER_PORT; 7050 7051 if (fconf & F_FCOE) 7052 mode |= T4_FILTER_FCoE; 7053 7054 return (mode); 7055 } 7056 7057 static uint32_t 7058 mode_to_fconf(uint32_t mode) 7059 { 7060 uint32_t fconf = 0; 7061 7062 if (mode & T4_FILTER_IP_FRAGMENT) 7063 fconf |= F_FRAGMENTATION; 7064 7065 if (mode & T4_FILTER_MPS_HIT_TYPE) 7066 fconf |= F_MPSHITTYPE; 7067 7068 if (mode & T4_FILTER_MAC_IDX) 7069 fconf |= F_MACMATCH; 7070 7071 if (mode & T4_FILTER_ETH_TYPE) 7072 fconf |= F_ETHERTYPE; 7073 7074 if (mode & T4_FILTER_IP_PROTO) 7075 fconf |= F_PROTOCOL; 7076 7077 if (mode & T4_FILTER_IP_TOS) 7078 fconf |= F_TOS; 7079 7080 if (mode & T4_FILTER_VLAN) 7081 fconf |= F_VLAN; 7082 7083 if (mode & T4_FILTER_VNIC) 7084 fconf |= F_VNIC_ID; 7085 7086 if (mode & T4_FILTER_PORT) 7087 fconf |= F_PORT; 7088 7089 if (mode & T4_FILTER_FCoE) 7090 fconf |= F_FCOE; 7091 7092 return (fconf); 7093 } 7094 7095 static uint32_t 7096 fspec_to_fconf(struct t4_filter_specification *fs) 7097 { 7098 uint32_t fconf = 0; 7099 7100 if (fs->val.frag || fs->mask.frag) 7101 fconf |= F_FRAGMENTATION; 7102 7103 if (fs->val.matchtype || fs->mask.matchtype) 7104 fconf |= F_MPSHITTYPE; 7105 7106 if (fs->val.macidx || fs->mask.macidx) 7107 fconf |= F_MACMATCH; 7108 7109 if (fs->val.ethtype || fs->mask.ethtype) 7110 fconf |= F_ETHERTYPE; 7111 7112 if (fs->val.proto || fs->mask.proto) 7113 fconf |= F_PROTOCOL; 7114 7115 if (fs->val.tos || fs->mask.tos) 7116 fconf |= F_TOS; 7117 7118 if (fs->val.vlan_vld || fs->mask.vlan_vld) 7119 fconf |= F_VLAN; 7120 7121 if (fs->val.vnic_vld || fs->mask.vnic_vld) 7122 fconf |= F_VNIC_ID; 7123 7124 if (fs->val.iport || fs->mask.iport) 7125 fconf |= F_PORT; 7126 7127 if (fs->val.fcoe || fs->mask.fcoe) 7128 fconf |= F_FCOE; 7129 7130 return (fconf); 7131 } 7132 7133 static int 7134 get_filter_mode(struct adapter *sc, uint32_t *mode) 7135 { 7136 int rc; 7137 uint32_t fconf; 7138 7139 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7140 "t4getfm"); 7141 if (rc) 7142 return (rc); 7143 7144 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 7145 A_TP_VLAN_PRI_MAP); 7146 7147 if (sc->params.tp.vlan_pri_map != fconf) { 7148 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 7149 device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map, 7150 fconf); 7151 sc->params.tp.vlan_pri_map = fconf; 7152 } 7153 7154 *mode = fconf_to_mode(sc->params.tp.vlan_pri_map); 7155 7156 end_synchronized_op(sc, LOCK_HELD); 7157 return (0); 7158 } 7159 7160 static int 7161 set_filter_mode(struct adapter *sc, uint32_t mode) 7162 { 7163 uint32_t fconf; 7164 int rc; 7165 7166 fconf = mode_to_fconf(mode); 7167 7168 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7169 "t4setfm"); 7170 if (rc) 7171 return (rc); 7172 7173 if (sc->tids.ftids_in_use > 0) { 7174 rc = EBUSY; 7175 goto done; 7176 } 7177 7178 #ifdef TCP_OFFLOAD 7179 if (sc->offload_map) { 7180 rc = EBUSY; 7181 goto done; 7182 } 7183 #endif 7184 7185 #ifdef notyet 7186 rc = -t4_set_filter_mode(sc, fconf); 7187 if (rc == 0) 7188 sc->filter_mode = fconf; 7189 #else 7190 rc = ENOTSUP; 7191 #endif 7192 7193 done: 7194 end_synchronized_op(sc, LOCK_HELD); 7195 return (rc); 7196 } 7197 7198 static inline uint64_t 7199 get_filter_hits(struct adapter *sc, uint32_t fid) 7200 { 7201 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 7202 uint64_t hits; 7203 7204 memwin_info(sc, 0, &mw_base, NULL); 7205 off = position_memwin(sc, 0, 7206 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 7207 if (is_t4(sc)) { 7208 hits = t4_read_reg64(sc, mw_base + off + 16); 7209 hits = be64toh(hits); 7210 } else { 7211 hits = t4_read_reg(sc, mw_base + off + 24); 7212 hits = be32toh(hits); 7213 } 7214 7215 return (hits); 7216 } 7217 7218 static int 7219 get_filter(struct adapter *sc, struct t4_filter *t) 7220 { 7221 int i, rc, nfilters = sc->tids.nftids; 7222 struct filter_entry *f; 7223 7224 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7225 "t4getf"); 7226 if (rc) 7227 return (rc); 7228 7229 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 7230 t->idx >= nfilters) { 7231 t->idx = 0xffffffff; 7232 goto done; 7233 } 7234 7235 f = &sc->tids.ftid_tab[t->idx]; 7236 for (i = t->idx; i < nfilters; i++, f++) { 7237 if (f->valid) { 7238 t->idx = i; 7239 t->l2tidx = f->l2t ? f->l2t->idx : 0; 7240 t->smtidx = f->smtidx; 7241 if (f->fs.hitcnts) 7242 t->hits = get_filter_hits(sc, t->idx); 7243 else 7244 t->hits = UINT64_MAX; 7245 t->fs = f->fs; 7246 7247 goto done; 7248 } 7249 } 7250 7251 t->idx = 0xffffffff; 7252 done: 7253 end_synchronized_op(sc, LOCK_HELD); 7254 return (0); 7255 } 7256 7257 static int 7258 set_filter(struct adapter *sc, struct t4_filter *t) 7259 { 7260 unsigned int nfilters, nports; 7261 struct filter_entry *f; 7262 int i, rc; 7263 7264 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 7265 if (rc) 7266 return (rc); 7267 7268 nfilters = sc->tids.nftids; 7269 nports = sc->params.nports; 7270 7271 if (nfilters == 0) { 7272 rc = ENOTSUP; 7273 goto done; 7274 } 7275 7276 if (!(sc->flags & FULL_INIT_DONE)) { 7277 rc = EAGAIN; 7278 goto done; 7279 } 7280 7281 if (t->idx >= nfilters) { 7282 rc = EINVAL; 7283 goto done; 7284 } 7285 7286 /* Validate against the global filter mode */ 7287 if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) != 7288 sc->params.tp.vlan_pri_map) { 7289 rc = E2BIG; 7290 goto done; 7291 } 7292 7293 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 7294 rc = EINVAL; 7295 goto done; 7296 } 7297 7298 if (t->fs.val.iport >= nports) { 7299 rc = EINVAL; 7300 goto done; 7301 } 7302 7303 /* Can't specify an iq if not steering to it */ 7304 if (!t->fs.dirsteer && t->fs.iq) { 7305 rc = EINVAL; 7306 goto done; 7307 } 7308 7309 /* IPv6 filter idx must be 4 aligned */ 7310 if (t->fs.type == 1 && 7311 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 7312 rc = EINVAL; 7313 goto done; 7314 } 7315 7316 if (sc->tids.ftid_tab == NULL) { 7317 KASSERT(sc->tids.ftids_in_use == 0, 7318 ("%s: no memory allocated but filters_in_use > 0", 7319 __func__)); 7320 7321 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 7322 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 7323 if (sc->tids.ftid_tab == NULL) { 7324 rc = ENOMEM; 7325 goto done; 7326 } 7327 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 7328 } 7329 7330 for (i = 0; i < 4; i++) { 7331 f = &sc->tids.ftid_tab[t->idx + i]; 7332 7333 if (f->pending || f->valid) { 7334 rc = EBUSY; 7335 goto done; 7336 } 7337 if (f->locked) { 7338 rc = EPERM; 7339 goto done; 7340 } 7341 7342 if (t->fs.type == 0) 7343 break; 7344 } 7345 7346 f = &sc->tids.ftid_tab[t->idx]; 7347 f->fs = t->fs; 7348 7349 rc = set_filter_wr(sc, t->idx); 7350 done: 7351 end_synchronized_op(sc, 0); 7352 7353 if (rc == 0) { 7354 mtx_lock(&sc->tids.ftid_lock); 7355 for (;;) { 7356 if (f->pending == 0) { 7357 rc = f->valid ? 0 : EIO; 7358 break; 7359 } 7360 7361 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7362 PCATCH, "t4setfw", 0)) { 7363 rc = EINPROGRESS; 7364 break; 7365 } 7366 } 7367 mtx_unlock(&sc->tids.ftid_lock); 7368 } 7369 return (rc); 7370 } 7371 7372 static int 7373 del_filter(struct adapter *sc, struct t4_filter *t) 7374 { 7375 unsigned int nfilters; 7376 struct filter_entry *f; 7377 int rc; 7378 7379 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 7380 if (rc) 7381 return (rc); 7382 7383 nfilters = sc->tids.nftids; 7384 7385 if (nfilters == 0) { 7386 rc = ENOTSUP; 7387 goto done; 7388 } 7389 7390 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 7391 t->idx >= nfilters) { 7392 rc = EINVAL; 7393 goto done; 7394 } 7395 7396 if (!(sc->flags & FULL_INIT_DONE)) { 7397 rc = EAGAIN; 7398 goto done; 7399 } 7400 7401 f = &sc->tids.ftid_tab[t->idx]; 7402 7403 if (f->pending) { 7404 rc = EBUSY; 7405 goto done; 7406 } 7407 if (f->locked) { 7408 rc = EPERM; 7409 goto done; 7410 } 7411 7412 if (f->valid) { 7413 t->fs = f->fs; /* extra info for the caller */ 7414 rc = del_filter_wr(sc, t->idx); 7415 } 7416 7417 done: 7418 end_synchronized_op(sc, 0); 7419 7420 if (rc == 0) { 7421 mtx_lock(&sc->tids.ftid_lock); 7422 for (;;) { 7423 if (f->pending == 0) { 7424 rc = f->valid ? EIO : 0; 7425 break; 7426 } 7427 7428 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7429 PCATCH, "t4delfw", 0)) { 7430 rc = EINPROGRESS; 7431 break; 7432 } 7433 } 7434 mtx_unlock(&sc->tids.ftid_lock); 7435 } 7436 7437 return (rc); 7438 } 7439 7440 static void 7441 clear_filter(struct filter_entry *f) 7442 { 7443 if (f->l2t) 7444 t4_l2t_release(f->l2t); 7445 7446 bzero(f, sizeof (*f)); 7447 } 7448 7449 static int 7450 set_filter_wr(struct adapter *sc, int fidx) 7451 { 7452 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7453 struct wrqe *wr; 7454 struct fw_filter_wr *fwr; 7455 unsigned int ftid; 7456 7457 ASSERT_SYNCHRONIZED_OP(sc); 7458 7459 if (f->fs.newdmac || f->fs.newvlan) { 7460 /* This filter needs an L2T entry; allocate one. */ 7461 f->l2t = t4_l2t_alloc_switching(sc->l2t); 7462 if (f->l2t == NULL) 7463 return (EAGAIN); 7464 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 7465 f->fs.dmac)) { 7466 t4_l2t_release(f->l2t); 7467 f->l2t = NULL; 7468 return (ENOMEM); 7469 } 7470 } 7471 7472 ftid = sc->tids.ftid_base + fidx; 7473 7474 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 7475 if (wr == NULL) 7476 return (ENOMEM); 7477 7478 fwr = wrtod(wr); 7479 bzero(fwr, sizeof (*fwr)); 7480 7481 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 7482 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 7483 fwr->tid_to_iq = 7484 htobe32(V_FW_FILTER_WR_TID(ftid) | 7485 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 7486 V_FW_FILTER_WR_NOREPLY(0) | 7487 V_FW_FILTER_WR_IQ(f->fs.iq)); 7488 fwr->del_filter_to_l2tix = 7489 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 7490 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 7491 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 7492 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 7493 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 7494 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 7495 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 7496 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 7497 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 7498 f->fs.newvlan == VLAN_REWRITE) | 7499 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 7500 f->fs.newvlan == VLAN_REWRITE) | 7501 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 7502 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 7503 V_FW_FILTER_WR_PRIO(f->fs.prio) | 7504 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 7505 fwr->ethtype = htobe16(f->fs.val.ethtype); 7506 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 7507 fwr->frag_to_ovlan_vldm = 7508 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 7509 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 7510 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 7511 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 7512 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 7513 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 7514 fwr->smac_sel = 0; 7515 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 7516 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 7517 fwr->maci_to_matchtypem = 7518 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 7519 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 7520 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 7521 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 7522 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 7523 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 7524 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 7525 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 7526 fwr->ptcl = f->fs.val.proto; 7527 fwr->ptclm = f->fs.mask.proto; 7528 fwr->ttyp = f->fs.val.tos; 7529 fwr->ttypm = f->fs.mask.tos; 7530 fwr->ivlan = htobe16(f->fs.val.vlan); 7531 fwr->ivlanm = htobe16(f->fs.mask.vlan); 7532 fwr->ovlan = htobe16(f->fs.val.vnic); 7533 fwr->ovlanm = htobe16(f->fs.mask.vnic); 7534 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 7535 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 7536 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 7537 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 7538 fwr->lp = htobe16(f->fs.val.dport); 7539 fwr->lpm = htobe16(f->fs.mask.dport); 7540 fwr->fp = htobe16(f->fs.val.sport); 7541 fwr->fpm = htobe16(f->fs.mask.sport); 7542 if (f->fs.newsmac) 7543 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 7544 7545 f->pending = 1; 7546 sc->tids.ftids_in_use++; 7547 7548 t4_wrq_tx(sc, wr); 7549 return (0); 7550 } 7551 7552 static int 7553 del_filter_wr(struct adapter *sc, int fidx) 7554 { 7555 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7556 struct wrqe *wr; 7557 struct fw_filter_wr *fwr; 7558 unsigned int ftid; 7559 7560 ftid = sc->tids.ftid_base + fidx; 7561 7562 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 7563 if (wr == NULL) 7564 return (ENOMEM); 7565 fwr = wrtod(wr); 7566 bzero(fwr, sizeof (*fwr)); 7567 7568 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 7569 7570 f->pending = 1; 7571 t4_wrq_tx(sc, wr); 7572 return (0); 7573 } 7574 7575 int 7576 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 7577 { 7578 struct adapter *sc = iq->adapter; 7579 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 7580 unsigned int idx = GET_TID(rpl); 7581 unsigned int rc; 7582 struct filter_entry *f; 7583 7584 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 7585 rss->opcode)); 7586 7587 if (is_ftid(sc, idx)) { 7588 7589 idx -= sc->tids.ftid_base; 7590 f = &sc->tids.ftid_tab[idx]; 7591 rc = G_COOKIE(rpl->cookie); 7592 7593 mtx_lock(&sc->tids.ftid_lock); 7594 if (rc == FW_FILTER_WR_FLT_ADDED) { 7595 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 7596 __func__, idx)); 7597 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 7598 f->pending = 0; /* asynchronous setup completed */ 7599 f->valid = 1; 7600 } else { 7601 if (rc != FW_FILTER_WR_FLT_DELETED) { 7602 /* Add or delete failed, display an error */ 7603 log(LOG_ERR, 7604 "filter %u setup failed with error %u\n", 7605 idx, rc); 7606 } 7607 7608 clear_filter(f); 7609 sc->tids.ftids_in_use--; 7610 } 7611 wakeup(&sc->tids.ftid_tab); 7612 mtx_unlock(&sc->tids.ftid_lock); 7613 } 7614 7615 return (0); 7616 } 7617 7618 static int 7619 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 7620 { 7621 int rc; 7622 7623 if (cntxt->cid > M_CTXTQID) 7624 return (EINVAL); 7625 7626 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 7627 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 7628 return (EINVAL); 7629 7630 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 7631 if (rc) 7632 return (rc); 7633 7634 if (sc->flags & FW_OK) { 7635 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 7636 &cntxt->data[0]); 7637 if (rc == 0) 7638 goto done; 7639 } 7640 7641 /* 7642 * Read via firmware failed or wasn't even attempted. Read directly via 7643 * the backdoor. 7644 */ 7645 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 7646 done: 7647 end_synchronized_op(sc, 0); 7648 return (rc); 7649 } 7650 7651 static int 7652 load_fw(struct adapter *sc, struct t4_data *fw) 7653 { 7654 int rc; 7655 uint8_t *fw_data; 7656 7657 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 7658 if (rc) 7659 return (rc); 7660 7661 if (sc->flags & FULL_INIT_DONE) { 7662 rc = EBUSY; 7663 goto done; 7664 } 7665 7666 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 7667 if (fw_data == NULL) { 7668 rc = ENOMEM; 7669 goto done; 7670 } 7671 7672 rc = copyin(fw->data, fw_data, fw->len); 7673 if (rc == 0) 7674 rc = -t4_load_fw(sc, fw_data, fw->len); 7675 7676 free(fw_data, M_CXGBE); 7677 done: 7678 end_synchronized_op(sc, 0); 7679 return (rc); 7680 } 7681 7682 static int 7683 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 7684 { 7685 uint32_t addr, off, remaining, i, n; 7686 uint32_t *buf, *b; 7687 uint32_t mw_base, mw_aperture; 7688 int rc; 7689 uint8_t *dst; 7690 7691 rc = validate_mem_range(sc, mr->addr, mr->len); 7692 if (rc != 0) 7693 return (rc); 7694 7695 memwin_info(sc, win, &mw_base, &mw_aperture); 7696 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 7697 addr = mr->addr; 7698 remaining = mr->len; 7699 dst = (void *)mr->data; 7700 7701 while (remaining) { 7702 off = position_memwin(sc, win, addr); 7703 7704 /* number of bytes that we'll copy in the inner loop */ 7705 n = min(remaining, mw_aperture - off); 7706 for (i = 0; i < n; i += 4) 7707 *b++ = t4_read_reg(sc, mw_base + off + i); 7708 7709 rc = copyout(buf, dst, n); 7710 if (rc != 0) 7711 break; 7712 7713 b = buf; 7714 dst += n; 7715 remaining -= n; 7716 addr += n; 7717 } 7718 7719 free(buf, M_CXGBE); 7720 return (rc); 7721 } 7722 7723 static int 7724 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 7725 { 7726 int rc; 7727 7728 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 7729 return (EINVAL); 7730 7731 if (i2cd->len > sizeof(i2cd->data)) 7732 return (EFBIG); 7733 7734 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 7735 if (rc) 7736 return (rc); 7737 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 7738 i2cd->offset, i2cd->len, &i2cd->data[0]); 7739 end_synchronized_op(sc, 0); 7740 7741 return (rc); 7742 } 7743 7744 static int 7745 in_range(int val, int lo, int hi) 7746 { 7747 7748 return (val < 0 || (val <= hi && val >= lo)); 7749 } 7750 7751 static int 7752 set_sched_class(struct adapter *sc, struct t4_sched_params *p) 7753 { 7754 int fw_subcmd, fw_type, rc; 7755 7756 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc"); 7757 if (rc) 7758 return (rc); 7759 7760 if (!(sc->flags & FULL_INIT_DONE)) { 7761 rc = EAGAIN; 7762 goto done; 7763 } 7764 7765 /* 7766 * Translate the cxgbetool parameters into T4 firmware parameters. (The 7767 * sub-command and type are in common locations.) 7768 */ 7769 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG) 7770 fw_subcmd = FW_SCHED_SC_CONFIG; 7771 else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS) 7772 fw_subcmd = FW_SCHED_SC_PARAMS; 7773 else { 7774 rc = EINVAL; 7775 goto done; 7776 } 7777 if (p->type == SCHED_CLASS_TYPE_PACKET) 7778 fw_type = FW_SCHED_TYPE_PKTSCHED; 7779 else { 7780 rc = EINVAL; 7781 goto done; 7782 } 7783 7784 if (fw_subcmd == FW_SCHED_SC_CONFIG) { 7785 /* Vet our parameters ..*/ 7786 if (p->u.config.minmax < 0) { 7787 rc = EINVAL; 7788 goto done; 7789 } 7790 7791 /* And pass the request to the firmware ...*/ 7792 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1); 7793 goto done; 7794 } 7795 7796 if (fw_subcmd == FW_SCHED_SC_PARAMS) { 7797 int fw_level; 7798 int fw_mode; 7799 int fw_rateunit; 7800 int fw_ratemode; 7801 7802 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL) 7803 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL; 7804 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) 7805 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR; 7806 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) 7807 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL; 7808 else { 7809 rc = EINVAL; 7810 goto done; 7811 } 7812 7813 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS) 7814 fw_mode = FW_SCHED_PARAMS_MODE_CLASS; 7815 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW) 7816 fw_mode = FW_SCHED_PARAMS_MODE_FLOW; 7817 else { 7818 rc = EINVAL; 7819 goto done; 7820 } 7821 7822 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS) 7823 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 7824 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS) 7825 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE; 7826 else { 7827 rc = EINVAL; 7828 goto done; 7829 } 7830 7831 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL) 7832 fw_ratemode = FW_SCHED_PARAMS_RATE_REL; 7833 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS) 7834 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS; 7835 else { 7836 rc = EINVAL; 7837 goto done; 7838 } 7839 7840 /* Vet our parameters ... */ 7841 if (!in_range(p->u.params.channel, 0, 3) || 7842 !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) || 7843 !in_range(p->u.params.minrate, 0, 10000000) || 7844 !in_range(p->u.params.maxrate, 0, 10000000) || 7845 !in_range(p->u.params.weight, 0, 100)) { 7846 rc = ERANGE; 7847 goto done; 7848 } 7849 7850 /* 7851 * Translate any unset parameters into the firmware's 7852 * nomenclature and/or fail the call if the parameters 7853 * are required ... 7854 */ 7855 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 || 7856 p->u.params.channel < 0 || p->u.params.cl < 0) { 7857 rc = EINVAL; 7858 goto done; 7859 } 7860 if (p->u.params.minrate < 0) 7861 p->u.params.minrate = 0; 7862 if (p->u.params.maxrate < 0) { 7863 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7864 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7865 rc = EINVAL; 7866 goto done; 7867 } else 7868 p->u.params.maxrate = 0; 7869 } 7870 if (p->u.params.weight < 0) { 7871 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) { 7872 rc = EINVAL; 7873 goto done; 7874 } else 7875 p->u.params.weight = 0; 7876 } 7877 if (p->u.params.pktsize < 0) { 7878 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7879 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7880 rc = EINVAL; 7881 goto done; 7882 } else 7883 p->u.params.pktsize = 0; 7884 } 7885 7886 /* See what the firmware thinks of the request ... */ 7887 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode, 7888 fw_rateunit, fw_ratemode, p->u.params.channel, 7889 p->u.params.cl, p->u.params.minrate, p->u.params.maxrate, 7890 p->u.params.weight, p->u.params.pktsize, 1); 7891 goto done; 7892 } 7893 7894 rc = EINVAL; 7895 done: 7896 end_synchronized_op(sc, 0); 7897 return (rc); 7898 } 7899 7900 static int 7901 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p) 7902 { 7903 struct port_info *pi = NULL; 7904 struct sge_txq *txq; 7905 uint32_t fw_mnem, fw_queue, fw_class; 7906 int i, rc; 7907 7908 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq"); 7909 if (rc) 7910 return (rc); 7911 7912 if (!(sc->flags & FULL_INIT_DONE)) { 7913 rc = EAGAIN; 7914 goto done; 7915 } 7916 7917 if (p->port >= sc->params.nports) { 7918 rc = EINVAL; 7919 goto done; 7920 } 7921 7922 pi = sc->port[p->port]; 7923 if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) { 7924 rc = EINVAL; 7925 goto done; 7926 } 7927 7928 /* 7929 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX 7930 * Scheduling Class in this case). 7931 */ 7932 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 7933 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH)); 7934 fw_class = p->cl < 0 ? 0xffffffff : p->cl; 7935 7936 /* 7937 * If op.queue is non-negative, then we're only changing the scheduling 7938 * on a single specified TX queue. 7939 */ 7940 if (p->queue >= 0) { 7941 txq = &sc->sge.txq[pi->first_txq + p->queue]; 7942 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7943 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7944 &fw_class); 7945 goto done; 7946 } 7947 7948 /* 7949 * Change the scheduling on all the TX queues for the 7950 * interface. 7951 */ 7952 for_each_txq(pi, i, txq) { 7953 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7954 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7955 &fw_class); 7956 if (rc) 7957 goto done; 7958 } 7959 7960 rc = 0; 7961 done: 7962 end_synchronized_op(sc, 0); 7963 return (rc); 7964 } 7965 7966 int 7967 t4_os_find_pci_capability(struct adapter *sc, int cap) 7968 { 7969 int i; 7970 7971 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 7972 } 7973 7974 int 7975 t4_os_pci_save_state(struct adapter *sc) 7976 { 7977 device_t dev; 7978 struct pci_devinfo *dinfo; 7979 7980 dev = sc->dev; 7981 dinfo = device_get_ivars(dev); 7982 7983 pci_cfg_save(dev, dinfo, 0); 7984 return (0); 7985 } 7986 7987 int 7988 t4_os_pci_restore_state(struct adapter *sc) 7989 { 7990 device_t dev; 7991 struct pci_devinfo *dinfo; 7992 7993 dev = sc->dev; 7994 dinfo = device_get_ivars(dev); 7995 7996 pci_cfg_restore(dev, dinfo); 7997 return (0); 7998 } 7999 8000 void 8001 t4_os_portmod_changed(const struct adapter *sc, int idx) 8002 { 8003 struct port_info *pi = sc->port[idx]; 8004 static const char *mod_str[] = { 8005 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 8006 }; 8007 8008 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 8009 if_printf(pi->ifp, "transceiver unplugged.\n"); 8010 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 8011 if_printf(pi->ifp, "unknown transceiver inserted.\n"); 8012 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 8013 if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 8014 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 8015 if_printf(pi->ifp, "%s transceiver inserted.\n", 8016 mod_str[pi->mod_type]); 8017 } else { 8018 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 8019 pi->mod_type); 8020 } 8021 } 8022 8023 void 8024 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason) 8025 { 8026 struct port_info *pi = sc->port[idx]; 8027 struct ifnet *ifp = pi->ifp; 8028 8029 if (link_stat) { 8030 pi->linkdnrc = -1; 8031 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 8032 if_link_state_change(ifp, LINK_STATE_UP); 8033 } else { 8034 if (reason >= 0) 8035 pi->linkdnrc = reason; 8036 if_link_state_change(ifp, LINK_STATE_DOWN); 8037 } 8038 } 8039 8040 void 8041 t4_iterate(void (*func)(struct adapter *, void *), void *arg) 8042 { 8043 struct adapter *sc; 8044 8045 sx_slock(&t4_list_lock); 8046 SLIST_FOREACH(sc, &t4_list, link) { 8047 /* 8048 * func should not make any assumptions about what state sc is 8049 * in - the only guarantee is that sc->sc_lock is a valid lock. 8050 */ 8051 func(sc, arg); 8052 } 8053 sx_sunlock(&t4_list_lock); 8054 } 8055 8056 static int 8057 t4_open(struct cdev *dev, int flags, int type, struct thread *td) 8058 { 8059 return (0); 8060 } 8061 8062 static int 8063 t4_close(struct cdev *dev, int flags, int type, struct thread *td) 8064 { 8065 return (0); 8066 } 8067 8068 static int 8069 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 8070 struct thread *td) 8071 { 8072 int rc; 8073 struct adapter *sc = dev->si_drv1; 8074 8075 rc = priv_check(td, PRIV_DRIVER); 8076 if (rc != 0) 8077 return (rc); 8078 8079 switch (cmd) { 8080 case CHELSIO_T4_GETREG: { 8081 struct t4_reg *edata = (struct t4_reg *)data; 8082 8083 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8084 return (EFAULT); 8085 8086 if (edata->size == 4) 8087 edata->val = t4_read_reg(sc, edata->addr); 8088 else if (edata->size == 8) 8089 edata->val = t4_read_reg64(sc, edata->addr); 8090 else 8091 return (EINVAL); 8092 8093 break; 8094 } 8095 case CHELSIO_T4_SETREG: { 8096 struct t4_reg *edata = (struct t4_reg *)data; 8097 8098 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8099 return (EFAULT); 8100 8101 if (edata->size == 4) { 8102 if (edata->val & 0xffffffff00000000) 8103 return (EINVAL); 8104 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 8105 } else if (edata->size == 8) 8106 t4_write_reg64(sc, edata->addr, edata->val); 8107 else 8108 return (EINVAL); 8109 break; 8110 } 8111 case CHELSIO_T4_REGDUMP: { 8112 struct t4_regdump *regs = (struct t4_regdump *)data; 8113 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 8114 uint8_t *buf; 8115 8116 if (regs->len < reglen) { 8117 regs->len = reglen; /* hint to the caller */ 8118 return (ENOBUFS); 8119 } 8120 8121 regs->len = reglen; 8122 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 8123 t4_get_regs(sc, regs, buf); 8124 rc = copyout(buf, regs->data, reglen); 8125 free(buf, M_CXGBE); 8126 break; 8127 } 8128 case CHELSIO_T4_GET_FILTER_MODE: 8129 rc = get_filter_mode(sc, (uint32_t *)data); 8130 break; 8131 case CHELSIO_T4_SET_FILTER_MODE: 8132 rc = set_filter_mode(sc, *(uint32_t *)data); 8133 break; 8134 case CHELSIO_T4_GET_FILTER: 8135 rc = get_filter(sc, (struct t4_filter *)data); 8136 break; 8137 case CHELSIO_T4_SET_FILTER: 8138 rc = set_filter(sc, (struct t4_filter *)data); 8139 break; 8140 case CHELSIO_T4_DEL_FILTER: 8141 rc = del_filter(sc, (struct t4_filter *)data); 8142 break; 8143 case CHELSIO_T4_GET_SGE_CONTEXT: 8144 rc = get_sge_context(sc, (struct t4_sge_context *)data); 8145 break; 8146 case CHELSIO_T4_LOAD_FW: 8147 rc = load_fw(sc, (struct t4_data *)data); 8148 break; 8149 case CHELSIO_T4_GET_MEM: 8150 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 8151 break; 8152 case CHELSIO_T4_GET_I2C: 8153 rc = read_i2c(sc, (struct t4_i2c_data *)data); 8154 break; 8155 case CHELSIO_T4_CLEAR_STATS: { 8156 int i; 8157 u_int port_id = *(uint32_t *)data; 8158 struct port_info *pi; 8159 8160 if (port_id >= sc->params.nports) 8161 return (EINVAL); 8162 pi = sc->port[port_id]; 8163 8164 /* MAC stats */ 8165 t4_clr_port_stats(sc, pi->tx_chan); 8166 8167 if (pi->flags & PORT_INIT_DONE) { 8168 struct sge_rxq *rxq; 8169 struct sge_txq *txq; 8170 struct sge_wrq *wrq; 8171 8172 for_each_rxq(pi, i, rxq) { 8173 #if defined(INET) || defined(INET6) 8174 rxq->lro.lro_queued = 0; 8175 rxq->lro.lro_flushed = 0; 8176 #endif 8177 rxq->rxcsum = 0; 8178 rxq->vlan_extraction = 0; 8179 } 8180 8181 for_each_txq(pi, i, txq) { 8182 txq->txcsum = 0; 8183 txq->tso_wrs = 0; 8184 txq->vlan_insertion = 0; 8185 txq->imm_wrs = 0; 8186 txq->sgl_wrs = 0; 8187 txq->txpkt_wrs = 0; 8188 txq->txpkts_wrs = 0; 8189 txq->txpkts_pkts = 0; 8190 txq->br->br_drops = 0; 8191 txq->no_dmamap = 0; 8192 txq->no_desc = 0; 8193 } 8194 8195 #ifdef TCP_OFFLOAD 8196 /* nothing to clear for each ofld_rxq */ 8197 8198 for_each_ofld_txq(pi, i, wrq) { 8199 wrq->tx_wrs = 0; 8200 wrq->no_desc = 0; 8201 } 8202 #endif 8203 wrq = &sc->sge.ctrlq[pi->port_id]; 8204 wrq->tx_wrs = 0; 8205 wrq->no_desc = 0; 8206 } 8207 break; 8208 } 8209 case CHELSIO_T4_SCHED_CLASS: 8210 rc = set_sched_class(sc, (struct t4_sched_params *)data); 8211 break; 8212 case CHELSIO_T4_SCHED_QUEUE: 8213 rc = set_sched_queue(sc, (struct t4_sched_queue *)data); 8214 break; 8215 case CHELSIO_T4_GET_TRACER: 8216 rc = t4_get_tracer(sc, (struct t4_tracer *)data); 8217 break; 8218 case CHELSIO_T4_SET_TRACER: 8219 rc = t4_set_tracer(sc, (struct t4_tracer *)data); 8220 break; 8221 default: 8222 rc = EINVAL; 8223 } 8224 8225 return (rc); 8226 } 8227 8228 #ifdef TCP_OFFLOAD 8229 void 8230 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask, 8231 const unsigned int *pgsz_order) 8232 { 8233 struct port_info *pi = ifp->if_softc; 8234 struct adapter *sc = pi->adapter; 8235 8236 t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask); 8237 t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) | 8238 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) | 8239 V_HPZ3(pgsz_order[3])); 8240 } 8241 8242 static int 8243 toe_capability(struct port_info *pi, int enable) 8244 { 8245 int rc; 8246 struct adapter *sc = pi->adapter; 8247 8248 ASSERT_SYNCHRONIZED_OP(sc); 8249 8250 if (!is_offload(sc)) 8251 return (ENODEV); 8252 8253 if (enable) { 8254 if (!(sc->flags & FULL_INIT_DONE)) { 8255 rc = cxgbe_init_synchronized(pi); 8256 if (rc) 8257 return (rc); 8258 } 8259 8260 if (isset(&sc->offload_map, pi->port_id)) 8261 return (0); 8262 8263 if (!(sc->flags & TOM_INIT_DONE)) { 8264 rc = t4_activate_uld(sc, ULD_TOM); 8265 if (rc == EAGAIN) { 8266 log(LOG_WARNING, 8267 "You must kldload t4_tom.ko before trying " 8268 "to enable TOE on a cxgbe interface.\n"); 8269 } 8270 if (rc != 0) 8271 return (rc); 8272 KASSERT(sc->tom_softc != NULL, 8273 ("%s: TOM activated but softc NULL", __func__)); 8274 KASSERT(sc->flags & TOM_INIT_DONE, 8275 ("%s: TOM activated but flag not set", __func__)); 8276 } 8277 8278 setbit(&sc->offload_map, pi->port_id); 8279 } else { 8280 if (!isset(&sc->offload_map, pi->port_id)) 8281 return (0); 8282 8283 KASSERT(sc->flags & TOM_INIT_DONE, 8284 ("%s: TOM never initialized?", __func__)); 8285 clrbit(&sc->offload_map, pi->port_id); 8286 } 8287 8288 return (0); 8289 } 8290 8291 /* 8292 * Add an upper layer driver to the global list. 8293 */ 8294 int 8295 t4_register_uld(struct uld_info *ui) 8296 { 8297 int rc = 0; 8298 struct uld_info *u; 8299 8300 sx_xlock(&t4_uld_list_lock); 8301 SLIST_FOREACH(u, &t4_uld_list, link) { 8302 if (u->uld_id == ui->uld_id) { 8303 rc = EEXIST; 8304 goto done; 8305 } 8306 } 8307 8308 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 8309 ui->refcount = 0; 8310 done: 8311 sx_xunlock(&t4_uld_list_lock); 8312 return (rc); 8313 } 8314 8315 int 8316 t4_unregister_uld(struct uld_info *ui) 8317 { 8318 int rc = EINVAL; 8319 struct uld_info *u; 8320 8321 sx_xlock(&t4_uld_list_lock); 8322 8323 SLIST_FOREACH(u, &t4_uld_list, link) { 8324 if (u == ui) { 8325 if (ui->refcount > 0) { 8326 rc = EBUSY; 8327 goto done; 8328 } 8329 8330 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 8331 rc = 0; 8332 goto done; 8333 } 8334 } 8335 done: 8336 sx_xunlock(&t4_uld_list_lock); 8337 return (rc); 8338 } 8339 8340 int 8341 t4_activate_uld(struct adapter *sc, int id) 8342 { 8343 int rc = EAGAIN; 8344 struct uld_info *ui; 8345 8346 ASSERT_SYNCHRONIZED_OP(sc); 8347 8348 sx_slock(&t4_uld_list_lock); 8349 8350 SLIST_FOREACH(ui, &t4_uld_list, link) { 8351 if (ui->uld_id == id) { 8352 if (!(sc->flags & FULL_INIT_DONE)) { 8353 rc = adapter_full_init(sc); 8354 if (rc != 0) 8355 goto done; 8356 } 8357 8358 rc = ui->activate(sc); 8359 if (rc == 0) 8360 ui->refcount++; 8361 goto done; 8362 } 8363 } 8364 done: 8365 sx_sunlock(&t4_uld_list_lock); 8366 8367 return (rc); 8368 } 8369 8370 int 8371 t4_deactivate_uld(struct adapter *sc, int id) 8372 { 8373 int rc = EINVAL; 8374 struct uld_info *ui; 8375 8376 ASSERT_SYNCHRONIZED_OP(sc); 8377 8378 sx_slock(&t4_uld_list_lock); 8379 8380 SLIST_FOREACH(ui, &t4_uld_list, link) { 8381 if (ui->uld_id == id) { 8382 rc = ui->deactivate(sc); 8383 if (rc == 0) 8384 ui->refcount--; 8385 goto done; 8386 } 8387 } 8388 done: 8389 sx_sunlock(&t4_uld_list_lock); 8390 8391 return (rc); 8392 } 8393 #endif 8394 8395 /* 8396 * Come up with reasonable defaults for some of the tunables, provided they're 8397 * not set by the user (in which case we'll use the values as is). 8398 */ 8399 static void 8400 tweak_tunables(void) 8401 { 8402 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 8403 8404 if (t4_ntxq10g < 1) 8405 t4_ntxq10g = min(nc, NTXQ_10G); 8406 8407 if (t4_ntxq1g < 1) 8408 t4_ntxq1g = min(nc, NTXQ_1G); 8409 8410 if (t4_nrxq10g < 1) 8411 t4_nrxq10g = min(nc, NRXQ_10G); 8412 8413 if (t4_nrxq1g < 1) 8414 t4_nrxq1g = min(nc, NRXQ_1G); 8415 8416 #ifdef TCP_OFFLOAD 8417 if (t4_nofldtxq10g < 1) 8418 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 8419 8420 if (t4_nofldtxq1g < 1) 8421 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 8422 8423 if (t4_nofldrxq10g < 1) 8424 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 8425 8426 if (t4_nofldrxq1g < 1) 8427 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 8428 8429 if (t4_toecaps_allowed == -1) 8430 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 8431 #else 8432 if (t4_toecaps_allowed == -1) 8433 t4_toecaps_allowed = 0; 8434 #endif 8435 8436 #ifdef DEV_NETMAP 8437 if (t4_nnmtxq10g < 1) 8438 t4_nnmtxq10g = min(nc, NNMTXQ_10G); 8439 8440 if (t4_nnmtxq1g < 1) 8441 t4_nnmtxq1g = min(nc, NNMTXQ_1G); 8442 8443 if (t4_nnmrxq10g < 1) 8444 t4_nnmrxq10g = min(nc, NNMRXQ_10G); 8445 8446 if (t4_nnmrxq1g < 1) 8447 t4_nnmrxq1g = min(nc, NNMRXQ_1G); 8448 #endif 8449 8450 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 8451 t4_tmr_idx_10g = TMR_IDX_10G; 8452 8453 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 8454 t4_pktc_idx_10g = PKTC_IDX_10G; 8455 8456 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 8457 t4_tmr_idx_1g = TMR_IDX_1G; 8458 8459 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 8460 t4_pktc_idx_1g = PKTC_IDX_1G; 8461 8462 if (t4_qsize_txq < 128) 8463 t4_qsize_txq = 128; 8464 8465 if (t4_qsize_rxq < 128) 8466 t4_qsize_rxq = 128; 8467 while (t4_qsize_rxq & 7) 8468 t4_qsize_rxq++; 8469 8470 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 8471 } 8472 8473 static struct sx mlu; /* mod load unload */ 8474 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload"); 8475 8476 static int 8477 mod_event(module_t mod, int cmd, void *arg) 8478 { 8479 int rc = 0; 8480 static int loaded = 0; 8481 8482 switch (cmd) { 8483 case MOD_LOAD: 8484 sx_xlock(&mlu); 8485 if (loaded++ == 0) { 8486 t4_sge_modload(); 8487 sx_init(&t4_list_lock, "T4/T5 adapters"); 8488 SLIST_INIT(&t4_list); 8489 #ifdef TCP_OFFLOAD 8490 sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); 8491 SLIST_INIT(&t4_uld_list); 8492 #endif 8493 t4_tracer_modload(); 8494 tweak_tunables(); 8495 } 8496 sx_xunlock(&mlu); 8497 break; 8498 8499 case MOD_UNLOAD: 8500 sx_xlock(&mlu); 8501 if (--loaded == 0) { 8502 int tries; 8503 8504 sx_slock(&t4_list_lock); 8505 if (!SLIST_EMPTY(&t4_list)) { 8506 rc = EBUSY; 8507 sx_sunlock(&t4_list_lock); 8508 goto done_unload; 8509 } 8510 #ifdef TCP_OFFLOAD 8511 sx_slock(&t4_uld_list_lock); 8512 if (!SLIST_EMPTY(&t4_uld_list)) { 8513 rc = EBUSY; 8514 sx_sunlock(&t4_uld_list_lock); 8515 sx_sunlock(&t4_list_lock); 8516 goto done_unload; 8517 } 8518 #endif 8519 tries = 0; 8520 while (tries++ < 5 && t4_sge_extfree_refs() != 0) { 8521 uprintf("%ju clusters with custom free routine " 8522 "still is use.\n", t4_sge_extfree_refs()); 8523 pause("t4unload", 2 * hz); 8524 } 8525 #ifdef TCP_OFFLOAD 8526 sx_sunlock(&t4_uld_list_lock); 8527 #endif 8528 sx_sunlock(&t4_list_lock); 8529 8530 if (t4_sge_extfree_refs() == 0) { 8531 t4_tracer_modunload(); 8532 #ifdef TCP_OFFLOAD 8533 sx_destroy(&t4_uld_list_lock); 8534 #endif 8535 sx_destroy(&t4_list_lock); 8536 t4_sge_modunload(); 8537 loaded = 0; 8538 } else { 8539 rc = EBUSY; 8540 loaded++; /* undo earlier decrement */ 8541 } 8542 } 8543 done_unload: 8544 sx_xunlock(&mlu); 8545 break; 8546 } 8547 8548 return (rc); 8549 } 8550 8551 static devclass_t t4_devclass, t5_devclass; 8552 static devclass_t cxgbe_devclass, cxl_devclass; 8553 8554 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 8555 MODULE_VERSION(t4nex, 1); 8556 MODULE_DEPEND(t4nex, firmware, 1, 1, 1); 8557 8558 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 8559 MODULE_VERSION(t5nex, 1); 8560 MODULE_DEPEND(t5nex, firmware, 1, 1, 1); 8561 8562 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 8563 MODULE_VERSION(cxgbe, 1); 8564 8565 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 8566 MODULE_VERSION(cxl, 1); 8567