1 /*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_inet.h" 32 #include "opt_inet6.h" 33 34 #include <sys/param.h> 35 #include <sys/conf.h> 36 #include <sys/priv.h> 37 #include <sys/kernel.h> 38 #include <sys/bus.h> 39 #include <sys/module.h> 40 #include <sys/malloc.h> 41 #include <sys/queue.h> 42 #include <sys/taskqueue.h> 43 #include <sys/pciio.h> 44 #include <dev/pci/pcireg.h> 45 #include <dev/pci/pcivar.h> 46 #include <dev/pci/pci_private.h> 47 #include <sys/firmware.h> 48 #include <sys/sbuf.h> 49 #include <sys/smp.h> 50 #include <sys/socket.h> 51 #include <sys/sockio.h> 52 #include <sys/sysctl.h> 53 #include <net/ethernet.h> 54 #include <net/if.h> 55 #include <net/if_types.h> 56 #include <net/if_dl.h> 57 #include <net/if_vlan_var.h> 58 #if defined(__i386__) || defined(__amd64__) 59 #include <vm/vm.h> 60 #include <vm/pmap.h> 61 #endif 62 63 #include "common/common.h" 64 #include "common/t4_msg.h" 65 #include "common/t4_regs.h" 66 #include "common/t4_regs_values.h" 67 #include "t4_ioctl.h" 68 #include "t4_l2t.h" 69 70 /* T4 bus driver interface */ 71 static int t4_probe(device_t); 72 static int t4_attach(device_t); 73 static int t4_detach(device_t); 74 static device_method_t t4_methods[] = { 75 DEVMETHOD(device_probe, t4_probe), 76 DEVMETHOD(device_attach, t4_attach), 77 DEVMETHOD(device_detach, t4_detach), 78 79 DEVMETHOD_END 80 }; 81 static driver_t t4_driver = { 82 "t4nex", 83 t4_methods, 84 sizeof(struct adapter) 85 }; 86 87 88 /* T4 port (cxgbe) interface */ 89 static int cxgbe_probe(device_t); 90 static int cxgbe_attach(device_t); 91 static int cxgbe_detach(device_t); 92 static device_method_t cxgbe_methods[] = { 93 DEVMETHOD(device_probe, cxgbe_probe), 94 DEVMETHOD(device_attach, cxgbe_attach), 95 DEVMETHOD(device_detach, cxgbe_detach), 96 { 0, 0 } 97 }; 98 static driver_t cxgbe_driver = { 99 "cxgbe", 100 cxgbe_methods, 101 sizeof(struct port_info) 102 }; 103 104 static d_ioctl_t t4_ioctl; 105 static d_open_t t4_open; 106 static d_close_t t4_close; 107 108 static struct cdevsw t4_cdevsw = { 109 .d_version = D_VERSION, 110 .d_flags = 0, 111 .d_open = t4_open, 112 .d_close = t4_close, 113 .d_ioctl = t4_ioctl, 114 .d_name = "t4nex", 115 }; 116 117 /* T5 bus driver interface */ 118 static int t5_probe(device_t); 119 static device_method_t t5_methods[] = { 120 DEVMETHOD(device_probe, t5_probe), 121 DEVMETHOD(device_attach, t4_attach), 122 DEVMETHOD(device_detach, t4_detach), 123 124 DEVMETHOD_END 125 }; 126 static driver_t t5_driver = { 127 "t5nex", 128 t5_methods, 129 sizeof(struct adapter) 130 }; 131 132 133 /* T5 port (cxl) interface */ 134 static driver_t cxl_driver = { 135 "cxl", 136 cxgbe_methods, 137 sizeof(struct port_info) 138 }; 139 140 static struct cdevsw t5_cdevsw = { 141 .d_version = D_VERSION, 142 .d_flags = 0, 143 .d_open = t4_open, 144 .d_close = t4_close, 145 .d_ioctl = t4_ioctl, 146 .d_name = "t5nex", 147 }; 148 149 /* ifnet + media interface */ 150 static void cxgbe_init(void *); 151 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 152 static int cxgbe_transmit(struct ifnet *, struct mbuf *); 153 static void cxgbe_qflush(struct ifnet *); 154 static int cxgbe_media_change(struct ifnet *); 155 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 156 157 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 158 159 /* 160 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 161 * then ADAPTER_LOCK, then t4_uld_list_lock. 162 */ 163 static struct sx t4_list_lock; 164 SLIST_HEAD(, adapter) t4_list; 165 #ifdef TCP_OFFLOAD 166 static struct sx t4_uld_list_lock; 167 SLIST_HEAD(, uld_info) t4_uld_list; 168 #endif 169 170 /* 171 * Tunables. See tweak_tunables() too. 172 * 173 * Each tunable is set to a default value here if it's known at compile-time. 174 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 175 * provide a reasonable default when the driver is loaded. 176 * 177 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 178 * T5 are under hw.cxl. 179 */ 180 181 /* 182 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 183 */ 184 #define NTXQ_10G 16 185 static int t4_ntxq10g = -1; 186 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 187 188 #define NRXQ_10G 8 189 static int t4_nrxq10g = -1; 190 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 191 192 #define NTXQ_1G 4 193 static int t4_ntxq1g = -1; 194 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 195 196 #define NRXQ_1G 2 197 static int t4_nrxq1g = -1; 198 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 199 200 static int t4_rsrv_noflowq = 0; 201 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq); 202 203 #ifdef TCP_OFFLOAD 204 #define NOFLDTXQ_10G 8 205 static int t4_nofldtxq10g = -1; 206 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 207 208 #define NOFLDRXQ_10G 2 209 static int t4_nofldrxq10g = -1; 210 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 211 212 #define NOFLDTXQ_1G 2 213 static int t4_nofldtxq1g = -1; 214 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 215 216 #define NOFLDRXQ_1G 1 217 static int t4_nofldrxq1g = -1; 218 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 219 #endif 220 221 #ifdef DEV_NETMAP 222 #define NNMTXQ_10G 2 223 static int t4_nnmtxq10g = -1; 224 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g); 225 226 #define NNMRXQ_10G 2 227 static int t4_nnmrxq10g = -1; 228 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g); 229 230 #define NNMTXQ_1G 1 231 static int t4_nnmtxq1g = -1; 232 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g); 233 234 #define NNMRXQ_1G 1 235 static int t4_nnmrxq1g = -1; 236 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g); 237 #endif 238 239 /* 240 * Holdoff parameters for 10G and 1G ports. 241 */ 242 #define TMR_IDX_10G 1 243 static int t4_tmr_idx_10g = TMR_IDX_10G; 244 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 245 246 #define PKTC_IDX_10G (-1) 247 static int t4_pktc_idx_10g = PKTC_IDX_10G; 248 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 249 250 #define TMR_IDX_1G 1 251 static int t4_tmr_idx_1g = TMR_IDX_1G; 252 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 253 254 #define PKTC_IDX_1G (-1) 255 static int t4_pktc_idx_1g = PKTC_IDX_1G; 256 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 257 258 /* 259 * Size (# of entries) of each tx and rx queue. 260 */ 261 static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 262 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 263 264 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 265 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 266 267 /* 268 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 269 */ 270 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 271 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 272 273 /* 274 * Configuration file. 275 */ 276 #define DEFAULT_CF "default" 277 #define FLASH_CF "flash" 278 #define UWIRE_CF "uwire" 279 #define FPGA_CF "fpga" 280 static char t4_cfg_file[32] = DEFAULT_CF; 281 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 282 283 /* 284 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 285 * encouraged respectively). 286 */ 287 static unsigned int t4_fw_install = 1; 288 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 289 290 /* 291 * ASIC features that will be used. Disable the ones you don't want so that the 292 * chip resources aren't wasted on features that will not be used. 293 */ 294 static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 295 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 296 297 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 298 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 299 300 static int t4_toecaps_allowed = -1; 301 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 302 303 static int t4_rdmacaps_allowed = 0; 304 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 305 306 static int t4_iscsicaps_allowed = 0; 307 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 308 309 static int t4_fcoecaps_allowed = 0; 310 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 311 312 static int t5_write_combine = 0; 313 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 314 315 struct intrs_and_queues { 316 uint16_t intr_type; /* INTx, MSI, or MSI-X */ 317 uint16_t nirq; /* Total # of vectors */ 318 uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */ 319 uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */ 320 uint16_t ntxq10g; /* # of NIC txq's for each 10G port */ 321 uint16_t nrxq10g; /* # of NIC rxq's for each 10G port */ 322 uint16_t ntxq1g; /* # of NIC txq's for each 1G port */ 323 uint16_t nrxq1g; /* # of NIC rxq's for each 1G port */ 324 uint16_t rsrv_noflowq; /* Flag whether to reserve queue 0 */ 325 #ifdef TCP_OFFLOAD 326 uint16_t nofldtxq10g; /* # of TOE txq's for each 10G port */ 327 uint16_t nofldrxq10g; /* # of TOE rxq's for each 10G port */ 328 uint16_t nofldtxq1g; /* # of TOE txq's for each 1G port */ 329 uint16_t nofldrxq1g; /* # of TOE rxq's for each 1G port */ 330 #endif 331 #ifdef DEV_NETMAP 332 uint16_t nnmtxq10g; /* # of netmap txq's for each 10G port */ 333 uint16_t nnmrxq10g; /* # of netmap rxq's for each 10G port */ 334 uint16_t nnmtxq1g; /* # of netmap txq's for each 1G port */ 335 uint16_t nnmrxq1g; /* # of netmap rxq's for each 1G port */ 336 #endif 337 }; 338 339 struct filter_entry { 340 uint32_t valid:1; /* filter allocated and valid */ 341 uint32_t locked:1; /* filter is administratively locked */ 342 uint32_t pending:1; /* filter action is pending firmware reply */ 343 uint32_t smtidx:8; /* Source MAC Table index for smac */ 344 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 345 346 struct t4_filter_specification fs; 347 }; 348 349 static int map_bars_0_and_4(struct adapter *); 350 static int map_bar_2(struct adapter *); 351 static void setup_memwin(struct adapter *); 352 static int validate_mem_range(struct adapter *, uint32_t, int); 353 static int fwmtype_to_hwmtype(int); 354 static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 355 uint32_t *); 356 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 357 static uint32_t position_memwin(struct adapter *, int, uint32_t); 358 static int cfg_itype_and_nqueues(struct adapter *, int, int, 359 struct intrs_and_queues *); 360 static int prep_firmware(struct adapter *); 361 static int partition_resources(struct adapter *, const struct firmware *, 362 const char *); 363 static int get_params__pre_init(struct adapter *); 364 static int get_params__post_init(struct adapter *); 365 static int set_params__post_init(struct adapter *); 366 static void t4_set_desc(struct adapter *); 367 static void build_medialist(struct port_info *, struct ifmedia *); 368 static int cxgbe_init_synchronized(struct port_info *); 369 static int cxgbe_uninit_synchronized(struct port_info *); 370 static int setup_intr_handlers(struct adapter *); 371 static void quiesce_eq(struct adapter *, struct sge_eq *); 372 static void quiesce_iq(struct adapter *, struct sge_iq *); 373 static void quiesce_fl(struct adapter *, struct sge_fl *); 374 static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 375 driver_intr_t *, void *, char *); 376 static int t4_free_irq(struct adapter *, struct irq *); 377 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 378 unsigned int); 379 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 380 static void cxgbe_tick(void *); 381 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 382 static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 383 struct mbuf *); 384 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 385 static int fw_msg_not_handled(struct adapter *, const __be64 *); 386 static int t4_sysctls(struct adapter *); 387 static int cxgbe_sysctls(struct port_info *); 388 static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 389 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 390 static int sysctl_btphy(SYSCTL_HANDLER_ARGS); 391 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); 392 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 393 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 394 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 395 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 396 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 397 static int sysctl_temperature(SYSCTL_HANDLER_ARGS); 398 #ifdef SBUF_DRAIN 399 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 400 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 401 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 402 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); 403 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); 404 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 405 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 406 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 407 static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 408 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 409 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 410 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 411 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); 412 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 413 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); 414 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 415 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 416 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 417 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 418 static int sysctl_tids(SYSCTL_HANDLER_ARGS); 419 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 420 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 421 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 422 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 423 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 424 #endif 425 static inline void txq_start(struct ifnet *, struct sge_txq *); 426 static uint32_t fconf_to_mode(uint32_t); 427 static uint32_t mode_to_fconf(uint32_t); 428 static uint32_t fspec_to_fconf(struct t4_filter_specification *); 429 static int get_filter_mode(struct adapter *, uint32_t *); 430 static int set_filter_mode(struct adapter *, uint32_t); 431 static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 432 static int get_filter(struct adapter *, struct t4_filter *); 433 static int set_filter(struct adapter *, struct t4_filter *); 434 static int del_filter(struct adapter *, struct t4_filter *); 435 static void clear_filter(struct filter_entry *); 436 static int set_filter_wr(struct adapter *, int); 437 static int del_filter_wr(struct adapter *, int); 438 static int get_sge_context(struct adapter *, struct t4_sge_context *); 439 static int load_fw(struct adapter *, struct t4_data *); 440 static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 441 static int read_i2c(struct adapter *, struct t4_i2c_data *); 442 static int set_sched_class(struct adapter *, struct t4_sched_params *); 443 static int set_sched_queue(struct adapter *, struct t4_sched_queue *); 444 #ifdef TCP_OFFLOAD 445 static int toe_capability(struct port_info *, int); 446 #endif 447 static int mod_event(module_t, int, void *); 448 449 struct { 450 uint16_t device; 451 char *desc; 452 } t4_pciids[] = { 453 {0xa000, "Chelsio Terminator 4 FPGA"}, 454 {0x4400, "Chelsio T440-dbg"}, 455 {0x4401, "Chelsio T420-CR"}, 456 {0x4402, "Chelsio T422-CR"}, 457 {0x4403, "Chelsio T440-CR"}, 458 {0x4404, "Chelsio T420-BCH"}, 459 {0x4405, "Chelsio T440-BCH"}, 460 {0x4406, "Chelsio T440-CH"}, 461 {0x4407, "Chelsio T420-SO"}, 462 {0x4408, "Chelsio T420-CX"}, 463 {0x4409, "Chelsio T420-BT"}, 464 {0x440a, "Chelsio T404-BT"}, 465 {0x440e, "Chelsio T440-LP-CR"}, 466 }, t5_pciids[] = { 467 {0xb000, "Chelsio Terminator 5 FPGA"}, 468 {0x5400, "Chelsio T580-dbg"}, 469 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ 470 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ 471 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ 472 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ 473 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ 474 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ 475 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ 476 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ 477 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 478 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ 479 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ 480 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ 481 #ifdef notyet 482 {0x5404, "Chelsio T520-BCH"}, 483 {0x5405, "Chelsio T540-BCH"}, 484 {0x5406, "Chelsio T540-CH"}, 485 {0x5408, "Chelsio T520-CX"}, 486 {0x540b, "Chelsio B520-SR"}, 487 {0x540c, "Chelsio B504-BT"}, 488 {0x540f, "Chelsio Amsterdam"}, 489 {0x5413, "Chelsio T580-CHR"}, 490 #endif 491 }; 492 493 #ifdef TCP_OFFLOAD 494 /* 495 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 496 * exactly the same for both rxq and ofld_rxq. 497 */ 498 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 499 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 500 #endif 501 502 /* No easy way to include t4_msg.h before adapter.h so we check this way */ 503 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 504 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 505 506 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); 507 508 static int 509 t4_probe(device_t dev) 510 { 511 int i; 512 uint16_t v = pci_get_vendor(dev); 513 uint16_t d = pci_get_device(dev); 514 uint8_t f = pci_get_function(dev); 515 516 if (v != PCI_VENDOR_ID_CHELSIO) 517 return (ENXIO); 518 519 /* Attach only to PF0 of the FPGA */ 520 if (d == 0xa000 && f != 0) 521 return (ENXIO); 522 523 for (i = 0; i < nitems(t4_pciids); i++) { 524 if (d == t4_pciids[i].device) { 525 device_set_desc(dev, t4_pciids[i].desc); 526 return (BUS_PROBE_DEFAULT); 527 } 528 } 529 530 return (ENXIO); 531 } 532 533 static int 534 t5_probe(device_t dev) 535 { 536 int i; 537 uint16_t v = pci_get_vendor(dev); 538 uint16_t d = pci_get_device(dev); 539 uint8_t f = pci_get_function(dev); 540 541 if (v != PCI_VENDOR_ID_CHELSIO) 542 return (ENXIO); 543 544 /* Attach only to PF0 of the FPGA */ 545 if (d == 0xb000 && f != 0) 546 return (ENXIO); 547 548 for (i = 0; i < nitems(t5_pciids); i++) { 549 if (d == t5_pciids[i].device) { 550 device_set_desc(dev, t5_pciids[i].desc); 551 return (BUS_PROBE_DEFAULT); 552 } 553 } 554 555 return (ENXIO); 556 } 557 558 static int 559 t4_attach(device_t dev) 560 { 561 struct adapter *sc; 562 int rc = 0, i, n10g, n1g, rqidx, tqidx; 563 struct intrs_and_queues iaq; 564 struct sge *s; 565 #ifdef TCP_OFFLOAD 566 int ofld_rqidx, ofld_tqidx; 567 #endif 568 #ifdef DEV_NETMAP 569 int nm_rqidx, nm_tqidx; 570 #endif 571 const char *pcie_ts; 572 573 sc = device_get_softc(dev); 574 sc->dev = dev; 575 576 pci_enable_busmaster(dev); 577 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 578 uint32_t v; 579 580 pci_set_max_read_req(dev, 4096); 581 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 582 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 583 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 584 } 585 586 sc->traceq = -1; 587 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); 588 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", 589 device_get_nameunit(dev)); 590 591 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 592 device_get_nameunit(dev)); 593 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 594 sx_xlock(&t4_list_lock); 595 SLIST_INSERT_HEAD(&t4_list, sc, link); 596 sx_xunlock(&t4_list_lock); 597 598 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 599 TAILQ_INIT(&sc->sfl); 600 callout_init(&sc->sfl_callout, CALLOUT_MPSAFE); 601 602 rc = map_bars_0_and_4(sc); 603 if (rc != 0) 604 goto done; /* error message displayed already */ 605 606 /* 607 * This is the real PF# to which we're attaching. Works from within PCI 608 * passthrough environments too, where pci_get_function() could return a 609 * different PF# depending on the passthrough configuration. We need to 610 * use the real PF# in all our communication with the firmware. 611 */ 612 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 613 sc->mbox = sc->pf; 614 615 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 616 sc->an_handler = an_not_handled; 617 for (i = 0; i < nitems(sc->cpl_handler); i++) 618 sc->cpl_handler[i] = cpl_not_handled; 619 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 620 sc->fw_msg_handler[i] = fw_msg_not_handled; 621 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 622 t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt); 623 t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt); 624 t4_init_sge_cpl_handlers(sc); 625 626 /* Prepare the adapter for operation */ 627 rc = -t4_prep_adapter(sc); 628 if (rc != 0) { 629 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 630 goto done; 631 } 632 633 /* 634 * Do this really early, with the memory windows set up even before the 635 * character device. The userland tool's register i/o and mem read 636 * will work even in "recovery mode". 637 */ 638 setup_memwin(sc); 639 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 640 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 641 device_get_nameunit(dev)); 642 if (sc->cdev == NULL) 643 device_printf(dev, "failed to create nexus char device.\n"); 644 else 645 sc->cdev->si_drv1 = sc; 646 647 /* Go no further if recovery mode has been requested. */ 648 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 649 device_printf(dev, "recovery mode.\n"); 650 goto done; 651 } 652 653 /* Prepare the firmware for operation */ 654 rc = prep_firmware(sc); 655 if (rc != 0) 656 goto done; /* error message displayed already */ 657 658 rc = get_params__post_init(sc); 659 if (rc != 0) 660 goto done; /* error message displayed already */ 661 662 rc = set_params__post_init(sc); 663 if (rc != 0) 664 goto done; /* error message displayed already */ 665 666 rc = map_bar_2(sc); 667 if (rc != 0) 668 goto done; /* error message displayed already */ 669 670 rc = t4_create_dma_tag(sc); 671 if (rc != 0) 672 goto done; /* error message displayed already */ 673 674 /* 675 * First pass over all the ports - allocate VIs and initialize some 676 * basic parameters like mac address, port type, etc. We also figure 677 * out whether a port is 10G or 1G and use that information when 678 * calculating how many interrupts to attempt to allocate. 679 */ 680 n10g = n1g = 0; 681 for_each_port(sc, i) { 682 struct port_info *pi; 683 684 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 685 sc->port[i] = pi; 686 687 /* These must be set before t4_port_init */ 688 pi->adapter = sc; 689 pi->port_id = i; 690 691 /* Allocate the vi and initialize parameters like mac addr */ 692 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 693 if (rc != 0) { 694 device_printf(dev, "unable to initialize port %d: %d\n", 695 i, rc); 696 free(pi, M_CXGBE); 697 sc->port[i] = NULL; 698 goto done; 699 } 700 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 701 if (rc != 0) { 702 device_printf(dev, "port %d l1cfg failed: %d\n", i, rc); 703 free(pi, M_CXGBE); 704 sc->port[i] = NULL; 705 goto done; 706 } 707 708 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 709 device_get_nameunit(dev), i); 710 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 711 sc->chan_map[pi->tx_chan] = i; 712 713 if (is_10G_port(pi) || is_40G_port(pi)) { 714 n10g++; 715 pi->tmr_idx = t4_tmr_idx_10g; 716 pi->pktc_idx = t4_pktc_idx_10g; 717 } else { 718 n1g++; 719 pi->tmr_idx = t4_tmr_idx_1g; 720 pi->pktc_idx = t4_pktc_idx_1g; 721 } 722 723 pi->xact_addr_filt = -1; 724 pi->linkdnrc = -1; 725 726 pi->qsize_rxq = t4_qsize_rxq; 727 pi->qsize_txq = t4_qsize_txq; 728 729 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 730 if (pi->dev == NULL) { 731 device_printf(dev, 732 "failed to add device for port %d.\n", i); 733 rc = ENXIO; 734 goto done; 735 } 736 device_set_softc(pi->dev, pi); 737 } 738 739 /* 740 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 741 */ 742 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 743 if (rc != 0) 744 goto done; /* error message displayed already */ 745 746 sc->intr_type = iaq.intr_type; 747 sc->intr_count = iaq.nirq; 748 749 s = &sc->sge; 750 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 751 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 752 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 753 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 754 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 755 #ifdef TCP_OFFLOAD 756 if (is_offload(sc)) { 757 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 758 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 759 s->neq += s->nofldtxq + s->nofldrxq; 760 s->niq += s->nofldrxq; 761 762 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 763 M_CXGBE, M_ZERO | M_WAITOK); 764 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 765 M_CXGBE, M_ZERO | M_WAITOK); 766 } 767 #endif 768 #ifdef DEV_NETMAP 769 s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g; 770 s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g; 771 s->neq += s->nnmtxq + s->nnmrxq; 772 s->niq += s->nnmrxq; 773 774 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq), 775 M_CXGBE, M_ZERO | M_WAITOK); 776 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq), 777 M_CXGBE, M_ZERO | M_WAITOK); 778 #endif 779 780 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 781 M_ZERO | M_WAITOK); 782 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 783 M_ZERO | M_WAITOK); 784 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 785 M_ZERO | M_WAITOK); 786 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 787 M_ZERO | M_WAITOK); 788 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 789 M_ZERO | M_WAITOK); 790 791 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 792 M_ZERO | M_WAITOK); 793 794 t4_init_l2t(sc, M_WAITOK); 795 796 /* 797 * Second pass over the ports. This time we know the number of rx and 798 * tx queues that each port should get. 799 */ 800 rqidx = tqidx = 0; 801 #ifdef TCP_OFFLOAD 802 ofld_rqidx = ofld_tqidx = 0; 803 #endif 804 #ifdef DEV_NETMAP 805 nm_rqidx = nm_tqidx = 0; 806 #endif 807 for_each_port(sc, i) { 808 struct port_info *pi = sc->port[i]; 809 810 if (pi == NULL) 811 continue; 812 813 pi->first_rxq = rqidx; 814 pi->first_txq = tqidx; 815 if (is_10G_port(pi) || is_40G_port(pi)) { 816 pi->flags |= iaq.intr_flags_10g; 817 pi->nrxq = iaq.nrxq10g; 818 pi->ntxq = iaq.ntxq10g; 819 } else { 820 pi->flags |= iaq.intr_flags_1g; 821 pi->nrxq = iaq.nrxq1g; 822 pi->ntxq = iaq.ntxq1g; 823 } 824 825 if (pi->ntxq > 1) 826 pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0; 827 else 828 pi->rsrv_noflowq = 0; 829 830 rqidx += pi->nrxq; 831 tqidx += pi->ntxq; 832 #ifdef TCP_OFFLOAD 833 if (is_offload(sc)) { 834 pi->first_ofld_rxq = ofld_rqidx; 835 pi->first_ofld_txq = ofld_tqidx; 836 if (is_10G_port(pi) || is_40G_port(pi)) { 837 pi->nofldrxq = iaq.nofldrxq10g; 838 pi->nofldtxq = iaq.nofldtxq10g; 839 } else { 840 pi->nofldrxq = iaq.nofldrxq1g; 841 pi->nofldtxq = iaq.nofldtxq1g; 842 } 843 ofld_rqidx += pi->nofldrxq; 844 ofld_tqidx += pi->nofldtxq; 845 } 846 #endif 847 #ifdef DEV_NETMAP 848 pi->first_nm_rxq = nm_rqidx; 849 pi->first_nm_txq = nm_tqidx; 850 if (is_10G_port(pi) || is_40G_port(pi)) { 851 pi->nnmrxq = iaq.nnmrxq10g; 852 pi->nnmtxq = iaq.nnmtxq10g; 853 } else { 854 pi->nnmrxq = iaq.nnmrxq1g; 855 pi->nnmtxq = iaq.nnmtxq1g; 856 } 857 nm_rqidx += pi->nnmrxq; 858 nm_tqidx += pi->nnmtxq; 859 #endif 860 } 861 862 rc = setup_intr_handlers(sc); 863 if (rc != 0) { 864 device_printf(dev, 865 "failed to setup interrupt handlers: %d\n", rc); 866 goto done; 867 } 868 869 rc = bus_generic_attach(dev); 870 if (rc != 0) { 871 device_printf(dev, 872 "failed to attach all child ports: %d\n", rc); 873 goto done; 874 } 875 876 switch (sc->params.pci.speed) { 877 case 0x1: 878 pcie_ts = "2.5"; 879 break; 880 case 0x2: 881 pcie_ts = "5.0"; 882 break; 883 case 0x3: 884 pcie_ts = "8.0"; 885 break; 886 default: 887 pcie_ts = "??"; 888 break; 889 } 890 device_printf(dev, 891 "PCIe x%d (%s GTS/s) (%d), %d ports, %d %s interrupt%s, %d eq, %d iq\n", 892 sc->params.pci.width, pcie_ts, sc->params.pci.speed, 893 sc->params.nports, sc->intr_count, 894 sc->intr_type == INTR_MSIX ? "MSI-X" : 895 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 896 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 897 898 t4_set_desc(sc); 899 900 done: 901 if (rc != 0 && sc->cdev) { 902 /* cdev was created and so cxgbetool works; recover that way. */ 903 device_printf(dev, 904 "error during attach, adapter is now in recovery mode.\n"); 905 rc = 0; 906 } 907 908 if (rc != 0) 909 t4_detach(dev); 910 else 911 t4_sysctls(sc); 912 913 return (rc); 914 } 915 916 /* 917 * Idempotent 918 */ 919 static int 920 t4_detach(device_t dev) 921 { 922 struct adapter *sc; 923 struct port_info *pi; 924 int i, rc; 925 926 sc = device_get_softc(dev); 927 928 if (sc->flags & FULL_INIT_DONE) 929 t4_intr_disable(sc); 930 931 if (sc->cdev) { 932 destroy_dev(sc->cdev); 933 sc->cdev = NULL; 934 } 935 936 rc = bus_generic_detach(dev); 937 if (rc) { 938 device_printf(dev, 939 "failed to detach child devices: %d\n", rc); 940 return (rc); 941 } 942 943 for (i = 0; i < sc->intr_count; i++) 944 t4_free_irq(sc, &sc->irq[i]); 945 946 for (i = 0; i < MAX_NPORTS; i++) { 947 pi = sc->port[i]; 948 if (pi) { 949 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid); 950 if (pi->dev) 951 device_delete_child(dev, pi->dev); 952 953 mtx_destroy(&pi->pi_lock); 954 free(pi, M_CXGBE); 955 } 956 } 957 958 if (sc->flags & FULL_INIT_DONE) 959 adapter_full_uninit(sc); 960 961 if (sc->flags & FW_OK) 962 t4_fw_bye(sc, sc->mbox); 963 964 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 965 pci_release_msi(dev); 966 967 if (sc->regs_res) 968 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 969 sc->regs_res); 970 971 if (sc->udbs_res) 972 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 973 sc->udbs_res); 974 975 if (sc->msix_res) 976 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 977 sc->msix_res); 978 979 if (sc->l2t) 980 t4_free_l2t(sc->l2t); 981 982 #ifdef TCP_OFFLOAD 983 free(sc->sge.ofld_rxq, M_CXGBE); 984 free(sc->sge.ofld_txq, M_CXGBE); 985 #endif 986 #ifdef DEV_NETMAP 987 free(sc->sge.nm_rxq, M_CXGBE); 988 free(sc->sge.nm_txq, M_CXGBE); 989 #endif 990 free(sc->irq, M_CXGBE); 991 free(sc->sge.rxq, M_CXGBE); 992 free(sc->sge.txq, M_CXGBE); 993 free(sc->sge.ctrlq, M_CXGBE); 994 free(sc->sge.iqmap, M_CXGBE); 995 free(sc->sge.eqmap, M_CXGBE); 996 free(sc->tids.ftid_tab, M_CXGBE); 997 t4_destroy_dma_tag(sc); 998 if (mtx_initialized(&sc->sc_lock)) { 999 sx_xlock(&t4_list_lock); 1000 SLIST_REMOVE(&t4_list, sc, adapter, link); 1001 sx_xunlock(&t4_list_lock); 1002 mtx_destroy(&sc->sc_lock); 1003 } 1004 1005 if (mtx_initialized(&sc->tids.ftid_lock)) 1006 mtx_destroy(&sc->tids.ftid_lock); 1007 if (mtx_initialized(&sc->sfl_lock)) 1008 mtx_destroy(&sc->sfl_lock); 1009 if (mtx_initialized(&sc->ifp_lock)) 1010 mtx_destroy(&sc->ifp_lock); 1011 1012 bzero(sc, sizeof(*sc)); 1013 1014 return (0); 1015 } 1016 1017 static int 1018 cxgbe_probe(device_t dev) 1019 { 1020 char buf[128]; 1021 struct port_info *pi = device_get_softc(dev); 1022 1023 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 1024 device_set_desc_copy(dev, buf); 1025 1026 return (BUS_PROBE_DEFAULT); 1027 } 1028 1029 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 1030 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 1031 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS) 1032 #define T4_CAP_ENABLE (T4_CAP) 1033 1034 static int 1035 cxgbe_attach(device_t dev) 1036 { 1037 struct port_info *pi = device_get_softc(dev); 1038 struct ifnet *ifp; 1039 char *s; 1040 int n, o; 1041 1042 /* Allocate an ifnet and set it up */ 1043 ifp = if_alloc(IFT_ETHER); 1044 if (ifp == NULL) { 1045 device_printf(dev, "Cannot allocate ifnet\n"); 1046 return (ENOMEM); 1047 } 1048 pi->ifp = ifp; 1049 ifp->if_softc = pi; 1050 1051 callout_init(&pi->tick, CALLOUT_MPSAFE); 1052 1053 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1054 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1055 1056 ifp->if_init = cxgbe_init; 1057 ifp->if_ioctl = cxgbe_ioctl; 1058 ifp->if_transmit = cxgbe_transmit; 1059 ifp->if_qflush = cxgbe_qflush; 1060 1061 ifp->if_capabilities = T4_CAP; 1062 #ifdef TCP_OFFLOAD 1063 if (is_offload(pi->adapter)) 1064 ifp->if_capabilities |= IFCAP_TOE; 1065 #endif 1066 ifp->if_capenable = T4_CAP_ENABLE; 1067 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1068 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1069 1070 /* Initialize ifmedia for this port */ 1071 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 1072 cxgbe_media_status); 1073 build_medialist(pi, &pi->media); 1074 1075 pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 1076 EVENTHANDLER_PRI_ANY); 1077 1078 ether_ifattach(ifp, pi->hw_addr); 1079 1080 n = 128; 1081 s = malloc(n, M_CXGBE, M_WAITOK); 1082 o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq); 1083 MPASS(n > o); 1084 #ifdef TCP_OFFLOAD 1085 if (is_offload(pi->adapter)) { 1086 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)", 1087 pi->nofldtxq, pi->nofldrxq); 1088 MPASS(n > o); 1089 } 1090 #endif 1091 #ifdef DEV_NETMAP 1092 o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq, 1093 pi->nnmrxq); 1094 MPASS(n > o); 1095 #endif 1096 device_printf(dev, "%s\n", s); 1097 free(s, M_CXGBE); 1098 1099 #ifdef DEV_NETMAP 1100 /* nm_media handled here to keep implementation private to this file */ 1101 ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change, 1102 cxgbe_media_status); 1103 build_medialist(pi, &pi->nm_media); 1104 create_netmap_ifnet(pi); /* logs errors it something fails */ 1105 #endif 1106 cxgbe_sysctls(pi); 1107 1108 return (0); 1109 } 1110 1111 static int 1112 cxgbe_detach(device_t dev) 1113 { 1114 struct port_info *pi = device_get_softc(dev); 1115 struct adapter *sc = pi->adapter; 1116 struct ifnet *ifp = pi->ifp; 1117 1118 /* Tell if_ioctl and if_init that the port is going away */ 1119 ADAPTER_LOCK(sc); 1120 SET_DOOMED(pi); 1121 wakeup(&sc->flags); 1122 while (IS_BUSY(sc)) 1123 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 1124 SET_BUSY(sc); 1125 #ifdef INVARIANTS 1126 sc->last_op = "t4detach"; 1127 sc->last_op_thr = curthread; 1128 #endif 1129 ADAPTER_UNLOCK(sc); 1130 1131 if (pi->flags & HAS_TRACEQ) { 1132 sc->traceq = -1; /* cloner should not create ifnet */ 1133 t4_tracer_port_detach(sc); 1134 } 1135 1136 if (pi->vlan_c) 1137 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c); 1138 1139 PORT_LOCK(pi); 1140 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1141 callout_stop(&pi->tick); 1142 PORT_UNLOCK(pi); 1143 callout_drain(&pi->tick); 1144 1145 /* Let detach proceed even if these fail. */ 1146 cxgbe_uninit_synchronized(pi); 1147 port_full_uninit(pi); 1148 1149 ifmedia_removeall(&pi->media); 1150 ether_ifdetach(pi->ifp); 1151 if_free(pi->ifp); 1152 1153 #ifdef DEV_NETMAP 1154 /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */ 1155 destroy_netmap_ifnet(pi); 1156 #endif 1157 1158 ADAPTER_LOCK(sc); 1159 CLR_BUSY(sc); 1160 wakeup(&sc->flags); 1161 ADAPTER_UNLOCK(sc); 1162 1163 return (0); 1164 } 1165 1166 static void 1167 cxgbe_init(void *arg) 1168 { 1169 struct port_info *pi = arg; 1170 struct adapter *sc = pi->adapter; 1171 1172 if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0) 1173 return; 1174 cxgbe_init_synchronized(pi); 1175 end_synchronized_op(sc, 0); 1176 } 1177 1178 static int 1179 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1180 { 1181 int rc = 0, mtu, flags; 1182 struct port_info *pi = ifp->if_softc; 1183 struct adapter *sc = pi->adapter; 1184 struct ifreq *ifr = (struct ifreq *)data; 1185 uint32_t mask; 1186 1187 switch (cmd) { 1188 case SIOCSIFMTU: 1189 mtu = ifr->ifr_mtu; 1190 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1191 return (EINVAL); 1192 1193 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu"); 1194 if (rc) 1195 return (rc); 1196 ifp->if_mtu = mtu; 1197 if (pi->flags & PORT_INIT_DONE) { 1198 t4_update_fl_bufsize(ifp); 1199 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1200 rc = update_mac_settings(ifp, XGMAC_MTU); 1201 } 1202 end_synchronized_op(sc, 0); 1203 break; 1204 1205 case SIOCSIFFLAGS: 1206 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4flg"); 1207 if (rc) 1208 return (rc); 1209 1210 if (ifp->if_flags & IFF_UP) { 1211 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1212 flags = pi->if_flags; 1213 if ((ifp->if_flags ^ flags) & 1214 (IFF_PROMISC | IFF_ALLMULTI)) { 1215 rc = update_mac_settings(ifp, 1216 XGMAC_PROMISC | XGMAC_ALLMULTI); 1217 } 1218 } else 1219 rc = cxgbe_init_synchronized(pi); 1220 pi->if_flags = ifp->if_flags; 1221 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1222 rc = cxgbe_uninit_synchronized(pi); 1223 end_synchronized_op(sc, 0); 1224 break; 1225 1226 case SIOCADDMULTI: 1227 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1228 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi"); 1229 if (rc) 1230 return (rc); 1231 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1232 rc = update_mac_settings(ifp, XGMAC_MCADDRS); 1233 end_synchronized_op(sc, LOCK_HELD); 1234 break; 1235 1236 case SIOCSIFCAP: 1237 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap"); 1238 if (rc) 1239 return (rc); 1240 1241 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1242 if (mask & IFCAP_TXCSUM) { 1243 ifp->if_capenable ^= IFCAP_TXCSUM; 1244 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1245 1246 if (IFCAP_TSO4 & ifp->if_capenable && 1247 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1248 ifp->if_capenable &= ~IFCAP_TSO4; 1249 if_printf(ifp, 1250 "tso4 disabled due to -txcsum.\n"); 1251 } 1252 } 1253 if (mask & IFCAP_TXCSUM_IPV6) { 1254 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1255 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1256 1257 if (IFCAP_TSO6 & ifp->if_capenable && 1258 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1259 ifp->if_capenable &= ~IFCAP_TSO6; 1260 if_printf(ifp, 1261 "tso6 disabled due to -txcsum6.\n"); 1262 } 1263 } 1264 if (mask & IFCAP_RXCSUM) 1265 ifp->if_capenable ^= IFCAP_RXCSUM; 1266 if (mask & IFCAP_RXCSUM_IPV6) 1267 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1268 1269 /* 1270 * Note that we leave CSUM_TSO alone (it is always set). The 1271 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1272 * sending a TSO request our way, so it's sufficient to toggle 1273 * IFCAP_TSOx only. 1274 */ 1275 if (mask & IFCAP_TSO4) { 1276 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1277 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1278 if_printf(ifp, "enable txcsum first.\n"); 1279 rc = EAGAIN; 1280 goto fail; 1281 } 1282 ifp->if_capenable ^= IFCAP_TSO4; 1283 } 1284 if (mask & IFCAP_TSO6) { 1285 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1286 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1287 if_printf(ifp, "enable txcsum6 first.\n"); 1288 rc = EAGAIN; 1289 goto fail; 1290 } 1291 ifp->if_capenable ^= IFCAP_TSO6; 1292 } 1293 if (mask & IFCAP_LRO) { 1294 #if defined(INET) || defined(INET6) 1295 int i; 1296 struct sge_rxq *rxq; 1297 1298 ifp->if_capenable ^= IFCAP_LRO; 1299 for_each_rxq(pi, i, rxq) { 1300 if (ifp->if_capenable & IFCAP_LRO) 1301 rxq->iq.flags |= IQ_LRO_ENABLED; 1302 else 1303 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1304 } 1305 #endif 1306 } 1307 #ifdef TCP_OFFLOAD 1308 if (mask & IFCAP_TOE) { 1309 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1310 1311 rc = toe_capability(pi, enable); 1312 if (rc != 0) 1313 goto fail; 1314 1315 ifp->if_capenable ^= mask; 1316 } 1317 #endif 1318 if (mask & IFCAP_VLAN_HWTAGGING) { 1319 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1320 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1321 rc = update_mac_settings(ifp, XGMAC_VLANEX); 1322 } 1323 if (mask & IFCAP_VLAN_MTU) { 1324 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1325 1326 /* Need to find out how to disable auto-mtu-inflation */ 1327 } 1328 if (mask & IFCAP_VLAN_HWTSO) 1329 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1330 if (mask & IFCAP_VLAN_HWCSUM) 1331 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1332 1333 #ifdef VLAN_CAPABILITIES 1334 VLAN_CAPABILITIES(ifp); 1335 #endif 1336 fail: 1337 end_synchronized_op(sc, 0); 1338 break; 1339 1340 case SIOCSIFMEDIA: 1341 case SIOCGIFMEDIA: 1342 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1343 break; 1344 1345 default: 1346 rc = ether_ioctl(ifp, cmd, data); 1347 } 1348 1349 return (rc); 1350 } 1351 1352 static int 1353 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1354 { 1355 struct port_info *pi = ifp->if_softc; 1356 struct adapter *sc = pi->adapter; 1357 struct sge_txq *txq = &sc->sge.txq[pi->first_txq]; 1358 struct buf_ring *br; 1359 int rc; 1360 1361 M_ASSERTPKTHDR(m); 1362 1363 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1364 m_freem(m); 1365 return (ENETDOWN); 1366 } 1367 1368 if (m->m_flags & M_FLOWID) 1369 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) 1370 + pi->rsrv_noflowq); 1371 br = txq->br; 1372 1373 if (TXQ_TRYLOCK(txq) == 0) { 1374 struct sge_eq *eq = &txq->eq; 1375 1376 /* 1377 * It is possible that t4_eth_tx finishes up and releases the 1378 * lock between the TRYLOCK above and the drbr_enqueue here. We 1379 * need to make sure that this mbuf doesn't just sit there in 1380 * the drbr. 1381 */ 1382 1383 rc = drbr_enqueue(ifp, br, m); 1384 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 && 1385 !(eq->flags & EQ_DOOMED)) 1386 callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq); 1387 return (rc); 1388 } 1389 1390 /* 1391 * txq->m is the mbuf that is held up due to a temporary shortage of 1392 * resources and it should be put on the wire first. Then what's in 1393 * drbr and finally the mbuf that was just passed in to us. 1394 * 1395 * Return code should indicate the fate of the mbuf that was passed in 1396 * this time. 1397 */ 1398 1399 TXQ_LOCK_ASSERT_OWNED(txq); 1400 if (drbr_needs_enqueue(ifp, br) || txq->m) { 1401 1402 /* Queued for transmission. */ 1403 1404 rc = drbr_enqueue(ifp, br, m); 1405 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 1406 (void) t4_eth_tx(ifp, txq, m); 1407 TXQ_UNLOCK(txq); 1408 return (rc); 1409 } 1410 1411 /* Direct transmission. */ 1412 rc = t4_eth_tx(ifp, txq, m); 1413 if (rc != 0 && txq->m) 1414 rc = 0; /* held, will be transmitted soon (hopefully) */ 1415 1416 TXQ_UNLOCK(txq); 1417 return (rc); 1418 } 1419 1420 static void 1421 cxgbe_qflush(struct ifnet *ifp) 1422 { 1423 struct port_info *pi = ifp->if_softc; 1424 struct sge_txq *txq; 1425 int i; 1426 struct mbuf *m; 1427 1428 /* queues do not exist if !PORT_INIT_DONE. */ 1429 if (pi->flags & PORT_INIT_DONE) { 1430 for_each_txq(pi, i, txq) { 1431 TXQ_LOCK(txq); 1432 m_freem(txq->m); 1433 txq->m = NULL; 1434 while ((m = buf_ring_dequeue_sc(txq->br)) != NULL) 1435 m_freem(m); 1436 TXQ_UNLOCK(txq); 1437 } 1438 } 1439 if_qflush(ifp); 1440 } 1441 1442 static int 1443 cxgbe_media_change(struct ifnet *ifp) 1444 { 1445 struct port_info *pi = ifp->if_softc; 1446 1447 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1448 1449 return (EOPNOTSUPP); 1450 } 1451 1452 static void 1453 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1454 { 1455 struct port_info *pi = ifp->if_softc; 1456 struct ifmedia *media = NULL; 1457 struct ifmedia_entry *cur; 1458 int speed = pi->link_cfg.speed; 1459 int data = (pi->port_type << 8) | pi->mod_type; 1460 1461 if (ifp == pi->ifp) 1462 media = &pi->media; 1463 #ifdef DEV_NETMAP 1464 else if (ifp == pi->nm_ifp) 1465 media = &pi->nm_media; 1466 #endif 1467 MPASS(media != NULL); 1468 1469 cur = media->ifm_cur; 1470 if (cur->ifm_data != data) { 1471 build_medialist(pi, media); 1472 cur = media->ifm_cur; 1473 } 1474 1475 ifmr->ifm_status = IFM_AVALID; 1476 if (!pi->link_cfg.link_ok) 1477 return; 1478 1479 ifmr->ifm_status |= IFM_ACTIVE; 1480 1481 /* active and current will differ iff current media is autoselect. */ 1482 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1483 return; 1484 1485 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1486 if (speed == SPEED_10000) 1487 ifmr->ifm_active |= IFM_10G_T; 1488 else if (speed == SPEED_1000) 1489 ifmr->ifm_active |= IFM_1000_T; 1490 else if (speed == SPEED_100) 1491 ifmr->ifm_active |= IFM_100_TX; 1492 else if (speed == SPEED_10) 1493 ifmr->ifm_active |= IFM_10_T; 1494 else 1495 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1496 speed)); 1497 } 1498 1499 void 1500 t4_fatal_err(struct adapter *sc) 1501 { 1502 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1503 t4_intr_disable(sc); 1504 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1505 device_get_nameunit(sc->dev)); 1506 } 1507 1508 static int 1509 map_bars_0_and_4(struct adapter *sc) 1510 { 1511 sc->regs_rid = PCIR_BAR(0); 1512 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1513 &sc->regs_rid, RF_ACTIVE); 1514 if (sc->regs_res == NULL) { 1515 device_printf(sc->dev, "cannot map registers.\n"); 1516 return (ENXIO); 1517 } 1518 sc->bt = rman_get_bustag(sc->regs_res); 1519 sc->bh = rman_get_bushandle(sc->regs_res); 1520 sc->mmio_len = rman_get_size(sc->regs_res); 1521 setbit(&sc->doorbells, DOORBELL_KDB); 1522 1523 sc->msix_rid = PCIR_BAR(4); 1524 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1525 &sc->msix_rid, RF_ACTIVE); 1526 if (sc->msix_res == NULL) { 1527 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1528 return (ENXIO); 1529 } 1530 1531 return (0); 1532 } 1533 1534 static int 1535 map_bar_2(struct adapter *sc) 1536 { 1537 1538 /* 1539 * T4: only iWARP driver uses the userspace doorbells. There is no need 1540 * to map it if RDMA is disabled. 1541 */ 1542 if (is_t4(sc) && sc->rdmacaps == 0) 1543 return (0); 1544 1545 sc->udbs_rid = PCIR_BAR(2); 1546 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1547 &sc->udbs_rid, RF_ACTIVE); 1548 if (sc->udbs_res == NULL) { 1549 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1550 return (ENXIO); 1551 } 1552 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1553 1554 if (is_t5(sc)) { 1555 setbit(&sc->doorbells, DOORBELL_UDB); 1556 #if defined(__i386__) || defined(__amd64__) 1557 if (t5_write_combine) { 1558 int rc; 1559 1560 /* 1561 * Enable write combining on BAR2. This is the 1562 * userspace doorbell BAR and is split into 128B 1563 * (UDBS_SEG_SIZE) doorbell regions, each associated 1564 * with an egress queue. The first 64B has the doorbell 1565 * and the second 64B can be used to submit a tx work 1566 * request with an implicit doorbell. 1567 */ 1568 1569 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1570 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1571 if (rc == 0) { 1572 clrbit(&sc->doorbells, DOORBELL_UDB); 1573 setbit(&sc->doorbells, DOORBELL_WCWR); 1574 setbit(&sc->doorbells, DOORBELL_UDBWC); 1575 } else { 1576 device_printf(sc->dev, 1577 "couldn't enable write combining: %d\n", 1578 rc); 1579 } 1580 1581 t4_write_reg(sc, A_SGE_STAT_CFG, 1582 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1583 } 1584 #endif 1585 } 1586 1587 return (0); 1588 } 1589 1590 static const struct memwin t4_memwin[] = { 1591 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1592 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1593 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1594 }; 1595 1596 static const struct memwin t5_memwin[] = { 1597 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1598 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1599 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1600 }; 1601 1602 static void 1603 setup_memwin(struct adapter *sc) 1604 { 1605 const struct memwin *mw; 1606 int i, n; 1607 uint32_t bar0; 1608 1609 if (is_t4(sc)) { 1610 /* 1611 * Read low 32b of bar0 indirectly via the hardware backdoor 1612 * mechanism. Works from within PCI passthrough environments 1613 * too, where rman_get_start() can return a different value. We 1614 * need to program the T4 memory window decoders with the actual 1615 * addresses that will be coming across the PCIe link. 1616 */ 1617 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1618 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1619 1620 mw = &t4_memwin[0]; 1621 n = nitems(t4_memwin); 1622 } else { 1623 /* T5 uses the relative offset inside the PCIe BAR */ 1624 bar0 = 0; 1625 1626 mw = &t5_memwin[0]; 1627 n = nitems(t5_memwin); 1628 } 1629 1630 for (i = 0; i < n; i++, mw++) { 1631 t4_write_reg(sc, 1632 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 1633 (mw->base + bar0) | V_BIR(0) | 1634 V_WINDOW(ilog2(mw->aperture) - 10)); 1635 } 1636 1637 /* flush */ 1638 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1639 } 1640 1641 /* 1642 * Verify that the memory range specified by the addr/len pair is valid and lies 1643 * entirely within a single region (EDCx or MCx). 1644 */ 1645 static int 1646 validate_mem_range(struct adapter *sc, uint32_t addr, int len) 1647 { 1648 uint32_t em, addr_len, maddr, mlen; 1649 1650 /* Memory can only be accessed in naturally aligned 4 byte units */ 1651 if (addr & 3 || len & 3 || len == 0) 1652 return (EINVAL); 1653 1654 /* Enabled memories */ 1655 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1656 if (em & F_EDRAM0_ENABLE) { 1657 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1658 maddr = G_EDRAM0_BASE(addr_len) << 20; 1659 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1660 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1661 addr + len <= maddr + mlen) 1662 return (0); 1663 } 1664 if (em & F_EDRAM1_ENABLE) { 1665 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1666 maddr = G_EDRAM1_BASE(addr_len) << 20; 1667 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1668 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1669 addr + len <= maddr + mlen) 1670 return (0); 1671 } 1672 if (em & F_EXT_MEM_ENABLE) { 1673 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1674 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1675 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1676 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1677 addr + len <= maddr + mlen) 1678 return (0); 1679 } 1680 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 1681 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1682 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1683 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1684 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1685 addr + len <= maddr + mlen) 1686 return (0); 1687 } 1688 1689 return (EFAULT); 1690 } 1691 1692 static int 1693 fwmtype_to_hwmtype(int mtype) 1694 { 1695 1696 switch (mtype) { 1697 case FW_MEMTYPE_EDC0: 1698 return (MEM_EDC0); 1699 case FW_MEMTYPE_EDC1: 1700 return (MEM_EDC1); 1701 case FW_MEMTYPE_EXTMEM: 1702 return (MEM_MC0); 1703 case FW_MEMTYPE_EXTMEM1: 1704 return (MEM_MC1); 1705 default: 1706 panic("%s: cannot translate fw mtype %d.", __func__, mtype); 1707 } 1708 } 1709 1710 /* 1711 * Verify that the memory range specified by the memtype/offset/len pair is 1712 * valid and lies entirely within the memtype specified. The global address of 1713 * the start of the range is returned in addr. 1714 */ 1715 static int 1716 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 1717 uint32_t *addr) 1718 { 1719 uint32_t em, addr_len, maddr, mlen; 1720 1721 /* Memory can only be accessed in naturally aligned 4 byte units */ 1722 if (off & 3 || len & 3 || len == 0) 1723 return (EINVAL); 1724 1725 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1726 switch (fwmtype_to_hwmtype(mtype)) { 1727 case MEM_EDC0: 1728 if (!(em & F_EDRAM0_ENABLE)) 1729 return (EINVAL); 1730 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1731 maddr = G_EDRAM0_BASE(addr_len) << 20; 1732 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1733 break; 1734 case MEM_EDC1: 1735 if (!(em & F_EDRAM1_ENABLE)) 1736 return (EINVAL); 1737 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1738 maddr = G_EDRAM1_BASE(addr_len) << 20; 1739 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1740 break; 1741 case MEM_MC: 1742 if (!(em & F_EXT_MEM_ENABLE)) 1743 return (EINVAL); 1744 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1745 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1746 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1747 break; 1748 case MEM_MC1: 1749 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 1750 return (EINVAL); 1751 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1752 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1753 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1754 break; 1755 default: 1756 return (EINVAL); 1757 } 1758 1759 if (mlen > 0 && off < mlen && off + len <= mlen) { 1760 *addr = maddr + off; /* global address */ 1761 return (0); 1762 } 1763 1764 return (EFAULT); 1765 } 1766 1767 static void 1768 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 1769 { 1770 const struct memwin *mw; 1771 1772 if (is_t4(sc)) { 1773 KASSERT(win >= 0 && win < nitems(t4_memwin), 1774 ("%s: incorrect memwin# (%d)", __func__, win)); 1775 mw = &t4_memwin[win]; 1776 } else { 1777 KASSERT(win >= 0 && win < nitems(t5_memwin), 1778 ("%s: incorrect memwin# (%d)", __func__, win)); 1779 mw = &t5_memwin[win]; 1780 } 1781 1782 if (base != NULL) 1783 *base = mw->base; 1784 if (aperture != NULL) 1785 *aperture = mw->aperture; 1786 } 1787 1788 /* 1789 * Positions the memory window such that it can be used to access the specified 1790 * address in the chip's address space. The return value is the offset of addr 1791 * from the start of the window. 1792 */ 1793 static uint32_t 1794 position_memwin(struct adapter *sc, int n, uint32_t addr) 1795 { 1796 uint32_t start, pf; 1797 uint32_t reg; 1798 1799 KASSERT(n >= 0 && n <= 3, 1800 ("%s: invalid window %d.", __func__, n)); 1801 KASSERT((addr & 3) == 0, 1802 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 1803 1804 if (is_t4(sc)) { 1805 pf = 0; 1806 start = addr & ~0xf; /* start must be 16B aligned */ 1807 } else { 1808 pf = V_PFNUM(sc->pf); 1809 start = addr & ~0x7f; /* start must be 128B aligned */ 1810 } 1811 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 1812 1813 t4_write_reg(sc, reg, start | pf); 1814 t4_read_reg(sc, reg); 1815 1816 return (addr - start); 1817 } 1818 1819 static int 1820 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1821 struct intrs_and_queues *iaq) 1822 { 1823 int rc, itype, navail, nrxq10g, nrxq1g, n; 1824 int nofldrxq10g = 0, nofldrxq1g = 0; 1825 int nnmrxq10g = 0, nnmrxq1g = 0; 1826 1827 bzero(iaq, sizeof(*iaq)); 1828 1829 iaq->ntxq10g = t4_ntxq10g; 1830 iaq->ntxq1g = t4_ntxq1g; 1831 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1832 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1833 iaq->rsrv_noflowq = t4_rsrv_noflowq; 1834 #ifdef TCP_OFFLOAD 1835 if (is_offload(sc)) { 1836 iaq->nofldtxq10g = t4_nofldtxq10g; 1837 iaq->nofldtxq1g = t4_nofldtxq1g; 1838 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1839 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1840 } 1841 #endif 1842 #ifdef DEV_NETMAP 1843 iaq->nnmtxq10g = t4_nnmtxq10g; 1844 iaq->nnmtxq1g = t4_nnmtxq1g; 1845 iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g; 1846 iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g; 1847 #endif 1848 1849 for (itype = INTR_MSIX; itype; itype >>= 1) { 1850 1851 if ((itype & t4_intr_types) == 0) 1852 continue; /* not allowed */ 1853 1854 if (itype == INTR_MSIX) 1855 navail = pci_msix_count(sc->dev); 1856 else if (itype == INTR_MSI) 1857 navail = pci_msi_count(sc->dev); 1858 else 1859 navail = 1; 1860 restart: 1861 if (navail == 0) 1862 continue; 1863 1864 iaq->intr_type = itype; 1865 iaq->intr_flags_10g = 0; 1866 iaq->intr_flags_1g = 0; 1867 1868 /* 1869 * Best option: an interrupt vector for errors, one for the 1870 * firmware event queue, and one for every rxq (NIC, TOE, and 1871 * netmap). 1872 */ 1873 iaq->nirq = T4_EXTRA_INTR; 1874 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g); 1875 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g); 1876 if (iaq->nirq <= navail && 1877 (itype != INTR_MSI || powerof2(iaq->nirq))) { 1878 iaq->intr_flags_10g = INTR_ALL; 1879 iaq->intr_flags_1g = INTR_ALL; 1880 goto allocate; 1881 } 1882 1883 /* 1884 * Second best option: a vector for errors, one for the firmware 1885 * event queue, and vectors for either all the NIC rx queues or 1886 * all the TOE rx queues. The queues that don't get vectors 1887 * will forward their interrupts to those that do. 1888 * 1889 * Note: netmap rx queues cannot be created early and so they 1890 * can't be setup to receive forwarded interrupts for others. 1891 */ 1892 iaq->nirq = T4_EXTRA_INTR; 1893 if (nrxq10g >= nofldrxq10g) { 1894 iaq->intr_flags_10g = INTR_RXQ; 1895 iaq->nirq += n10g * nrxq10g; 1896 #ifdef DEV_NETMAP 1897 iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g); 1898 #endif 1899 } else { 1900 iaq->intr_flags_10g = INTR_OFLD_RXQ; 1901 iaq->nirq += n10g * nofldrxq10g; 1902 #ifdef DEV_NETMAP 1903 iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g); 1904 #endif 1905 } 1906 if (nrxq1g >= nofldrxq1g) { 1907 iaq->intr_flags_1g = INTR_RXQ; 1908 iaq->nirq += n1g * nrxq1g; 1909 #ifdef DEV_NETMAP 1910 iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g); 1911 #endif 1912 } else { 1913 iaq->intr_flags_1g = INTR_OFLD_RXQ; 1914 iaq->nirq += n1g * nofldrxq1g; 1915 #ifdef DEV_NETMAP 1916 iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g); 1917 #endif 1918 } 1919 if (iaq->nirq <= navail && 1920 (itype != INTR_MSI || powerof2(iaq->nirq))) 1921 goto allocate; 1922 1923 /* 1924 * Next best option: an interrupt vector for errors, one for the 1925 * firmware event queue, and at least one per port. At this 1926 * point we know we'll have to downsize nrxq and/or nofldrxq 1927 * and/or nnmrxq to fit what's available to us. 1928 */ 1929 iaq->nirq = T4_EXTRA_INTR; 1930 iaq->nirq += n10g + n1g; 1931 if (iaq->nirq <= navail) { 1932 int leftover = navail - iaq->nirq; 1933 1934 if (n10g > 0) { 1935 int target = max(nrxq10g, nofldrxq10g); 1936 1937 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ? 1938 INTR_RXQ : INTR_OFLD_RXQ; 1939 1940 n = 1; 1941 while (n < target && leftover >= n10g) { 1942 leftover -= n10g; 1943 iaq->nirq += n10g; 1944 n++; 1945 } 1946 iaq->nrxq10g = min(n, nrxq10g); 1947 #ifdef TCP_OFFLOAD 1948 iaq->nofldrxq10g = min(n, nofldrxq10g); 1949 #endif 1950 #ifdef DEV_NETMAP 1951 iaq->nnmrxq10g = min(n, nnmrxq10g); 1952 #endif 1953 } 1954 1955 if (n1g > 0) { 1956 int target = max(nrxq1g, nofldrxq1g); 1957 1958 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ? 1959 INTR_RXQ : INTR_OFLD_RXQ; 1960 1961 n = 1; 1962 while (n < target && leftover >= n1g) { 1963 leftover -= n1g; 1964 iaq->nirq += n1g; 1965 n++; 1966 } 1967 iaq->nrxq1g = min(n, nrxq1g); 1968 #ifdef TCP_OFFLOAD 1969 iaq->nofldrxq1g = min(n, nofldrxq1g); 1970 #endif 1971 #ifdef DEV_NETMAP 1972 iaq->nnmrxq1g = min(n, nnmrxq1g); 1973 #endif 1974 } 1975 1976 if (itype != INTR_MSI || powerof2(iaq->nirq)) 1977 goto allocate; 1978 } 1979 1980 /* 1981 * Least desirable option: one interrupt vector for everything. 1982 */ 1983 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 1984 iaq->intr_flags_10g = iaq->intr_flags_1g = 0; 1985 #ifdef TCP_OFFLOAD 1986 if (is_offload(sc)) 1987 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 1988 #endif 1989 #ifdef DEV_NETMAP 1990 iaq->nnmrxq10g = iaq->nnmrxq1g = 1; 1991 #endif 1992 1993 allocate: 1994 navail = iaq->nirq; 1995 rc = 0; 1996 if (itype == INTR_MSIX) 1997 rc = pci_alloc_msix(sc->dev, &navail); 1998 else if (itype == INTR_MSI) 1999 rc = pci_alloc_msi(sc->dev, &navail); 2000 2001 if (rc == 0) { 2002 if (navail == iaq->nirq) 2003 return (0); 2004 2005 /* 2006 * Didn't get the number requested. Use whatever number 2007 * the kernel is willing to allocate (it's in navail). 2008 */ 2009 device_printf(sc->dev, "fewer vectors than requested, " 2010 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 2011 itype, iaq->nirq, navail); 2012 pci_release_msi(sc->dev); 2013 goto restart; 2014 } 2015 2016 device_printf(sc->dev, 2017 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 2018 itype, rc, iaq->nirq, navail); 2019 } 2020 2021 device_printf(sc->dev, 2022 "failed to find a usable interrupt type. " 2023 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 2024 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 2025 2026 return (ENXIO); 2027 } 2028 2029 #define FW_VERSION(chip) ( \ 2030 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ 2031 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ 2032 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ 2033 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) 2034 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) 2035 2036 struct fw_info { 2037 uint8_t chip; 2038 char *kld_name; 2039 char *fw_mod_name; 2040 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 2041 } fw_info[] = { 2042 { 2043 .chip = CHELSIO_T4, 2044 .kld_name = "t4fw_cfg", 2045 .fw_mod_name = "t4fw", 2046 .fw_hdr = { 2047 .chip = FW_HDR_CHIP_T4, 2048 .fw_ver = htobe32_const(FW_VERSION(T4)), 2049 .intfver_nic = FW_INTFVER(T4, NIC), 2050 .intfver_vnic = FW_INTFVER(T4, VNIC), 2051 .intfver_ofld = FW_INTFVER(T4, OFLD), 2052 .intfver_ri = FW_INTFVER(T4, RI), 2053 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 2054 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 2055 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 2056 .intfver_fcoe = FW_INTFVER(T4, FCOE), 2057 }, 2058 }, { 2059 .chip = CHELSIO_T5, 2060 .kld_name = "t5fw_cfg", 2061 .fw_mod_name = "t5fw", 2062 .fw_hdr = { 2063 .chip = FW_HDR_CHIP_T5, 2064 .fw_ver = htobe32_const(FW_VERSION(T5)), 2065 .intfver_nic = FW_INTFVER(T5, NIC), 2066 .intfver_vnic = FW_INTFVER(T5, VNIC), 2067 .intfver_ofld = FW_INTFVER(T5, OFLD), 2068 .intfver_ri = FW_INTFVER(T5, RI), 2069 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 2070 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 2071 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 2072 .intfver_fcoe = FW_INTFVER(T5, FCOE), 2073 }, 2074 } 2075 }; 2076 2077 static struct fw_info * 2078 find_fw_info(int chip) 2079 { 2080 int i; 2081 2082 for (i = 0; i < nitems(fw_info); i++) { 2083 if (fw_info[i].chip == chip) 2084 return (&fw_info[i]); 2085 } 2086 return (NULL); 2087 } 2088 2089 /* 2090 * Is the given firmware API compatible with the one the driver was compiled 2091 * with? 2092 */ 2093 static int 2094 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 2095 { 2096 2097 /* short circuit if it's the exact same firmware version */ 2098 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 2099 return (1); 2100 2101 /* 2102 * XXX: Is this too conservative? Perhaps I should limit this to the 2103 * features that are supported in the driver. 2104 */ 2105 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 2106 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 2107 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 2108 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 2109 return (1); 2110 #undef SAME_INTF 2111 2112 return (0); 2113 } 2114 2115 /* 2116 * The firmware in the KLD is usable, but should it be installed? This routine 2117 * explains itself in detail if it indicates the KLD firmware should be 2118 * installed. 2119 */ 2120 static int 2121 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 2122 { 2123 const char *reason; 2124 2125 if (!card_fw_usable) { 2126 reason = "incompatible or unusable"; 2127 goto install; 2128 } 2129 2130 if (k > c) { 2131 reason = "older than the version bundled with this driver"; 2132 goto install; 2133 } 2134 2135 if (t4_fw_install == 2 && k != c) { 2136 reason = "different than the version bundled with this driver"; 2137 goto install; 2138 } 2139 2140 return (0); 2141 2142 install: 2143 if (t4_fw_install == 0) { 2144 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2145 "but the driver is prohibited from installing a different " 2146 "firmware on the card.\n", 2147 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2148 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 2149 2150 return (0); 2151 } 2152 2153 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2154 "installing firmware %u.%u.%u.%u on card.\n", 2155 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2156 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 2157 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2158 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2159 2160 return (1); 2161 } 2162 /* 2163 * Establish contact with the firmware and determine if we are the master driver 2164 * or not, and whether we are responsible for chip initialization. 2165 */ 2166 static int 2167 prep_firmware(struct adapter *sc) 2168 { 2169 const struct firmware *fw = NULL, *default_cfg; 2170 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 2171 enum dev_state state; 2172 struct fw_info *fw_info; 2173 struct fw_hdr *card_fw; /* fw on the card */ 2174 const struct fw_hdr *kld_fw; /* fw in the KLD */ 2175 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 2176 against */ 2177 2178 /* Contact firmware. */ 2179 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 2180 if (rc < 0 || state == DEV_STATE_ERR) { 2181 rc = -rc; 2182 device_printf(sc->dev, 2183 "failed to connect to the firmware: %d, %d.\n", rc, state); 2184 return (rc); 2185 } 2186 pf = rc; 2187 if (pf == sc->mbox) 2188 sc->flags |= MASTER_PF; 2189 else if (state == DEV_STATE_UNINIT) { 2190 /* 2191 * We didn't get to be the master so we definitely won't be 2192 * configuring the chip. It's a bug if someone else hasn't 2193 * configured it already. 2194 */ 2195 device_printf(sc->dev, "couldn't be master(%d), " 2196 "device not already initialized either(%d).\n", rc, state); 2197 return (EDOOFUS); 2198 } 2199 2200 /* This is the firmware whose headers the driver was compiled against */ 2201 fw_info = find_fw_info(chip_id(sc)); 2202 if (fw_info == NULL) { 2203 device_printf(sc->dev, 2204 "unable to look up firmware information for chip %d.\n", 2205 chip_id(sc)); 2206 return (EINVAL); 2207 } 2208 drv_fw = &fw_info->fw_hdr; 2209 2210 /* 2211 * The firmware KLD contains many modules. The KLD name is also the 2212 * name of the module that contains the default config file. 2213 */ 2214 default_cfg = firmware_get(fw_info->kld_name); 2215 2216 /* Read the header of the firmware on the card */ 2217 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 2218 rc = -t4_read_flash(sc, FLASH_FW_START, 2219 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 2220 if (rc == 0) 2221 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 2222 else { 2223 device_printf(sc->dev, 2224 "Unable to read card's firmware header: %d\n", rc); 2225 card_fw_usable = 0; 2226 } 2227 2228 /* This is the firmware in the KLD */ 2229 fw = firmware_get(fw_info->fw_mod_name); 2230 if (fw != NULL) { 2231 kld_fw = (const void *)fw->data; 2232 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 2233 } else { 2234 kld_fw = NULL; 2235 kld_fw_usable = 0; 2236 } 2237 2238 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2239 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) { 2240 /* 2241 * Common case: the firmware on the card is an exact match and 2242 * the KLD is an exact match too, or the KLD is 2243 * absent/incompatible. Note that t4_fw_install = 2 is ignored 2244 * here -- use cxgbetool loadfw if you want to reinstall the 2245 * same firmware as the one on the card. 2246 */ 2247 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2248 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2249 be32toh(card_fw->fw_ver))) { 2250 2251 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 2252 if (rc != 0) { 2253 device_printf(sc->dev, 2254 "failed to install firmware: %d\n", rc); 2255 goto done; 2256 } 2257 2258 /* Installed successfully, update the cached header too. */ 2259 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2260 card_fw_usable = 1; 2261 need_fw_reset = 0; /* already reset as part of load_fw */ 2262 } 2263 2264 if (!card_fw_usable) { 2265 uint32_t d, c, k; 2266 2267 d = ntohl(drv_fw->fw_ver); 2268 c = ntohl(card_fw->fw_ver); 2269 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2270 2271 device_printf(sc->dev, "Cannot find a usable firmware: " 2272 "fw_install %d, chip state %d, " 2273 "driver compiled with %d.%d.%d.%d, " 2274 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2275 t4_fw_install, state, 2276 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2277 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2278 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2279 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2280 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2281 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2282 rc = EINVAL; 2283 goto done; 2284 } 2285 2286 /* We're using whatever's on the card and it's known to be good. */ 2287 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2288 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2289 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2290 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2291 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2292 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2293 t4_get_tp_version(sc, &sc->params.tp_vers); 2294 2295 /* Reset device */ 2296 if (need_fw_reset && 2297 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2298 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2299 if (rc != ETIMEDOUT && rc != EIO) 2300 t4_fw_bye(sc, sc->mbox); 2301 goto done; 2302 } 2303 sc->flags |= FW_OK; 2304 2305 rc = get_params__pre_init(sc); 2306 if (rc != 0) 2307 goto done; /* error message displayed already */ 2308 2309 /* Partition adapter resources as specified in the config file. */ 2310 if (state == DEV_STATE_UNINIT) { 2311 2312 KASSERT(sc->flags & MASTER_PF, 2313 ("%s: trying to change chip settings when not master.", 2314 __func__)); 2315 2316 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2317 if (rc != 0) 2318 goto done; /* error message displayed already */ 2319 2320 t4_tweak_chip_settings(sc); 2321 2322 /* get basic stuff going */ 2323 rc = -t4_fw_initialize(sc, sc->mbox); 2324 if (rc != 0) { 2325 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2326 goto done; 2327 } 2328 } else { 2329 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2330 sc->cfcsum = 0; 2331 } 2332 2333 done: 2334 free(card_fw, M_CXGBE); 2335 if (fw != NULL) 2336 firmware_put(fw, FIRMWARE_UNLOAD); 2337 if (default_cfg != NULL) 2338 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2339 2340 return (rc); 2341 } 2342 2343 #define FW_PARAM_DEV(param) \ 2344 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2345 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2346 #define FW_PARAM_PFVF(param) \ 2347 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2348 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2349 2350 /* 2351 * Partition chip resources for use between various PFs, VFs, etc. 2352 */ 2353 static int 2354 partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2355 const char *name_prefix) 2356 { 2357 const struct firmware *cfg = NULL; 2358 int rc = 0; 2359 struct fw_caps_config_cmd caps; 2360 uint32_t mtype, moff, finicsum, cfcsum; 2361 2362 /* 2363 * Figure out what configuration file to use. Pick the default config 2364 * file for the card if the user hasn't specified one explicitly. 2365 */ 2366 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2367 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2368 /* Card specific overrides go here. */ 2369 if (pci_get_device(sc->dev) == 0x440a) 2370 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2371 if (is_fpga(sc)) 2372 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2373 } 2374 2375 /* 2376 * We need to load another module if the profile is anything except 2377 * "default" or "flash". 2378 */ 2379 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2380 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2381 char s[32]; 2382 2383 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2384 cfg = firmware_get(s); 2385 if (cfg == NULL) { 2386 if (default_cfg != NULL) { 2387 device_printf(sc->dev, 2388 "unable to load module \"%s\" for " 2389 "configuration profile \"%s\", will use " 2390 "the default config file instead.\n", 2391 s, sc->cfg_file); 2392 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2393 "%s", DEFAULT_CF); 2394 } else { 2395 device_printf(sc->dev, 2396 "unable to load module \"%s\" for " 2397 "configuration profile \"%s\", will use " 2398 "the config file on the card's flash " 2399 "instead.\n", s, sc->cfg_file); 2400 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2401 "%s", FLASH_CF); 2402 } 2403 } 2404 } 2405 2406 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2407 default_cfg == NULL) { 2408 device_printf(sc->dev, 2409 "default config file not available, will use the config " 2410 "file on the card's flash instead.\n"); 2411 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2412 } 2413 2414 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2415 u_int cflen, i, n; 2416 const uint32_t *cfdata; 2417 uint32_t param, val, addr, off, mw_base, mw_aperture; 2418 2419 KASSERT(cfg != NULL || default_cfg != NULL, 2420 ("%s: no config to upload", __func__)); 2421 2422 /* 2423 * Ask the firmware where it wants us to upload the config file. 2424 */ 2425 param = FW_PARAM_DEV(CF); 2426 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2427 if (rc != 0) { 2428 /* No support for config file? Shouldn't happen. */ 2429 device_printf(sc->dev, 2430 "failed to query config file location: %d.\n", rc); 2431 goto done; 2432 } 2433 mtype = G_FW_PARAMS_PARAM_Y(val); 2434 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2435 2436 /* 2437 * XXX: sheer laziness. We deliberately added 4 bytes of 2438 * useless stuffing/comments at the end of the config file so 2439 * it's ok to simply throw away the last remaining bytes when 2440 * the config file is not an exact multiple of 4. This also 2441 * helps with the validate_mt_off_len check. 2442 */ 2443 if (cfg != NULL) { 2444 cflen = cfg->datasize & ~3; 2445 cfdata = cfg->data; 2446 } else { 2447 cflen = default_cfg->datasize & ~3; 2448 cfdata = default_cfg->data; 2449 } 2450 2451 if (cflen > FLASH_CFG_MAX_SIZE) { 2452 device_printf(sc->dev, 2453 "config file too long (%d, max allowed is %d). " 2454 "Will try to use the config on the card, if any.\n", 2455 cflen, FLASH_CFG_MAX_SIZE); 2456 goto use_config_on_flash; 2457 } 2458 2459 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2460 if (rc != 0) { 2461 device_printf(sc->dev, 2462 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2463 "Will try to use the config on the card, if any.\n", 2464 __func__, mtype, moff, cflen, rc); 2465 goto use_config_on_flash; 2466 } 2467 2468 memwin_info(sc, 2, &mw_base, &mw_aperture); 2469 while (cflen) { 2470 off = position_memwin(sc, 2, addr); 2471 n = min(cflen, mw_aperture - off); 2472 for (i = 0; i < n; i += 4) 2473 t4_write_reg(sc, mw_base + off + i, *cfdata++); 2474 cflen -= n; 2475 addr += n; 2476 } 2477 } else { 2478 use_config_on_flash: 2479 mtype = FW_MEMTYPE_FLASH; 2480 moff = t4_flash_cfg_addr(sc); 2481 } 2482 2483 bzero(&caps, sizeof(caps)); 2484 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2485 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2486 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2487 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2488 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2489 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2490 if (rc != 0) { 2491 device_printf(sc->dev, 2492 "failed to pre-process config file: %d " 2493 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2494 goto done; 2495 } 2496 2497 finicsum = be32toh(caps.finicsum); 2498 cfcsum = be32toh(caps.cfcsum); 2499 if (finicsum != cfcsum) { 2500 device_printf(sc->dev, 2501 "WARNING: config file checksum mismatch: %08x %08x\n", 2502 finicsum, cfcsum); 2503 } 2504 sc->cfcsum = cfcsum; 2505 2506 #define LIMIT_CAPS(x) do { \ 2507 caps.x &= htobe16(t4_##x##_allowed); \ 2508 } while (0) 2509 2510 /* 2511 * Let the firmware know what features will (not) be used so it can tune 2512 * things accordingly. 2513 */ 2514 LIMIT_CAPS(linkcaps); 2515 LIMIT_CAPS(niccaps); 2516 LIMIT_CAPS(toecaps); 2517 LIMIT_CAPS(rdmacaps); 2518 LIMIT_CAPS(iscsicaps); 2519 LIMIT_CAPS(fcoecaps); 2520 #undef LIMIT_CAPS 2521 2522 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2523 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2524 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2525 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2526 if (rc != 0) { 2527 device_printf(sc->dev, 2528 "failed to process config file: %d.\n", rc); 2529 } 2530 done: 2531 if (cfg != NULL) 2532 firmware_put(cfg, FIRMWARE_UNLOAD); 2533 return (rc); 2534 } 2535 2536 /* 2537 * Retrieve parameters that are needed (or nice to have) very early. 2538 */ 2539 static int 2540 get_params__pre_init(struct adapter *sc) 2541 { 2542 int rc; 2543 uint32_t param[2], val[2]; 2544 struct fw_devlog_cmd cmd; 2545 struct devlog_params *dlog = &sc->params.devlog; 2546 2547 param[0] = FW_PARAM_DEV(PORTVEC); 2548 param[1] = FW_PARAM_DEV(CCLK); 2549 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2550 if (rc != 0) { 2551 device_printf(sc->dev, 2552 "failed to query parameters (pre_init): %d.\n", rc); 2553 return (rc); 2554 } 2555 2556 sc->params.portvec = val[0]; 2557 sc->params.nports = bitcount32(val[0]); 2558 sc->params.vpd.cclk = val[1]; 2559 2560 /* Read device log parameters. */ 2561 bzero(&cmd, sizeof(cmd)); 2562 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2563 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2564 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2565 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2566 if (rc != 0) { 2567 device_printf(sc->dev, 2568 "failed to get devlog parameters: %d.\n", rc); 2569 bzero(dlog, sizeof (*dlog)); 2570 rc = 0; /* devlog isn't critical for device operation */ 2571 } else { 2572 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2573 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2574 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2575 dlog->size = be32toh(cmd.memsize_devlog); 2576 } 2577 2578 return (rc); 2579 } 2580 2581 /* 2582 * Retrieve various parameters that are of interest to the driver. The device 2583 * has been initialized by the firmware at this point. 2584 */ 2585 static int 2586 get_params__post_init(struct adapter *sc) 2587 { 2588 int rc; 2589 uint32_t param[7], val[7]; 2590 struct fw_caps_config_cmd caps; 2591 2592 param[0] = FW_PARAM_PFVF(IQFLINT_START); 2593 param[1] = FW_PARAM_PFVF(EQ_START); 2594 param[2] = FW_PARAM_PFVF(FILTER_START); 2595 param[3] = FW_PARAM_PFVF(FILTER_END); 2596 param[4] = FW_PARAM_PFVF(L2T_START); 2597 param[5] = FW_PARAM_PFVF(L2T_END); 2598 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2599 if (rc != 0) { 2600 device_printf(sc->dev, 2601 "failed to query parameters (post_init): %d.\n", rc); 2602 return (rc); 2603 } 2604 2605 sc->sge.iq_start = val[0]; 2606 sc->sge.eq_start = val[1]; 2607 sc->tids.ftid_base = val[2]; 2608 sc->tids.nftids = val[3] - val[2] + 1; 2609 sc->params.ftid_min = val[2]; 2610 sc->params.ftid_max = val[3]; 2611 sc->vres.l2t.start = val[4]; 2612 sc->vres.l2t.size = val[5] - val[4] + 1; 2613 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2614 ("%s: L2 table size (%u) larger than expected (%u)", 2615 __func__, sc->vres.l2t.size, L2T_SIZE)); 2616 2617 /* get capabilites */ 2618 bzero(&caps, sizeof(caps)); 2619 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2620 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2621 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2622 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2623 if (rc != 0) { 2624 device_printf(sc->dev, 2625 "failed to get card capabilities: %d.\n", rc); 2626 return (rc); 2627 } 2628 2629 #define READ_CAPS(x) do { \ 2630 sc->x = htobe16(caps.x); \ 2631 } while (0) 2632 READ_CAPS(linkcaps); 2633 READ_CAPS(niccaps); 2634 READ_CAPS(toecaps); 2635 READ_CAPS(rdmacaps); 2636 READ_CAPS(iscsicaps); 2637 READ_CAPS(fcoecaps); 2638 2639 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { 2640 param[0] = FW_PARAM_PFVF(ETHOFLD_START); 2641 param[1] = FW_PARAM_PFVF(ETHOFLD_END); 2642 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2643 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); 2644 if (rc != 0) { 2645 device_printf(sc->dev, 2646 "failed to query NIC parameters: %d.\n", rc); 2647 return (rc); 2648 } 2649 sc->tids.etid_base = val[0]; 2650 sc->params.etid_min = val[0]; 2651 sc->tids.netids = val[1] - val[0] + 1; 2652 sc->params.netids = sc->tids.netids; 2653 sc->params.eo_wr_cred = val[2]; 2654 sc->params.ethoffload = 1; 2655 } 2656 2657 if (sc->toecaps) { 2658 /* query offload-related parameters */ 2659 param[0] = FW_PARAM_DEV(NTID); 2660 param[1] = FW_PARAM_PFVF(SERVER_START); 2661 param[2] = FW_PARAM_PFVF(SERVER_END); 2662 param[3] = FW_PARAM_PFVF(TDDP_START); 2663 param[4] = FW_PARAM_PFVF(TDDP_END); 2664 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2665 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2666 if (rc != 0) { 2667 device_printf(sc->dev, 2668 "failed to query TOE parameters: %d.\n", rc); 2669 return (rc); 2670 } 2671 sc->tids.ntids = val[0]; 2672 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 2673 sc->tids.stid_base = val[1]; 2674 sc->tids.nstids = val[2] - val[1] + 1; 2675 sc->vres.ddp.start = val[3]; 2676 sc->vres.ddp.size = val[4] - val[3] + 1; 2677 sc->params.ofldq_wr_cred = val[5]; 2678 sc->params.offload = 1; 2679 } 2680 if (sc->rdmacaps) { 2681 param[0] = FW_PARAM_PFVF(STAG_START); 2682 param[1] = FW_PARAM_PFVF(STAG_END); 2683 param[2] = FW_PARAM_PFVF(RQ_START); 2684 param[3] = FW_PARAM_PFVF(RQ_END); 2685 param[4] = FW_PARAM_PFVF(PBL_START); 2686 param[5] = FW_PARAM_PFVF(PBL_END); 2687 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2688 if (rc != 0) { 2689 device_printf(sc->dev, 2690 "failed to query RDMA parameters(1): %d.\n", rc); 2691 return (rc); 2692 } 2693 sc->vres.stag.start = val[0]; 2694 sc->vres.stag.size = val[1] - val[0] + 1; 2695 sc->vres.rq.start = val[2]; 2696 sc->vres.rq.size = val[3] - val[2] + 1; 2697 sc->vres.pbl.start = val[4]; 2698 sc->vres.pbl.size = val[5] - val[4] + 1; 2699 2700 param[0] = FW_PARAM_PFVF(SQRQ_START); 2701 param[1] = FW_PARAM_PFVF(SQRQ_END); 2702 param[2] = FW_PARAM_PFVF(CQ_START); 2703 param[3] = FW_PARAM_PFVF(CQ_END); 2704 param[4] = FW_PARAM_PFVF(OCQ_START); 2705 param[5] = FW_PARAM_PFVF(OCQ_END); 2706 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2707 if (rc != 0) { 2708 device_printf(sc->dev, 2709 "failed to query RDMA parameters(2): %d.\n", rc); 2710 return (rc); 2711 } 2712 sc->vres.qp.start = val[0]; 2713 sc->vres.qp.size = val[1] - val[0] + 1; 2714 sc->vres.cq.start = val[2]; 2715 sc->vres.cq.size = val[3] - val[2] + 1; 2716 sc->vres.ocq.start = val[4]; 2717 sc->vres.ocq.size = val[5] - val[4] + 1; 2718 } 2719 if (sc->iscsicaps) { 2720 param[0] = FW_PARAM_PFVF(ISCSI_START); 2721 param[1] = FW_PARAM_PFVF(ISCSI_END); 2722 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2723 if (rc != 0) { 2724 device_printf(sc->dev, 2725 "failed to query iSCSI parameters: %d.\n", rc); 2726 return (rc); 2727 } 2728 sc->vres.iscsi.start = val[0]; 2729 sc->vres.iscsi.size = val[1] - val[0] + 1; 2730 } 2731 2732 /* 2733 * We've got the params we wanted to query via the firmware. Now grab 2734 * some others directly from the chip. 2735 */ 2736 rc = t4_read_chip_settings(sc); 2737 2738 return (rc); 2739 } 2740 2741 static int 2742 set_params__post_init(struct adapter *sc) 2743 { 2744 uint32_t param, val; 2745 2746 /* ask for encapsulated CPLs */ 2747 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 2748 val = 1; 2749 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2750 2751 return (0); 2752 } 2753 2754 #undef FW_PARAM_PFVF 2755 #undef FW_PARAM_DEV 2756 2757 static void 2758 t4_set_desc(struct adapter *sc) 2759 { 2760 char buf[128]; 2761 struct adapter_params *p = &sc->params; 2762 2763 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, " 2764 "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "", 2765 chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec); 2766 2767 device_set_desc_copy(sc->dev, buf); 2768 } 2769 2770 static void 2771 build_medialist(struct port_info *pi, struct ifmedia *media) 2772 { 2773 int data, m; 2774 2775 PORT_LOCK(pi); 2776 2777 ifmedia_removeall(media); 2778 2779 m = IFM_ETHER | IFM_FDX; 2780 data = (pi->port_type << 8) | pi->mod_type; 2781 2782 switch(pi->port_type) { 2783 case FW_PORT_TYPE_BT_XFI: 2784 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2785 break; 2786 2787 case FW_PORT_TYPE_BT_XAUI: 2788 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2789 /* fall through */ 2790 2791 case FW_PORT_TYPE_BT_SGMII: 2792 ifmedia_add(media, m | IFM_1000_T, data, NULL); 2793 ifmedia_add(media, m | IFM_100_TX, data, NULL); 2794 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL); 2795 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2796 break; 2797 2798 case FW_PORT_TYPE_CX4: 2799 ifmedia_add(media, m | IFM_10G_CX4, data, NULL); 2800 ifmedia_set(media, m | IFM_10G_CX4); 2801 break; 2802 2803 case FW_PORT_TYPE_QSFP_10G: 2804 case FW_PORT_TYPE_SFP: 2805 case FW_PORT_TYPE_FIBER_XFI: 2806 case FW_PORT_TYPE_FIBER_XAUI: 2807 switch (pi->mod_type) { 2808 2809 case FW_PORT_MOD_TYPE_LR: 2810 ifmedia_add(media, m | IFM_10G_LR, data, NULL); 2811 ifmedia_set(media, m | IFM_10G_LR); 2812 break; 2813 2814 case FW_PORT_MOD_TYPE_SR: 2815 ifmedia_add(media, m | IFM_10G_SR, data, NULL); 2816 ifmedia_set(media, m | IFM_10G_SR); 2817 break; 2818 2819 case FW_PORT_MOD_TYPE_LRM: 2820 ifmedia_add(media, m | IFM_10G_LRM, data, NULL); 2821 ifmedia_set(media, m | IFM_10G_LRM); 2822 break; 2823 2824 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2825 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2826 ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL); 2827 ifmedia_set(media, m | IFM_10G_TWINAX); 2828 break; 2829 2830 case FW_PORT_MOD_TYPE_NONE: 2831 m &= ~IFM_FDX; 2832 ifmedia_add(media, m | IFM_NONE, data, NULL); 2833 ifmedia_set(media, m | IFM_NONE); 2834 break; 2835 2836 case FW_PORT_MOD_TYPE_NA: 2837 case FW_PORT_MOD_TYPE_ER: 2838 default: 2839 device_printf(pi->dev, 2840 "unknown port_type (%d), mod_type (%d)\n", 2841 pi->port_type, pi->mod_type); 2842 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2843 ifmedia_set(media, m | IFM_UNKNOWN); 2844 break; 2845 } 2846 break; 2847 2848 case FW_PORT_TYPE_QSFP: 2849 switch (pi->mod_type) { 2850 2851 case FW_PORT_MOD_TYPE_LR: 2852 ifmedia_add(media, m | IFM_40G_LR4, data, NULL); 2853 ifmedia_set(media, m | IFM_40G_LR4); 2854 break; 2855 2856 case FW_PORT_MOD_TYPE_SR: 2857 ifmedia_add(media, m | IFM_40G_SR4, data, NULL); 2858 ifmedia_set(media, m | IFM_40G_SR4); 2859 break; 2860 2861 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2862 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2863 ifmedia_add(media, m | IFM_40G_CR4, data, NULL); 2864 ifmedia_set(media, m | IFM_40G_CR4); 2865 break; 2866 2867 case FW_PORT_MOD_TYPE_NONE: 2868 m &= ~IFM_FDX; 2869 ifmedia_add(media, m | IFM_NONE, data, NULL); 2870 ifmedia_set(media, m | IFM_NONE); 2871 break; 2872 2873 default: 2874 device_printf(pi->dev, 2875 "unknown port_type (%d), mod_type (%d)\n", 2876 pi->port_type, pi->mod_type); 2877 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2878 ifmedia_set(media, m | IFM_UNKNOWN); 2879 break; 2880 } 2881 break; 2882 2883 default: 2884 device_printf(pi->dev, 2885 "unknown port_type (%d), mod_type (%d)\n", pi->port_type, 2886 pi->mod_type); 2887 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2888 ifmedia_set(media, m | IFM_UNKNOWN); 2889 break; 2890 } 2891 2892 PORT_UNLOCK(pi); 2893 } 2894 2895 #define FW_MAC_EXACT_CHUNK 7 2896 2897 /* 2898 * Program the port's XGMAC based on parameters in ifnet. The caller also 2899 * indicates which parameters should be programmed (the rest are left alone). 2900 */ 2901 int 2902 update_mac_settings(struct ifnet *ifp, int flags) 2903 { 2904 int rc = 0; 2905 struct port_info *pi = ifp->if_softc; 2906 struct adapter *sc = pi->adapter; 2907 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 2908 uint16_t viid = 0xffff; 2909 int16_t *xact_addr_filt = NULL; 2910 2911 ASSERT_SYNCHRONIZED_OP(sc); 2912 KASSERT(flags, ("%s: not told what to update.", __func__)); 2913 2914 if (ifp == pi->ifp) { 2915 viid = pi->viid; 2916 xact_addr_filt = &pi->xact_addr_filt; 2917 } 2918 #ifdef DEV_NETMAP 2919 else if (ifp == pi->nm_ifp) { 2920 viid = pi->nm_viid; 2921 xact_addr_filt = &pi->nm_xact_addr_filt; 2922 } 2923 #endif 2924 if (flags & XGMAC_MTU) 2925 mtu = ifp->if_mtu; 2926 2927 if (flags & XGMAC_PROMISC) 2928 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 2929 2930 if (flags & XGMAC_ALLMULTI) 2931 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 2932 2933 if (flags & XGMAC_VLANEX) 2934 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 2935 2936 if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) { 2937 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti, 2938 1, vlanex, false); 2939 if (rc) { 2940 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, 2941 rc); 2942 return (rc); 2943 } 2944 } 2945 2946 if (flags & XGMAC_UCADDR) { 2947 uint8_t ucaddr[ETHER_ADDR_LEN]; 2948 2949 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 2950 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr, 2951 true, true); 2952 if (rc < 0) { 2953 rc = -rc; 2954 if_printf(ifp, "change_mac failed: %d\n", rc); 2955 return (rc); 2956 } else { 2957 *xact_addr_filt = rc; 2958 rc = 0; 2959 } 2960 } 2961 2962 if (flags & XGMAC_MCADDRS) { 2963 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 2964 int del = 1; 2965 uint64_t hash = 0; 2966 struct ifmultiaddr *ifma; 2967 int i = 0, j; 2968 2969 if_maddr_rlock(ifp); 2970 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 2971 if (ifma->ifma_addr->sa_family != AF_LINK) 2972 continue; 2973 mcaddr[i++] = 2974 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 2975 2976 if (i == FW_MAC_EXACT_CHUNK) { 2977 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, 2978 i, mcaddr, NULL, &hash, 0); 2979 if (rc < 0) { 2980 rc = -rc; 2981 for (j = 0; j < i; j++) { 2982 if_printf(ifp, 2983 "failed to add mc address" 2984 " %02x:%02x:%02x:" 2985 "%02x:%02x:%02x rc=%d\n", 2986 mcaddr[j][0], mcaddr[j][1], 2987 mcaddr[j][2], mcaddr[j][3], 2988 mcaddr[j][4], mcaddr[j][5], 2989 rc); 2990 } 2991 goto mcfail; 2992 } 2993 del = 0; 2994 i = 0; 2995 } 2996 } 2997 if (i > 0) { 2998 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i, 2999 mcaddr, NULL, &hash, 0); 3000 if (rc < 0) { 3001 rc = -rc; 3002 for (j = 0; j < i; j++) { 3003 if_printf(ifp, 3004 "failed to add mc address" 3005 " %02x:%02x:%02x:" 3006 "%02x:%02x:%02x rc=%d\n", 3007 mcaddr[j][0], mcaddr[j][1], 3008 mcaddr[j][2], mcaddr[j][3], 3009 mcaddr[j][4], mcaddr[j][5], 3010 rc); 3011 } 3012 goto mcfail; 3013 } 3014 } 3015 3016 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0); 3017 if (rc != 0) 3018 if_printf(ifp, "failed to set mc address hash: %d", rc); 3019 mcfail: 3020 if_maddr_runlock(ifp); 3021 } 3022 3023 return (rc); 3024 } 3025 3026 int 3027 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags, 3028 char *wmesg) 3029 { 3030 int rc, pri; 3031 3032 #ifdef WITNESS 3033 /* the caller thinks it's ok to sleep, but is it really? */ 3034 if (flags & SLEEP_OK) 3035 pause("t4slptst", 1); 3036 #endif 3037 3038 if (INTR_OK) 3039 pri = PCATCH; 3040 else 3041 pri = 0; 3042 3043 ADAPTER_LOCK(sc); 3044 for (;;) { 3045 3046 if (pi && IS_DOOMED(pi)) { 3047 rc = ENXIO; 3048 goto done; 3049 } 3050 3051 if (!IS_BUSY(sc)) { 3052 rc = 0; 3053 break; 3054 } 3055 3056 if (!(flags & SLEEP_OK)) { 3057 rc = EBUSY; 3058 goto done; 3059 } 3060 3061 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 3062 rc = EINTR; 3063 goto done; 3064 } 3065 } 3066 3067 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 3068 SET_BUSY(sc); 3069 #ifdef INVARIANTS 3070 sc->last_op = wmesg; 3071 sc->last_op_thr = curthread; 3072 #endif 3073 3074 done: 3075 if (!(flags & HOLD_LOCK) || rc) 3076 ADAPTER_UNLOCK(sc); 3077 3078 return (rc); 3079 } 3080 3081 void 3082 end_synchronized_op(struct adapter *sc, int flags) 3083 { 3084 3085 if (flags & LOCK_HELD) 3086 ADAPTER_LOCK_ASSERT_OWNED(sc); 3087 else 3088 ADAPTER_LOCK(sc); 3089 3090 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 3091 CLR_BUSY(sc); 3092 wakeup(&sc->flags); 3093 ADAPTER_UNLOCK(sc); 3094 } 3095 3096 static int 3097 cxgbe_init_synchronized(struct port_info *pi) 3098 { 3099 struct adapter *sc = pi->adapter; 3100 struct ifnet *ifp = pi->ifp; 3101 int rc = 0; 3102 3103 ASSERT_SYNCHRONIZED_OP(sc); 3104 3105 if (isset(&sc->open_device_map, pi->port_id)) { 3106 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 3107 ("mismatch between open_device_map and if_drv_flags")); 3108 return (0); /* already running */ 3109 } 3110 3111 if (!(sc->flags & FULL_INIT_DONE) && 3112 ((rc = adapter_full_init(sc)) != 0)) 3113 return (rc); /* error message displayed already */ 3114 3115 if (!(pi->flags & PORT_INIT_DONE) && 3116 ((rc = port_full_init(pi)) != 0)) 3117 return (rc); /* error message displayed already */ 3118 3119 rc = update_mac_settings(ifp, XGMAC_ALL); 3120 if (rc) 3121 goto done; /* error message displayed already */ 3122 3123 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 3124 if (rc != 0) { 3125 if_printf(ifp, "enable_vi failed: %d\n", rc); 3126 goto done; 3127 } 3128 3129 /* 3130 * The first iq of the first port to come up is used for tracing. 3131 */ 3132 if (sc->traceq < 0) { 3133 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id; 3134 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : 3135 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | 3136 V_QUEUENUMBER(sc->traceq)); 3137 pi->flags |= HAS_TRACEQ; 3138 } 3139 3140 /* all ok */ 3141 setbit(&sc->open_device_map, pi->port_id); 3142 PORT_LOCK(pi); 3143 ifp->if_drv_flags |= IFF_DRV_RUNNING; 3144 PORT_UNLOCK(pi); 3145 3146 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 3147 done: 3148 if (rc != 0) 3149 cxgbe_uninit_synchronized(pi); 3150 3151 return (rc); 3152 } 3153 3154 /* 3155 * Idempotent. 3156 */ 3157 static int 3158 cxgbe_uninit_synchronized(struct port_info *pi) 3159 { 3160 struct adapter *sc = pi->adapter; 3161 struct ifnet *ifp = pi->ifp; 3162 int rc; 3163 3164 ASSERT_SYNCHRONIZED_OP(sc); 3165 3166 /* 3167 * Disable the VI so that all its data in either direction is discarded 3168 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 3169 * tick) intact as the TP can deliver negative advice or data that it's 3170 * holding in its RAM (for an offloaded connection) even after the VI is 3171 * disabled. 3172 */ 3173 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 3174 if (rc) { 3175 if_printf(ifp, "disable_vi failed: %d\n", rc); 3176 return (rc); 3177 } 3178 3179 clrbit(&sc->open_device_map, pi->port_id); 3180 PORT_LOCK(pi); 3181 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3182 PORT_UNLOCK(pi); 3183 3184 pi->link_cfg.link_ok = 0; 3185 pi->link_cfg.speed = 0; 3186 pi->linkdnrc = -1; 3187 t4_os_link_changed(sc, pi->port_id, 0, -1); 3188 3189 return (0); 3190 } 3191 3192 /* 3193 * It is ok for this function to fail midway and return right away. t4_detach 3194 * will walk the entire sc->irq list and clean up whatever is valid. 3195 */ 3196 static int 3197 setup_intr_handlers(struct adapter *sc) 3198 { 3199 int rc, rid, p, q; 3200 char s[8]; 3201 struct irq *irq; 3202 struct port_info *pi; 3203 struct sge_rxq *rxq; 3204 #ifdef TCP_OFFLOAD 3205 struct sge_ofld_rxq *ofld_rxq; 3206 #endif 3207 #ifdef DEV_NETMAP 3208 struct sge_nm_rxq *nm_rxq; 3209 #endif 3210 3211 /* 3212 * Setup interrupts. 3213 */ 3214 irq = &sc->irq[0]; 3215 rid = sc->intr_type == INTR_INTX ? 0 : 1; 3216 if (sc->intr_count == 1) 3217 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all")); 3218 3219 /* Multiple interrupts. */ 3220 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 3221 ("%s: too few intr.", __func__)); 3222 3223 /* The first one is always error intr */ 3224 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 3225 if (rc != 0) 3226 return (rc); 3227 irq++; 3228 rid++; 3229 3230 /* The second one is always the firmware event queue */ 3231 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt"); 3232 if (rc != 0) 3233 return (rc); 3234 irq++; 3235 rid++; 3236 3237 for_each_port(sc, p) { 3238 pi = sc->port[p]; 3239 3240 if (pi->flags & INTR_RXQ) { 3241 for_each_rxq(pi, q, rxq) { 3242 snprintf(s, sizeof(s), "%d.%d", p, q); 3243 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq, 3244 s); 3245 if (rc != 0) 3246 return (rc); 3247 irq++; 3248 rid++; 3249 } 3250 } 3251 #ifdef TCP_OFFLOAD 3252 if (pi->flags & INTR_OFLD_RXQ) { 3253 for_each_ofld_rxq(pi, q, ofld_rxq) { 3254 snprintf(s, sizeof(s), "%d,%d", p, q); 3255 rc = t4_alloc_irq(sc, irq, rid, t4_intr, 3256 ofld_rxq, s); 3257 if (rc != 0) 3258 return (rc); 3259 irq++; 3260 rid++; 3261 } 3262 } 3263 #endif 3264 #ifdef DEV_NETMAP 3265 if (pi->flags & INTR_NM_RXQ) { 3266 for_each_nm_rxq(pi, q, nm_rxq) { 3267 snprintf(s, sizeof(s), "%d-%d", p, q); 3268 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr, 3269 nm_rxq, s); 3270 if (rc != 0) 3271 return (rc); 3272 irq++; 3273 rid++; 3274 } 3275 } 3276 #endif 3277 } 3278 MPASS(irq == &sc->irq[sc->intr_count]); 3279 3280 return (0); 3281 } 3282 3283 int 3284 adapter_full_init(struct adapter *sc) 3285 { 3286 int rc, i; 3287 3288 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3289 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 3290 ("%s: FULL_INIT_DONE already", __func__)); 3291 3292 /* 3293 * queues that belong to the adapter (not any particular port). 3294 */ 3295 rc = t4_setup_adapter_queues(sc); 3296 if (rc != 0) 3297 goto done; 3298 3299 for (i = 0; i < nitems(sc->tq); i++) { 3300 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 3301 taskqueue_thread_enqueue, &sc->tq[i]); 3302 if (sc->tq[i] == NULL) { 3303 device_printf(sc->dev, 3304 "failed to allocate task queue %d\n", i); 3305 rc = ENOMEM; 3306 goto done; 3307 } 3308 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3309 device_get_nameunit(sc->dev), i); 3310 } 3311 3312 t4_intr_enable(sc); 3313 sc->flags |= FULL_INIT_DONE; 3314 done: 3315 if (rc != 0) 3316 adapter_full_uninit(sc); 3317 3318 return (rc); 3319 } 3320 3321 int 3322 adapter_full_uninit(struct adapter *sc) 3323 { 3324 int i; 3325 3326 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3327 3328 t4_teardown_adapter_queues(sc); 3329 3330 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3331 taskqueue_free(sc->tq[i]); 3332 sc->tq[i] = NULL; 3333 } 3334 3335 sc->flags &= ~FULL_INIT_DONE; 3336 3337 return (0); 3338 } 3339 3340 int 3341 port_full_init(struct port_info *pi) 3342 { 3343 struct adapter *sc = pi->adapter; 3344 struct ifnet *ifp = pi->ifp; 3345 uint16_t *rss; 3346 struct sge_rxq *rxq; 3347 int rc, i, j; 3348 3349 ASSERT_SYNCHRONIZED_OP(sc); 3350 KASSERT((pi->flags & PORT_INIT_DONE) == 0, 3351 ("%s: PORT_INIT_DONE already", __func__)); 3352 3353 sysctl_ctx_init(&pi->ctx); 3354 pi->flags |= PORT_SYSCTL_CTX; 3355 3356 /* 3357 * Allocate tx/rx/fl queues for this port. 3358 */ 3359 rc = t4_setup_port_queues(pi); 3360 if (rc != 0) 3361 goto done; /* error message displayed already */ 3362 3363 /* 3364 * Setup RSS for this port. Save a copy of the RSS table for later use. 3365 */ 3366 rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); 3367 for (i = 0; i < pi->rss_size;) { 3368 for_each_rxq(pi, j, rxq) { 3369 rss[i++] = rxq->iq.abs_id; 3370 if (i == pi->rss_size) 3371 break; 3372 } 3373 } 3374 3375 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss, 3376 pi->rss_size); 3377 if (rc != 0) { 3378 if_printf(ifp, "rss_config failed: %d\n", rc); 3379 goto done; 3380 } 3381 3382 pi->rss = rss; 3383 pi->flags |= PORT_INIT_DONE; 3384 done: 3385 if (rc != 0) 3386 port_full_uninit(pi); 3387 3388 return (rc); 3389 } 3390 3391 /* 3392 * Idempotent. 3393 */ 3394 int 3395 port_full_uninit(struct port_info *pi) 3396 { 3397 struct adapter *sc = pi->adapter; 3398 int i; 3399 struct sge_rxq *rxq; 3400 struct sge_txq *txq; 3401 #ifdef TCP_OFFLOAD 3402 struct sge_ofld_rxq *ofld_rxq; 3403 struct sge_wrq *ofld_txq; 3404 #endif 3405 3406 if (pi->flags & PORT_INIT_DONE) { 3407 3408 /* Need to quiesce queues. XXX: ctrl queues? */ 3409 3410 for_each_txq(pi, i, txq) { 3411 quiesce_eq(sc, &txq->eq); 3412 } 3413 3414 #ifdef TCP_OFFLOAD 3415 for_each_ofld_txq(pi, i, ofld_txq) { 3416 quiesce_eq(sc, &ofld_txq->eq); 3417 } 3418 #endif 3419 3420 for_each_rxq(pi, i, rxq) { 3421 quiesce_iq(sc, &rxq->iq); 3422 quiesce_fl(sc, &rxq->fl); 3423 } 3424 3425 #ifdef TCP_OFFLOAD 3426 for_each_ofld_rxq(pi, i, ofld_rxq) { 3427 quiesce_iq(sc, &ofld_rxq->iq); 3428 quiesce_fl(sc, &ofld_rxq->fl); 3429 } 3430 #endif 3431 free(pi->rss, M_CXGBE); 3432 } 3433 3434 t4_teardown_port_queues(pi); 3435 pi->flags &= ~PORT_INIT_DONE; 3436 3437 return (0); 3438 } 3439 3440 static void 3441 quiesce_eq(struct adapter *sc, struct sge_eq *eq) 3442 { 3443 EQ_LOCK(eq); 3444 eq->flags |= EQ_DOOMED; 3445 3446 /* 3447 * Wait for the response to a credit flush if one's 3448 * pending. 3449 */ 3450 while (eq->flags & EQ_CRFLUSHED) 3451 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0); 3452 EQ_UNLOCK(eq); 3453 3454 callout_drain(&eq->tx_callout); /* XXX: iffy */ 3455 pause("callout", 10); /* Still iffy */ 3456 3457 taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task); 3458 } 3459 3460 static void 3461 quiesce_iq(struct adapter *sc, struct sge_iq *iq) 3462 { 3463 (void) sc; /* unused */ 3464 3465 /* Synchronize with the interrupt handler */ 3466 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 3467 pause("iqfree", 1); 3468 } 3469 3470 static void 3471 quiesce_fl(struct adapter *sc, struct sge_fl *fl) 3472 { 3473 mtx_lock(&sc->sfl_lock); 3474 FL_LOCK(fl); 3475 fl->flags |= FL_DOOMED; 3476 FL_UNLOCK(fl); 3477 mtx_unlock(&sc->sfl_lock); 3478 3479 callout_drain(&sc->sfl_callout); 3480 KASSERT((fl->flags & FL_STARVING) == 0, 3481 ("%s: still starving", __func__)); 3482 } 3483 3484 static int 3485 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 3486 driver_intr_t *handler, void *arg, char *name) 3487 { 3488 int rc; 3489 3490 irq->rid = rid; 3491 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 3492 RF_SHAREABLE | RF_ACTIVE); 3493 if (irq->res == NULL) { 3494 device_printf(sc->dev, 3495 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 3496 return (ENOMEM); 3497 } 3498 3499 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 3500 NULL, handler, arg, &irq->tag); 3501 if (rc != 0) { 3502 device_printf(sc->dev, 3503 "failed to setup interrupt for rid %d, name %s: %d\n", 3504 rid, name, rc); 3505 } else if (name) 3506 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 3507 3508 return (rc); 3509 } 3510 3511 static int 3512 t4_free_irq(struct adapter *sc, struct irq *irq) 3513 { 3514 if (irq->tag) 3515 bus_teardown_intr(sc->dev, irq->res, irq->tag); 3516 if (irq->res) 3517 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 3518 3519 bzero(irq, sizeof(*irq)); 3520 3521 return (0); 3522 } 3523 3524 static void 3525 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 3526 unsigned int end) 3527 { 3528 uint32_t *p = (uint32_t *)(buf + start); 3529 3530 for ( ; start <= end; start += sizeof(uint32_t)) 3531 *p++ = t4_read_reg(sc, start); 3532 } 3533 3534 static void 3535 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 3536 { 3537 int i, n; 3538 const unsigned int *reg_ranges; 3539 static const unsigned int t4_reg_ranges[] = { 3540 0x1008, 0x1108, 3541 0x1180, 0x11b4, 3542 0x11fc, 0x123c, 3543 0x1300, 0x173c, 3544 0x1800, 0x18fc, 3545 0x3000, 0x30d8, 3546 0x30e0, 0x5924, 3547 0x5960, 0x59d4, 3548 0x5a00, 0x5af8, 3549 0x6000, 0x6098, 3550 0x6100, 0x6150, 3551 0x6200, 0x6208, 3552 0x6240, 0x6248, 3553 0x6280, 0x6338, 3554 0x6370, 0x638c, 3555 0x6400, 0x643c, 3556 0x6500, 0x6524, 3557 0x6a00, 0x6a38, 3558 0x6a60, 0x6a78, 3559 0x6b00, 0x6b84, 3560 0x6bf0, 0x6c84, 3561 0x6cf0, 0x6d84, 3562 0x6df0, 0x6e84, 3563 0x6ef0, 0x6f84, 3564 0x6ff0, 0x7084, 3565 0x70f0, 0x7184, 3566 0x71f0, 0x7284, 3567 0x72f0, 0x7384, 3568 0x73f0, 0x7450, 3569 0x7500, 0x7530, 3570 0x7600, 0x761c, 3571 0x7680, 0x76cc, 3572 0x7700, 0x7798, 3573 0x77c0, 0x77fc, 3574 0x7900, 0x79fc, 3575 0x7b00, 0x7c38, 3576 0x7d00, 0x7efc, 3577 0x8dc0, 0x8e1c, 3578 0x8e30, 0x8e78, 3579 0x8ea0, 0x8f6c, 3580 0x8fc0, 0x9074, 3581 0x90fc, 0x90fc, 3582 0x9400, 0x9458, 3583 0x9600, 0x96bc, 3584 0x9800, 0x9808, 3585 0x9820, 0x983c, 3586 0x9850, 0x9864, 3587 0x9c00, 0x9c6c, 3588 0x9c80, 0x9cec, 3589 0x9d00, 0x9d6c, 3590 0x9d80, 0x9dec, 3591 0x9e00, 0x9e6c, 3592 0x9e80, 0x9eec, 3593 0x9f00, 0x9f6c, 3594 0x9f80, 0x9fec, 3595 0xd004, 0xd03c, 3596 0xdfc0, 0xdfe0, 3597 0xe000, 0xea7c, 3598 0xf000, 0x11110, 3599 0x11118, 0x11190, 3600 0x19040, 0x1906c, 3601 0x19078, 0x19080, 3602 0x1908c, 0x19124, 3603 0x19150, 0x191b0, 3604 0x191d0, 0x191e8, 3605 0x19238, 0x1924c, 3606 0x193f8, 0x19474, 3607 0x19490, 0x194f8, 3608 0x19800, 0x19f30, 3609 0x1a000, 0x1a06c, 3610 0x1a0b0, 0x1a120, 3611 0x1a128, 0x1a138, 3612 0x1a190, 0x1a1c4, 3613 0x1a1fc, 0x1a1fc, 3614 0x1e040, 0x1e04c, 3615 0x1e284, 0x1e28c, 3616 0x1e2c0, 0x1e2c0, 3617 0x1e2e0, 0x1e2e0, 3618 0x1e300, 0x1e384, 3619 0x1e3c0, 0x1e3c8, 3620 0x1e440, 0x1e44c, 3621 0x1e684, 0x1e68c, 3622 0x1e6c0, 0x1e6c0, 3623 0x1e6e0, 0x1e6e0, 3624 0x1e700, 0x1e784, 3625 0x1e7c0, 0x1e7c8, 3626 0x1e840, 0x1e84c, 3627 0x1ea84, 0x1ea8c, 3628 0x1eac0, 0x1eac0, 3629 0x1eae0, 0x1eae0, 3630 0x1eb00, 0x1eb84, 3631 0x1ebc0, 0x1ebc8, 3632 0x1ec40, 0x1ec4c, 3633 0x1ee84, 0x1ee8c, 3634 0x1eec0, 0x1eec0, 3635 0x1eee0, 0x1eee0, 3636 0x1ef00, 0x1ef84, 3637 0x1efc0, 0x1efc8, 3638 0x1f040, 0x1f04c, 3639 0x1f284, 0x1f28c, 3640 0x1f2c0, 0x1f2c0, 3641 0x1f2e0, 0x1f2e0, 3642 0x1f300, 0x1f384, 3643 0x1f3c0, 0x1f3c8, 3644 0x1f440, 0x1f44c, 3645 0x1f684, 0x1f68c, 3646 0x1f6c0, 0x1f6c0, 3647 0x1f6e0, 0x1f6e0, 3648 0x1f700, 0x1f784, 3649 0x1f7c0, 0x1f7c8, 3650 0x1f840, 0x1f84c, 3651 0x1fa84, 0x1fa8c, 3652 0x1fac0, 0x1fac0, 3653 0x1fae0, 0x1fae0, 3654 0x1fb00, 0x1fb84, 3655 0x1fbc0, 0x1fbc8, 3656 0x1fc40, 0x1fc4c, 3657 0x1fe84, 0x1fe8c, 3658 0x1fec0, 0x1fec0, 3659 0x1fee0, 0x1fee0, 3660 0x1ff00, 0x1ff84, 3661 0x1ffc0, 0x1ffc8, 3662 0x20000, 0x2002c, 3663 0x20100, 0x2013c, 3664 0x20190, 0x201c8, 3665 0x20200, 0x20318, 3666 0x20400, 0x20528, 3667 0x20540, 0x20614, 3668 0x21000, 0x21040, 3669 0x2104c, 0x21060, 3670 0x210c0, 0x210ec, 3671 0x21200, 0x21268, 3672 0x21270, 0x21284, 3673 0x212fc, 0x21388, 3674 0x21400, 0x21404, 3675 0x21500, 0x21518, 3676 0x2152c, 0x2153c, 3677 0x21550, 0x21554, 3678 0x21600, 0x21600, 3679 0x21608, 0x21628, 3680 0x21630, 0x2163c, 3681 0x21700, 0x2171c, 3682 0x21780, 0x2178c, 3683 0x21800, 0x21c38, 3684 0x21c80, 0x21d7c, 3685 0x21e00, 0x21e04, 3686 0x22000, 0x2202c, 3687 0x22100, 0x2213c, 3688 0x22190, 0x221c8, 3689 0x22200, 0x22318, 3690 0x22400, 0x22528, 3691 0x22540, 0x22614, 3692 0x23000, 0x23040, 3693 0x2304c, 0x23060, 3694 0x230c0, 0x230ec, 3695 0x23200, 0x23268, 3696 0x23270, 0x23284, 3697 0x232fc, 0x23388, 3698 0x23400, 0x23404, 3699 0x23500, 0x23518, 3700 0x2352c, 0x2353c, 3701 0x23550, 0x23554, 3702 0x23600, 0x23600, 3703 0x23608, 0x23628, 3704 0x23630, 0x2363c, 3705 0x23700, 0x2371c, 3706 0x23780, 0x2378c, 3707 0x23800, 0x23c38, 3708 0x23c80, 0x23d7c, 3709 0x23e00, 0x23e04, 3710 0x24000, 0x2402c, 3711 0x24100, 0x2413c, 3712 0x24190, 0x241c8, 3713 0x24200, 0x24318, 3714 0x24400, 0x24528, 3715 0x24540, 0x24614, 3716 0x25000, 0x25040, 3717 0x2504c, 0x25060, 3718 0x250c0, 0x250ec, 3719 0x25200, 0x25268, 3720 0x25270, 0x25284, 3721 0x252fc, 0x25388, 3722 0x25400, 0x25404, 3723 0x25500, 0x25518, 3724 0x2552c, 0x2553c, 3725 0x25550, 0x25554, 3726 0x25600, 0x25600, 3727 0x25608, 0x25628, 3728 0x25630, 0x2563c, 3729 0x25700, 0x2571c, 3730 0x25780, 0x2578c, 3731 0x25800, 0x25c38, 3732 0x25c80, 0x25d7c, 3733 0x25e00, 0x25e04, 3734 0x26000, 0x2602c, 3735 0x26100, 0x2613c, 3736 0x26190, 0x261c8, 3737 0x26200, 0x26318, 3738 0x26400, 0x26528, 3739 0x26540, 0x26614, 3740 0x27000, 0x27040, 3741 0x2704c, 0x27060, 3742 0x270c0, 0x270ec, 3743 0x27200, 0x27268, 3744 0x27270, 0x27284, 3745 0x272fc, 0x27388, 3746 0x27400, 0x27404, 3747 0x27500, 0x27518, 3748 0x2752c, 0x2753c, 3749 0x27550, 0x27554, 3750 0x27600, 0x27600, 3751 0x27608, 0x27628, 3752 0x27630, 0x2763c, 3753 0x27700, 0x2771c, 3754 0x27780, 0x2778c, 3755 0x27800, 0x27c38, 3756 0x27c80, 0x27d7c, 3757 0x27e00, 0x27e04 3758 }; 3759 static const unsigned int t5_reg_ranges[] = { 3760 0x1008, 0x1148, 3761 0x1180, 0x11b4, 3762 0x11fc, 0x123c, 3763 0x1280, 0x173c, 3764 0x1800, 0x18fc, 3765 0x3000, 0x3028, 3766 0x3060, 0x30d8, 3767 0x30e0, 0x30fc, 3768 0x3140, 0x357c, 3769 0x35a8, 0x35cc, 3770 0x35ec, 0x35ec, 3771 0x3600, 0x5624, 3772 0x56cc, 0x575c, 3773 0x580c, 0x5814, 3774 0x5890, 0x58bc, 3775 0x5940, 0x59dc, 3776 0x59fc, 0x5a18, 3777 0x5a60, 0x5a9c, 3778 0x5b94, 0x5bfc, 3779 0x6000, 0x6040, 3780 0x6058, 0x614c, 3781 0x7700, 0x7798, 3782 0x77c0, 0x78fc, 3783 0x7b00, 0x7c54, 3784 0x7d00, 0x7efc, 3785 0x8dc0, 0x8de0, 3786 0x8df8, 0x8e84, 3787 0x8ea0, 0x8f84, 3788 0x8fc0, 0x90f8, 3789 0x9400, 0x9470, 3790 0x9600, 0x96f4, 3791 0x9800, 0x9808, 3792 0x9820, 0x983c, 3793 0x9850, 0x9864, 3794 0x9c00, 0x9c6c, 3795 0x9c80, 0x9cec, 3796 0x9d00, 0x9d6c, 3797 0x9d80, 0x9dec, 3798 0x9e00, 0x9e6c, 3799 0x9e80, 0x9eec, 3800 0x9f00, 0x9f6c, 3801 0x9f80, 0xa020, 3802 0xd004, 0xd03c, 3803 0xdfc0, 0xdfe0, 3804 0xe000, 0x11088, 3805 0x1109c, 0x11110, 3806 0x11118, 0x1117c, 3807 0x11190, 0x11204, 3808 0x19040, 0x1906c, 3809 0x19078, 0x19080, 3810 0x1908c, 0x19124, 3811 0x19150, 0x191b0, 3812 0x191d0, 0x191e8, 3813 0x19238, 0x19290, 3814 0x193f8, 0x19474, 3815 0x19490, 0x194cc, 3816 0x194f0, 0x194f8, 3817 0x19c00, 0x19c60, 3818 0x19c94, 0x19e10, 3819 0x19e50, 0x19f34, 3820 0x19f40, 0x19f50, 3821 0x19f90, 0x19fe4, 3822 0x1a000, 0x1a06c, 3823 0x1a0b0, 0x1a120, 3824 0x1a128, 0x1a138, 3825 0x1a190, 0x1a1c4, 3826 0x1a1fc, 0x1a1fc, 3827 0x1e008, 0x1e00c, 3828 0x1e040, 0x1e04c, 3829 0x1e284, 0x1e290, 3830 0x1e2c0, 0x1e2c0, 3831 0x1e2e0, 0x1e2e0, 3832 0x1e300, 0x1e384, 3833 0x1e3c0, 0x1e3c8, 3834 0x1e408, 0x1e40c, 3835 0x1e440, 0x1e44c, 3836 0x1e684, 0x1e690, 3837 0x1e6c0, 0x1e6c0, 3838 0x1e6e0, 0x1e6e0, 3839 0x1e700, 0x1e784, 3840 0x1e7c0, 0x1e7c8, 3841 0x1e808, 0x1e80c, 3842 0x1e840, 0x1e84c, 3843 0x1ea84, 0x1ea90, 3844 0x1eac0, 0x1eac0, 3845 0x1eae0, 0x1eae0, 3846 0x1eb00, 0x1eb84, 3847 0x1ebc0, 0x1ebc8, 3848 0x1ec08, 0x1ec0c, 3849 0x1ec40, 0x1ec4c, 3850 0x1ee84, 0x1ee90, 3851 0x1eec0, 0x1eec0, 3852 0x1eee0, 0x1eee0, 3853 0x1ef00, 0x1ef84, 3854 0x1efc0, 0x1efc8, 3855 0x1f008, 0x1f00c, 3856 0x1f040, 0x1f04c, 3857 0x1f284, 0x1f290, 3858 0x1f2c0, 0x1f2c0, 3859 0x1f2e0, 0x1f2e0, 3860 0x1f300, 0x1f384, 3861 0x1f3c0, 0x1f3c8, 3862 0x1f408, 0x1f40c, 3863 0x1f440, 0x1f44c, 3864 0x1f684, 0x1f690, 3865 0x1f6c0, 0x1f6c0, 3866 0x1f6e0, 0x1f6e0, 3867 0x1f700, 0x1f784, 3868 0x1f7c0, 0x1f7c8, 3869 0x1f808, 0x1f80c, 3870 0x1f840, 0x1f84c, 3871 0x1fa84, 0x1fa90, 3872 0x1fac0, 0x1fac0, 3873 0x1fae0, 0x1fae0, 3874 0x1fb00, 0x1fb84, 3875 0x1fbc0, 0x1fbc8, 3876 0x1fc08, 0x1fc0c, 3877 0x1fc40, 0x1fc4c, 3878 0x1fe84, 0x1fe90, 3879 0x1fec0, 0x1fec0, 3880 0x1fee0, 0x1fee0, 3881 0x1ff00, 0x1ff84, 3882 0x1ffc0, 0x1ffc8, 3883 0x30000, 0x30030, 3884 0x30100, 0x30144, 3885 0x30190, 0x301d0, 3886 0x30200, 0x30318, 3887 0x30400, 0x3052c, 3888 0x30540, 0x3061c, 3889 0x30800, 0x30834, 3890 0x308c0, 0x30908, 3891 0x30910, 0x309ac, 3892 0x30a00, 0x30a2c, 3893 0x30a44, 0x30a50, 3894 0x30a74, 0x30c24, 3895 0x30d00, 0x30d00, 3896 0x30d08, 0x30d14, 3897 0x30d1c, 0x30d20, 3898 0x30d3c, 0x30d50, 3899 0x31200, 0x3120c, 3900 0x31220, 0x31220, 3901 0x31240, 0x31240, 3902 0x31600, 0x3160c, 3903 0x31a00, 0x31a1c, 3904 0x31e00, 0x31e20, 3905 0x31e38, 0x31e3c, 3906 0x31e80, 0x31e80, 3907 0x31e88, 0x31ea8, 3908 0x31eb0, 0x31eb4, 3909 0x31ec8, 0x31ed4, 3910 0x31fb8, 0x32004, 3911 0x32200, 0x32200, 3912 0x32208, 0x32240, 3913 0x32248, 0x32280, 3914 0x32288, 0x322c0, 3915 0x322c8, 0x322fc, 3916 0x32600, 0x32630, 3917 0x32a00, 0x32abc, 3918 0x32b00, 0x32b70, 3919 0x33000, 0x33048, 3920 0x33060, 0x3309c, 3921 0x330f0, 0x33148, 3922 0x33160, 0x3319c, 3923 0x331f0, 0x332e4, 3924 0x332f8, 0x333e4, 3925 0x333f8, 0x33448, 3926 0x33460, 0x3349c, 3927 0x334f0, 0x33548, 3928 0x33560, 0x3359c, 3929 0x335f0, 0x336e4, 3930 0x336f8, 0x337e4, 3931 0x337f8, 0x337fc, 3932 0x33814, 0x33814, 3933 0x3382c, 0x3382c, 3934 0x33880, 0x3388c, 3935 0x338e8, 0x338ec, 3936 0x33900, 0x33948, 3937 0x33960, 0x3399c, 3938 0x339f0, 0x33ae4, 3939 0x33af8, 0x33b10, 3940 0x33b28, 0x33b28, 3941 0x33b3c, 0x33b50, 3942 0x33bf0, 0x33c10, 3943 0x33c28, 0x33c28, 3944 0x33c3c, 0x33c50, 3945 0x33cf0, 0x33cfc, 3946 0x34000, 0x34030, 3947 0x34100, 0x34144, 3948 0x34190, 0x341d0, 3949 0x34200, 0x34318, 3950 0x34400, 0x3452c, 3951 0x34540, 0x3461c, 3952 0x34800, 0x34834, 3953 0x348c0, 0x34908, 3954 0x34910, 0x349ac, 3955 0x34a00, 0x34a2c, 3956 0x34a44, 0x34a50, 3957 0x34a74, 0x34c24, 3958 0x34d00, 0x34d00, 3959 0x34d08, 0x34d14, 3960 0x34d1c, 0x34d20, 3961 0x34d3c, 0x34d50, 3962 0x35200, 0x3520c, 3963 0x35220, 0x35220, 3964 0x35240, 0x35240, 3965 0x35600, 0x3560c, 3966 0x35a00, 0x35a1c, 3967 0x35e00, 0x35e20, 3968 0x35e38, 0x35e3c, 3969 0x35e80, 0x35e80, 3970 0x35e88, 0x35ea8, 3971 0x35eb0, 0x35eb4, 3972 0x35ec8, 0x35ed4, 3973 0x35fb8, 0x36004, 3974 0x36200, 0x36200, 3975 0x36208, 0x36240, 3976 0x36248, 0x36280, 3977 0x36288, 0x362c0, 3978 0x362c8, 0x362fc, 3979 0x36600, 0x36630, 3980 0x36a00, 0x36abc, 3981 0x36b00, 0x36b70, 3982 0x37000, 0x37048, 3983 0x37060, 0x3709c, 3984 0x370f0, 0x37148, 3985 0x37160, 0x3719c, 3986 0x371f0, 0x372e4, 3987 0x372f8, 0x373e4, 3988 0x373f8, 0x37448, 3989 0x37460, 0x3749c, 3990 0x374f0, 0x37548, 3991 0x37560, 0x3759c, 3992 0x375f0, 0x376e4, 3993 0x376f8, 0x377e4, 3994 0x377f8, 0x377fc, 3995 0x37814, 0x37814, 3996 0x3782c, 0x3782c, 3997 0x37880, 0x3788c, 3998 0x378e8, 0x378ec, 3999 0x37900, 0x37948, 4000 0x37960, 0x3799c, 4001 0x379f0, 0x37ae4, 4002 0x37af8, 0x37b10, 4003 0x37b28, 0x37b28, 4004 0x37b3c, 0x37b50, 4005 0x37bf0, 0x37c10, 4006 0x37c28, 0x37c28, 4007 0x37c3c, 0x37c50, 4008 0x37cf0, 0x37cfc, 4009 0x38000, 0x38030, 4010 0x38100, 0x38144, 4011 0x38190, 0x381d0, 4012 0x38200, 0x38318, 4013 0x38400, 0x3852c, 4014 0x38540, 0x3861c, 4015 0x38800, 0x38834, 4016 0x388c0, 0x38908, 4017 0x38910, 0x389ac, 4018 0x38a00, 0x38a2c, 4019 0x38a44, 0x38a50, 4020 0x38a74, 0x38c24, 4021 0x38d00, 0x38d00, 4022 0x38d08, 0x38d14, 4023 0x38d1c, 0x38d20, 4024 0x38d3c, 0x38d50, 4025 0x39200, 0x3920c, 4026 0x39220, 0x39220, 4027 0x39240, 0x39240, 4028 0x39600, 0x3960c, 4029 0x39a00, 0x39a1c, 4030 0x39e00, 0x39e20, 4031 0x39e38, 0x39e3c, 4032 0x39e80, 0x39e80, 4033 0x39e88, 0x39ea8, 4034 0x39eb0, 0x39eb4, 4035 0x39ec8, 0x39ed4, 4036 0x39fb8, 0x3a004, 4037 0x3a200, 0x3a200, 4038 0x3a208, 0x3a240, 4039 0x3a248, 0x3a280, 4040 0x3a288, 0x3a2c0, 4041 0x3a2c8, 0x3a2fc, 4042 0x3a600, 0x3a630, 4043 0x3aa00, 0x3aabc, 4044 0x3ab00, 0x3ab70, 4045 0x3b000, 0x3b048, 4046 0x3b060, 0x3b09c, 4047 0x3b0f0, 0x3b148, 4048 0x3b160, 0x3b19c, 4049 0x3b1f0, 0x3b2e4, 4050 0x3b2f8, 0x3b3e4, 4051 0x3b3f8, 0x3b448, 4052 0x3b460, 0x3b49c, 4053 0x3b4f0, 0x3b548, 4054 0x3b560, 0x3b59c, 4055 0x3b5f0, 0x3b6e4, 4056 0x3b6f8, 0x3b7e4, 4057 0x3b7f8, 0x3b7fc, 4058 0x3b814, 0x3b814, 4059 0x3b82c, 0x3b82c, 4060 0x3b880, 0x3b88c, 4061 0x3b8e8, 0x3b8ec, 4062 0x3b900, 0x3b948, 4063 0x3b960, 0x3b99c, 4064 0x3b9f0, 0x3bae4, 4065 0x3baf8, 0x3bb10, 4066 0x3bb28, 0x3bb28, 4067 0x3bb3c, 0x3bb50, 4068 0x3bbf0, 0x3bc10, 4069 0x3bc28, 0x3bc28, 4070 0x3bc3c, 0x3bc50, 4071 0x3bcf0, 0x3bcfc, 4072 0x3c000, 0x3c030, 4073 0x3c100, 0x3c144, 4074 0x3c190, 0x3c1d0, 4075 0x3c200, 0x3c318, 4076 0x3c400, 0x3c52c, 4077 0x3c540, 0x3c61c, 4078 0x3c800, 0x3c834, 4079 0x3c8c0, 0x3c908, 4080 0x3c910, 0x3c9ac, 4081 0x3ca00, 0x3ca2c, 4082 0x3ca44, 0x3ca50, 4083 0x3ca74, 0x3cc24, 4084 0x3cd00, 0x3cd00, 4085 0x3cd08, 0x3cd14, 4086 0x3cd1c, 0x3cd20, 4087 0x3cd3c, 0x3cd50, 4088 0x3d200, 0x3d20c, 4089 0x3d220, 0x3d220, 4090 0x3d240, 0x3d240, 4091 0x3d600, 0x3d60c, 4092 0x3da00, 0x3da1c, 4093 0x3de00, 0x3de20, 4094 0x3de38, 0x3de3c, 4095 0x3de80, 0x3de80, 4096 0x3de88, 0x3dea8, 4097 0x3deb0, 0x3deb4, 4098 0x3dec8, 0x3ded4, 4099 0x3dfb8, 0x3e004, 4100 0x3e200, 0x3e200, 4101 0x3e208, 0x3e240, 4102 0x3e248, 0x3e280, 4103 0x3e288, 0x3e2c0, 4104 0x3e2c8, 0x3e2fc, 4105 0x3e600, 0x3e630, 4106 0x3ea00, 0x3eabc, 4107 0x3eb00, 0x3eb70, 4108 0x3f000, 0x3f048, 4109 0x3f060, 0x3f09c, 4110 0x3f0f0, 0x3f148, 4111 0x3f160, 0x3f19c, 4112 0x3f1f0, 0x3f2e4, 4113 0x3f2f8, 0x3f3e4, 4114 0x3f3f8, 0x3f448, 4115 0x3f460, 0x3f49c, 4116 0x3f4f0, 0x3f548, 4117 0x3f560, 0x3f59c, 4118 0x3f5f0, 0x3f6e4, 4119 0x3f6f8, 0x3f7e4, 4120 0x3f7f8, 0x3f7fc, 4121 0x3f814, 0x3f814, 4122 0x3f82c, 0x3f82c, 4123 0x3f880, 0x3f88c, 4124 0x3f8e8, 0x3f8ec, 4125 0x3f900, 0x3f948, 4126 0x3f960, 0x3f99c, 4127 0x3f9f0, 0x3fae4, 4128 0x3faf8, 0x3fb10, 4129 0x3fb28, 0x3fb28, 4130 0x3fb3c, 0x3fb50, 4131 0x3fbf0, 0x3fc10, 4132 0x3fc28, 0x3fc28, 4133 0x3fc3c, 0x3fc50, 4134 0x3fcf0, 0x3fcfc, 4135 0x40000, 0x4000c, 4136 0x40040, 0x40068, 4137 0x4007c, 0x40144, 4138 0x40180, 0x4018c, 4139 0x40200, 0x40298, 4140 0x402ac, 0x4033c, 4141 0x403f8, 0x403fc, 4142 0x41304, 0x413c4, 4143 0x41400, 0x4141c, 4144 0x41480, 0x414d0, 4145 0x44000, 0x44078, 4146 0x440c0, 0x44278, 4147 0x442c0, 0x44478, 4148 0x444c0, 0x44678, 4149 0x446c0, 0x44878, 4150 0x448c0, 0x449fc, 4151 0x45000, 0x45068, 4152 0x45080, 0x45084, 4153 0x450a0, 0x450b0, 4154 0x45200, 0x45268, 4155 0x45280, 0x45284, 4156 0x452a0, 0x452b0, 4157 0x460c0, 0x460e4, 4158 0x47000, 0x4708c, 4159 0x47200, 0x47250, 4160 0x47400, 0x47420, 4161 0x47600, 0x47618, 4162 0x47800, 0x47814, 4163 0x48000, 0x4800c, 4164 0x48040, 0x48068, 4165 0x4807c, 0x48144, 4166 0x48180, 0x4818c, 4167 0x48200, 0x48298, 4168 0x482ac, 0x4833c, 4169 0x483f8, 0x483fc, 4170 0x49304, 0x493c4, 4171 0x49400, 0x4941c, 4172 0x49480, 0x494d0, 4173 0x4c000, 0x4c078, 4174 0x4c0c0, 0x4c278, 4175 0x4c2c0, 0x4c478, 4176 0x4c4c0, 0x4c678, 4177 0x4c6c0, 0x4c878, 4178 0x4c8c0, 0x4c9fc, 4179 0x4d000, 0x4d068, 4180 0x4d080, 0x4d084, 4181 0x4d0a0, 0x4d0b0, 4182 0x4d200, 0x4d268, 4183 0x4d280, 0x4d284, 4184 0x4d2a0, 0x4d2b0, 4185 0x4e0c0, 0x4e0e4, 4186 0x4f000, 0x4f08c, 4187 0x4f200, 0x4f250, 4188 0x4f400, 0x4f420, 4189 0x4f600, 0x4f618, 4190 0x4f800, 0x4f814, 4191 0x50000, 0x500cc, 4192 0x50400, 0x50400, 4193 0x50800, 0x508cc, 4194 0x50c00, 0x50c00, 4195 0x51000, 0x5101c, 4196 0x51300, 0x51308, 4197 }; 4198 4199 if (is_t4(sc)) { 4200 reg_ranges = &t4_reg_ranges[0]; 4201 n = nitems(t4_reg_ranges); 4202 } else { 4203 reg_ranges = &t5_reg_ranges[0]; 4204 n = nitems(t5_reg_ranges); 4205 } 4206 4207 regs->version = chip_id(sc) | chip_rev(sc) << 10; 4208 for (i = 0; i < n; i += 2) 4209 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 4210 } 4211 4212 static void 4213 cxgbe_tick(void *arg) 4214 { 4215 struct port_info *pi = arg; 4216 struct adapter *sc = pi->adapter; 4217 struct ifnet *ifp = pi->ifp; 4218 struct sge_txq *txq; 4219 int i, drops; 4220 struct port_stats *s = &pi->stats; 4221 4222 PORT_LOCK(pi); 4223 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 4224 PORT_UNLOCK(pi); 4225 return; /* without scheduling another callout */ 4226 } 4227 4228 t4_get_port_stats(sc, pi->tx_chan, s); 4229 4230 ifp->if_opackets = s->tx_frames - s->tx_pause; 4231 ifp->if_ipackets = s->rx_frames - s->rx_pause; 4232 ifp->if_obytes = s->tx_octets - s->tx_pause * 64; 4233 ifp->if_ibytes = s->rx_octets - s->rx_pause * 64; 4234 ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause; 4235 ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause; 4236 ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 4237 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 4238 s->rx_trunc3; 4239 for (i = 0; i < 4; i++) { 4240 if (pi->rx_chan_map & (1 << i)) { 4241 uint32_t v; 4242 4243 /* 4244 * XXX: indirect reads from the same ADDR/DATA pair can 4245 * race with each other. 4246 */ 4247 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 4248 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 4249 ifp->if_iqdrops += v; 4250 } 4251 } 4252 4253 drops = s->tx_drop; 4254 for_each_txq(pi, i, txq) 4255 drops += txq->br->br_drops; 4256 ifp->if_oqdrops = drops; 4257 4258 ifp->if_oerrors = s->tx_error_frames; 4259 ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long + 4260 s->rx_fcs_err + s->rx_len_err; 4261 4262 callout_schedule(&pi->tick, hz); 4263 PORT_UNLOCK(pi); 4264 } 4265 4266 static void 4267 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 4268 { 4269 struct ifnet *vlan; 4270 4271 if (arg != ifp || ifp->if_type != IFT_ETHER) 4272 return; 4273 4274 vlan = VLAN_DEVAT(ifp, vid); 4275 VLAN_SETCOOKIE(vlan, ifp); 4276 } 4277 4278 static int 4279 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 4280 { 4281 4282 #ifdef INVARIANTS 4283 panic("%s: opcode 0x%02x on iq %p with payload %p", 4284 __func__, rss->opcode, iq, m); 4285 #else 4286 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 4287 __func__, rss->opcode, iq, m); 4288 m_freem(m); 4289 #endif 4290 return (EDOOFUS); 4291 } 4292 4293 int 4294 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 4295 { 4296 uintptr_t *loc, new; 4297 4298 if (opcode >= nitems(sc->cpl_handler)) 4299 return (EINVAL); 4300 4301 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 4302 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 4303 atomic_store_rel_ptr(loc, new); 4304 4305 return (0); 4306 } 4307 4308 static int 4309 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 4310 { 4311 4312 #ifdef INVARIANTS 4313 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 4314 #else 4315 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 4316 __func__, iq, ctrl); 4317 #endif 4318 return (EDOOFUS); 4319 } 4320 4321 int 4322 t4_register_an_handler(struct adapter *sc, an_handler_t h) 4323 { 4324 uintptr_t *loc, new; 4325 4326 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 4327 loc = (uintptr_t *) &sc->an_handler; 4328 atomic_store_rel_ptr(loc, new); 4329 4330 return (0); 4331 } 4332 4333 static int 4334 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 4335 { 4336 const struct cpl_fw6_msg *cpl = 4337 __containerof(rpl, struct cpl_fw6_msg, data[0]); 4338 4339 #ifdef INVARIANTS 4340 panic("%s: fw_msg type %d", __func__, cpl->type); 4341 #else 4342 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 4343 #endif 4344 return (EDOOFUS); 4345 } 4346 4347 int 4348 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 4349 { 4350 uintptr_t *loc, new; 4351 4352 if (type >= nitems(sc->fw_msg_handler)) 4353 return (EINVAL); 4354 4355 /* 4356 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 4357 * handler dispatch table. Reject any attempt to install a handler for 4358 * this subtype. 4359 */ 4360 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 4361 return (EINVAL); 4362 4363 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 4364 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 4365 atomic_store_rel_ptr(loc, new); 4366 4367 return (0); 4368 } 4369 4370 static int 4371 t4_sysctls(struct adapter *sc) 4372 { 4373 struct sysctl_ctx_list *ctx; 4374 struct sysctl_oid *oid; 4375 struct sysctl_oid_list *children, *c0; 4376 static char *caps[] = { 4377 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 4378 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL" /* caps[1] niccaps */ 4379 "\6HASHFILTER\7ETHOFLD", 4380 "\20\1TOE", /* caps[2] toecaps */ 4381 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 4382 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 4383 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 4384 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 4385 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 4386 "\4PO_INITIAOR\5PO_TARGET" 4387 }; 4388 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 4389 4390 ctx = device_get_sysctl_ctx(sc->dev); 4391 4392 /* 4393 * dev.t4nex.X. 4394 */ 4395 oid = device_get_sysctl_tree(sc->dev); 4396 c0 = children = SYSCTL_CHILDREN(oid); 4397 4398 sc->sc_do_rxcopy = 1; 4399 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, 4400 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); 4401 4402 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 4403 sc->params.nports, "# of ports"); 4404 4405 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 4406 NULL, chip_rev(sc), "chip hardware revision"); 4407 4408 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 4409 CTLFLAG_RD, &sc->fw_version, 0, "firmware version"); 4410 4411 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 4412 CTLFLAG_RD, &sc->cfg_file, 0, "configuration file"); 4413 4414 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 4415 sc->cfcsum, "config file checksum"); 4416 4417 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 4418 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 4419 sysctl_bitfield, "A", "available doorbells"); 4420 4421 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 4422 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 4423 sysctl_bitfield, "A", "available link capabilities"); 4424 4425 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 4426 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 4427 sysctl_bitfield, "A", "available NIC capabilities"); 4428 4429 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 4430 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 4431 sysctl_bitfield, "A", "available TCP offload capabilities"); 4432 4433 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 4434 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 4435 sysctl_bitfield, "A", "available RDMA capabilities"); 4436 4437 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 4438 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 4439 sysctl_bitfield, "A", "available iSCSI capabilities"); 4440 4441 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 4442 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 4443 sysctl_bitfield, "A", "available FCoE capabilities"); 4444 4445 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 4446 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 4447 4448 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 4449 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 4450 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 4451 "interrupt holdoff timer values (us)"); 4452 4453 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 4454 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 4455 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 4456 "interrupt holdoff packet counter values"); 4457 4458 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, 4459 NULL, sc->tids.nftids, "number of filters"); 4460 4461 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | 4462 CTLFLAG_RD, sc, 0, sysctl_temperature, "I", 4463 "chip temperature (in Celsius)"); 4464 4465 t4_sge_sysctls(sc, ctx, children); 4466 4467 sc->lro_timeout = 100; 4468 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, 4469 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); 4470 4471 #ifdef SBUF_DRAIN 4472 /* 4473 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 4474 */ 4475 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 4476 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 4477 "logs and miscellaneous information"); 4478 children = SYSCTL_CHILDREN(oid); 4479 4480 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 4481 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4482 sysctl_cctrl, "A", "congestion control"); 4483 4484 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 4485 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4486 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 4487 4488 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 4489 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 4490 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 4491 4492 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 4493 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 4494 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 4495 4496 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 4497 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 4498 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 4499 4500 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 4501 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 4502 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 4503 4504 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 4505 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 4506 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 4507 4508 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 4509 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4510 sysctl_cim_la, "A", "CIM logic analyzer"); 4511 4512 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", 4513 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4514 sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); 4515 4516 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 4517 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 4518 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 4519 4520 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 4521 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 4522 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 4523 4524 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 4525 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 4526 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 4527 4528 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 4529 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 4530 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 4531 4532 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 4533 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 4534 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 4535 4536 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 4537 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 4538 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 4539 4540 if (is_t5(sc)) { 4541 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 4542 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 4543 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 4544 4545 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 4546 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 4547 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 4548 } 4549 4550 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", 4551 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4552 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); 4553 4554 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 4555 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4556 sysctl_cim_qcfg, "A", "CIM queue configuration"); 4557 4558 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 4559 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4560 sysctl_cpl_stats, "A", "CPL statistics"); 4561 4562 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 4563 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4564 sysctl_ddp_stats, "A", "DDP statistics"); 4565 4566 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 4567 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4568 sysctl_devlog, "A", "firmware's device log"); 4569 4570 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 4571 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4572 sysctl_fcoe_stats, "A", "FCoE statistics"); 4573 4574 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 4575 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4576 sysctl_hw_sched, "A", "hardware scheduler "); 4577 4578 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 4579 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4580 sysctl_l2t, "A", "hardware L2 table"); 4581 4582 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 4583 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4584 sysctl_lb_stats, "A", "loopback statistics"); 4585 4586 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 4587 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4588 sysctl_meminfo, "A", "memory regions"); 4589 4590 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", 4591 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4592 sysctl_mps_tcam, "A", "MPS TCAM entries"); 4593 4594 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 4595 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4596 sysctl_path_mtus, "A", "path MTUs"); 4597 4598 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 4599 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4600 sysctl_pm_stats, "A", "PM statistics"); 4601 4602 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 4603 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4604 sysctl_rdma_stats, "A", "RDMA statistics"); 4605 4606 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 4607 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4608 sysctl_tcp_stats, "A", "TCP statistics"); 4609 4610 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 4611 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4612 sysctl_tids, "A", "TID information"); 4613 4614 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 4615 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4616 sysctl_tp_err_stats, "A", "TP error statistics"); 4617 4618 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", 4619 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4620 sysctl_tp_la, "A", "TP logic analyzer"); 4621 4622 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 4623 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4624 sysctl_tx_rate, "A", "Tx rate"); 4625 4626 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", 4627 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4628 sysctl_ulprx_la, "A", "ULPRX logic analyzer"); 4629 4630 if (is_t5(sc)) { 4631 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 4632 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4633 sysctl_wcwr_stats, "A", "write combined work requests"); 4634 } 4635 #endif 4636 4637 #ifdef TCP_OFFLOAD 4638 if (is_offload(sc)) { 4639 /* 4640 * dev.t4nex.X.toe. 4641 */ 4642 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 4643 NULL, "TOE parameters"); 4644 children = SYSCTL_CHILDREN(oid); 4645 4646 sc->tt.sndbuf = 256 * 1024; 4647 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 4648 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 4649 4650 sc->tt.ddp = 0; 4651 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 4652 &sc->tt.ddp, 0, "DDP allowed"); 4653 4654 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 4655 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 4656 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 4657 4658 sc->tt.ddp_thres = 4659 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 4660 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 4661 &sc->tt.ddp_thres, 0, "DDP threshold"); 4662 4663 sc->tt.rx_coalesce = 1; 4664 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", 4665 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); 4666 } 4667 #endif 4668 4669 4670 return (0); 4671 } 4672 4673 static int 4674 cxgbe_sysctls(struct port_info *pi) 4675 { 4676 struct sysctl_ctx_list *ctx; 4677 struct sysctl_oid *oid; 4678 struct sysctl_oid_list *children; 4679 struct adapter *sc = pi->adapter; 4680 4681 ctx = device_get_sysctl_ctx(pi->dev); 4682 4683 /* 4684 * dev.cxgbe.X. 4685 */ 4686 oid = device_get_sysctl_tree(pi->dev); 4687 children = SYSCTL_CHILDREN(oid); 4688 4689 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | 4690 CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); 4691 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { 4692 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 4693 CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I", 4694 "PHY temperature (in Celsius)"); 4695 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", 4696 CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I", 4697 "PHY firmware version"); 4698 } 4699 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 4700 &pi->nrxq, 0, "# of rx queues"); 4701 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 4702 &pi->ntxq, 0, "# of tx queues"); 4703 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 4704 &pi->first_rxq, 0, "index of first rx queue"); 4705 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 4706 &pi->first_txq, 0, "index of first tx queue"); 4707 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT | 4708 CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU", 4709 "Reserve queue 0 for non-flowid packets"); 4710 4711 #ifdef TCP_OFFLOAD 4712 if (is_offload(sc)) { 4713 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 4714 &pi->nofldrxq, 0, 4715 "# of rx queues for offloaded TCP connections"); 4716 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 4717 &pi->nofldtxq, 0, 4718 "# of tx queues for offloaded TCP connections"); 4719 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 4720 CTLFLAG_RD, &pi->first_ofld_rxq, 0, 4721 "index of first TOE rx queue"); 4722 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 4723 CTLFLAG_RD, &pi->first_ofld_txq, 0, 4724 "index of first TOE tx queue"); 4725 } 4726 #endif 4727 #ifdef DEV_NETMAP 4728 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD, 4729 &pi->nnmrxq, 0, "# of rx queues for netmap"); 4730 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD, 4731 &pi->nnmtxq, 0, "# of tx queues for netmap"); 4732 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq", 4733 CTLFLAG_RD, &pi->first_nm_rxq, 0, 4734 "index of first netmap rx queue"); 4735 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq", 4736 CTLFLAG_RD, &pi->first_nm_txq, 0, 4737 "index of first netmap tx queue"); 4738 #endif 4739 4740 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 4741 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 4742 "holdoff timer index"); 4743 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 4744 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 4745 "holdoff packet counter index"); 4746 4747 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 4748 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 4749 "rx queue size"); 4750 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 4751 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 4752 "tx queue size"); 4753 4754 /* 4755 * dev.cxgbe.X.stats. 4756 */ 4757 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 4758 NULL, "port statistics"); 4759 children = SYSCTL_CHILDREN(oid); 4760 4761 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 4762 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 4763 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ 4764 sysctl_handle_t4_reg64, "QU", desc) 4765 4766 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 4767 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 4768 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 4769 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 4770 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 4771 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 4772 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 4773 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 4774 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 4775 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 4776 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 4777 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 4778 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 4779 "# of tx frames in this range", 4780 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 4781 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 4782 "# of tx frames in this range", 4783 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 4784 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 4785 "# of tx frames in this range", 4786 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 4787 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 4788 "# of tx frames in this range", 4789 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 4790 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 4791 "# of tx frames in this range", 4792 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 4793 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 4794 "# of tx frames in this range", 4795 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 4796 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 4797 "# of tx frames in this range", 4798 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 4799 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 4800 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 4801 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 4802 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 4803 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 4804 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 4805 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 4806 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 4807 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 4808 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 4809 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 4810 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 4811 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 4812 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 4813 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 4814 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 4815 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 4816 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 4817 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 4818 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 4819 4820 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 4821 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 4822 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 4823 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 4824 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 4825 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 4826 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 4827 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 4828 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 4829 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 4830 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 4831 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 4832 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 4833 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 4834 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 4835 "# of frames received with bad FCS", 4836 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 4837 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 4838 "# of frames received with length error", 4839 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 4840 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 4841 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 4842 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 4843 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 4844 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 4845 "# of rx frames in this range", 4846 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 4847 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 4848 "# of rx frames in this range", 4849 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 4850 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 4851 "# of rx frames in this range", 4852 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 4853 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 4854 "# of rx frames in this range", 4855 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 4856 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 4857 "# of rx frames in this range", 4858 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 4859 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 4860 "# of rx frames in this range", 4861 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 4862 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 4863 "# of rx frames in this range", 4864 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 4865 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 4866 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 4867 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 4868 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 4869 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 4870 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 4871 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 4872 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 4873 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 4874 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 4875 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 4876 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 4877 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 4878 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 4879 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 4880 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 4881 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 4882 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 4883 4884 #undef SYSCTL_ADD_T4_REG64 4885 4886 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 4887 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 4888 &pi->stats.name, desc) 4889 4890 /* We get these from port_stats and they may be stale by upto 1s */ 4891 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 4892 "# drops due to buffer-group 0 overflows"); 4893 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 4894 "# drops due to buffer-group 1 overflows"); 4895 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 4896 "# drops due to buffer-group 2 overflows"); 4897 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 4898 "# drops due to buffer-group 3 overflows"); 4899 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 4900 "# of buffer-group 0 truncated packets"); 4901 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 4902 "# of buffer-group 1 truncated packets"); 4903 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 4904 "# of buffer-group 2 truncated packets"); 4905 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 4906 "# of buffer-group 3 truncated packets"); 4907 4908 #undef SYSCTL_ADD_T4_PORTSTAT 4909 4910 return (0); 4911 } 4912 4913 static int 4914 sysctl_int_array(SYSCTL_HANDLER_ARGS) 4915 { 4916 int rc, *i; 4917 struct sbuf sb; 4918 4919 sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND); 4920 for (i = arg1; arg2; arg2 -= sizeof(int), i++) 4921 sbuf_printf(&sb, "%d ", *i); 4922 sbuf_trim(&sb); 4923 sbuf_finish(&sb); 4924 rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 4925 sbuf_delete(&sb); 4926 return (rc); 4927 } 4928 4929 static int 4930 sysctl_bitfield(SYSCTL_HANDLER_ARGS) 4931 { 4932 int rc; 4933 struct sbuf *sb; 4934 4935 rc = sysctl_wire_old_buffer(req, 0); 4936 if (rc != 0) 4937 return(rc); 4938 4939 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 4940 if (sb == NULL) 4941 return (ENOMEM); 4942 4943 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 4944 rc = sbuf_finish(sb); 4945 sbuf_delete(sb); 4946 4947 return (rc); 4948 } 4949 4950 static int 4951 sysctl_btphy(SYSCTL_HANDLER_ARGS) 4952 { 4953 struct port_info *pi = arg1; 4954 int op = arg2; 4955 struct adapter *sc = pi->adapter; 4956 u_int v; 4957 int rc; 4958 4959 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt"); 4960 if (rc) 4961 return (rc); 4962 /* XXX: magic numbers */ 4963 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, 4964 &v); 4965 end_synchronized_op(sc, 0); 4966 if (rc) 4967 return (rc); 4968 if (op == 0) 4969 v /= 256; 4970 4971 rc = sysctl_handle_int(oidp, &v, 0, req); 4972 return (rc); 4973 } 4974 4975 static int 4976 sysctl_noflowq(SYSCTL_HANDLER_ARGS) 4977 { 4978 struct port_info *pi = arg1; 4979 int rc, val; 4980 4981 val = pi->rsrv_noflowq; 4982 rc = sysctl_handle_int(oidp, &val, 0, req); 4983 if (rc != 0 || req->newptr == NULL) 4984 return (rc); 4985 4986 if ((val >= 1) && (pi->ntxq > 1)) 4987 pi->rsrv_noflowq = 1; 4988 else 4989 pi->rsrv_noflowq = 0; 4990 4991 return (rc); 4992 } 4993 4994 static int 4995 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 4996 { 4997 struct port_info *pi = arg1; 4998 struct adapter *sc = pi->adapter; 4999 int idx, rc, i; 5000 struct sge_rxq *rxq; 5001 #ifdef TCP_OFFLOAD 5002 struct sge_ofld_rxq *ofld_rxq; 5003 #endif 5004 uint8_t v; 5005 5006 idx = pi->tmr_idx; 5007 5008 rc = sysctl_handle_int(oidp, &idx, 0, req); 5009 if (rc != 0 || req->newptr == NULL) 5010 return (rc); 5011 5012 if (idx < 0 || idx >= SGE_NTIMERS) 5013 return (EINVAL); 5014 5015 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5016 "t4tmr"); 5017 if (rc) 5018 return (rc); 5019 5020 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 5021 for_each_rxq(pi, i, rxq) { 5022 #ifdef atomic_store_rel_8 5023 atomic_store_rel_8(&rxq->iq.intr_params, v); 5024 #else 5025 rxq->iq.intr_params = v; 5026 #endif 5027 } 5028 #ifdef TCP_OFFLOAD 5029 for_each_ofld_rxq(pi, i, ofld_rxq) { 5030 #ifdef atomic_store_rel_8 5031 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); 5032 #else 5033 ofld_rxq->iq.intr_params = v; 5034 #endif 5035 } 5036 #endif 5037 pi->tmr_idx = idx; 5038 5039 end_synchronized_op(sc, LOCK_HELD); 5040 return (0); 5041 } 5042 5043 static int 5044 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 5045 { 5046 struct port_info *pi = arg1; 5047 struct adapter *sc = pi->adapter; 5048 int idx, rc; 5049 5050 idx = pi->pktc_idx; 5051 5052 rc = sysctl_handle_int(oidp, &idx, 0, req); 5053 if (rc != 0 || req->newptr == NULL) 5054 return (rc); 5055 5056 if (idx < -1 || idx >= SGE_NCOUNTERS) 5057 return (EINVAL); 5058 5059 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5060 "t4pktc"); 5061 if (rc) 5062 return (rc); 5063 5064 if (pi->flags & PORT_INIT_DONE) 5065 rc = EBUSY; /* cannot be changed once the queues are created */ 5066 else 5067 pi->pktc_idx = idx; 5068 5069 end_synchronized_op(sc, LOCK_HELD); 5070 return (rc); 5071 } 5072 5073 static int 5074 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 5075 { 5076 struct port_info *pi = arg1; 5077 struct adapter *sc = pi->adapter; 5078 int qsize, rc; 5079 5080 qsize = pi->qsize_rxq; 5081 5082 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5083 if (rc != 0 || req->newptr == NULL) 5084 return (rc); 5085 5086 if (qsize < 128 || (qsize & 7)) 5087 return (EINVAL); 5088 5089 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5090 "t4rxqs"); 5091 if (rc) 5092 return (rc); 5093 5094 if (pi->flags & PORT_INIT_DONE) 5095 rc = EBUSY; /* cannot be changed once the queues are created */ 5096 else 5097 pi->qsize_rxq = qsize; 5098 5099 end_synchronized_op(sc, LOCK_HELD); 5100 return (rc); 5101 } 5102 5103 static int 5104 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 5105 { 5106 struct port_info *pi = arg1; 5107 struct adapter *sc = pi->adapter; 5108 int qsize, rc; 5109 5110 qsize = pi->qsize_txq; 5111 5112 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5113 if (rc != 0 || req->newptr == NULL) 5114 return (rc); 5115 5116 /* bufring size must be powerof2 */ 5117 if (qsize < 128 || !powerof2(qsize)) 5118 return (EINVAL); 5119 5120 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5121 "t4txqs"); 5122 if (rc) 5123 return (rc); 5124 5125 if (pi->flags & PORT_INIT_DONE) 5126 rc = EBUSY; /* cannot be changed once the queues are created */ 5127 else 5128 pi->qsize_txq = qsize; 5129 5130 end_synchronized_op(sc, LOCK_HELD); 5131 return (rc); 5132 } 5133 5134 static int 5135 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 5136 { 5137 struct adapter *sc = arg1; 5138 int reg = arg2; 5139 uint64_t val; 5140 5141 val = t4_read_reg64(sc, reg); 5142 5143 return (sysctl_handle_64(oidp, &val, 0, req)); 5144 } 5145 5146 static int 5147 sysctl_temperature(SYSCTL_HANDLER_ARGS) 5148 { 5149 struct adapter *sc = arg1; 5150 int rc, t; 5151 uint32_t param, val; 5152 5153 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); 5154 if (rc) 5155 return (rc); 5156 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 5157 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 5158 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); 5159 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 5160 end_synchronized_op(sc, 0); 5161 if (rc) 5162 return (rc); 5163 5164 /* unknown is returned as 0 but we display -1 in that case */ 5165 t = val == 0 ? -1 : val; 5166 5167 rc = sysctl_handle_int(oidp, &t, 0, req); 5168 return (rc); 5169 } 5170 5171 #ifdef SBUF_DRAIN 5172 static int 5173 sysctl_cctrl(SYSCTL_HANDLER_ARGS) 5174 { 5175 struct adapter *sc = arg1; 5176 struct sbuf *sb; 5177 int rc, i; 5178 uint16_t incr[NMTUS][NCCTRL_WIN]; 5179 static const char *dec_fac[] = { 5180 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 5181 "0.9375" 5182 }; 5183 5184 rc = sysctl_wire_old_buffer(req, 0); 5185 if (rc != 0) 5186 return (rc); 5187 5188 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5189 if (sb == NULL) 5190 return (ENOMEM); 5191 5192 t4_read_cong_tbl(sc, incr); 5193 5194 for (i = 0; i < NCCTRL_WIN; ++i) { 5195 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 5196 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 5197 incr[5][i], incr[6][i], incr[7][i]); 5198 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 5199 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 5200 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 5201 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 5202 } 5203 5204 rc = sbuf_finish(sb); 5205 sbuf_delete(sb); 5206 5207 return (rc); 5208 } 5209 5210 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 5211 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 5212 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 5213 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 5214 }; 5215 5216 static int 5217 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 5218 { 5219 struct adapter *sc = arg1; 5220 struct sbuf *sb; 5221 int rc, i, n, qid = arg2; 5222 uint32_t *buf, *p; 5223 char *qtype; 5224 u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5; 5225 5226 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 5227 ("%s: bad qid %d\n", __func__, qid)); 5228 5229 if (qid < CIM_NUM_IBQ) { 5230 /* inbound queue */ 5231 qtype = "IBQ"; 5232 n = 4 * CIM_IBQ_SIZE; 5233 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5234 rc = t4_read_cim_ibq(sc, qid, buf, n); 5235 } else { 5236 /* outbound queue */ 5237 qtype = "OBQ"; 5238 qid -= CIM_NUM_IBQ; 5239 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 5240 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5241 rc = t4_read_cim_obq(sc, qid, buf, n); 5242 } 5243 5244 if (rc < 0) { 5245 rc = -rc; 5246 goto done; 5247 } 5248 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 5249 5250 rc = sysctl_wire_old_buffer(req, 0); 5251 if (rc != 0) 5252 goto done; 5253 5254 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5255 if (sb == NULL) { 5256 rc = ENOMEM; 5257 goto done; 5258 } 5259 5260 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 5261 for (i = 0, p = buf; i < n; i += 16, p += 4) 5262 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 5263 p[2], p[3]); 5264 5265 rc = sbuf_finish(sb); 5266 sbuf_delete(sb); 5267 done: 5268 free(buf, M_CXGBE); 5269 return (rc); 5270 } 5271 5272 static int 5273 sysctl_cim_la(SYSCTL_HANDLER_ARGS) 5274 { 5275 struct adapter *sc = arg1; 5276 u_int cfg; 5277 struct sbuf *sb; 5278 uint32_t *buf, *p; 5279 int rc; 5280 5281 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 5282 if (rc != 0) 5283 return (rc); 5284 5285 rc = sysctl_wire_old_buffer(req, 0); 5286 if (rc != 0) 5287 return (rc); 5288 5289 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5290 if (sb == NULL) 5291 return (ENOMEM); 5292 5293 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 5294 M_ZERO | M_WAITOK); 5295 5296 rc = -t4_cim_read_la(sc, buf, NULL); 5297 if (rc != 0) 5298 goto done; 5299 5300 sbuf_printf(sb, "Status Data PC%s", 5301 cfg & F_UPDBGLACAPTPCONLY ? "" : 5302 " LS0Stat LS0Addr LS0Data"); 5303 5304 KASSERT((sc->params.cim_la_size & 7) == 0, 5305 ("%s: p will walk off the end of buf", __func__)); 5306 5307 for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) { 5308 if (cfg & F_UPDBGLACAPTPCONLY) { 5309 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 5310 p[6], p[7]); 5311 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 5312 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 5313 p[4] & 0xff, p[5] >> 8); 5314 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 5315 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5316 p[1] & 0xf, p[2] >> 4); 5317 } else { 5318 sbuf_printf(sb, 5319 "\n %02x %x%07x %x%07x %08x %08x " 5320 "%08x%08x%08x%08x", 5321 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5322 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 5323 p[6], p[7]); 5324 } 5325 } 5326 5327 rc = sbuf_finish(sb); 5328 sbuf_delete(sb); 5329 done: 5330 free(buf, M_CXGBE); 5331 return (rc); 5332 } 5333 5334 static int 5335 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) 5336 { 5337 struct adapter *sc = arg1; 5338 u_int i; 5339 struct sbuf *sb; 5340 uint32_t *buf, *p; 5341 int rc; 5342 5343 rc = sysctl_wire_old_buffer(req, 0); 5344 if (rc != 0) 5345 return (rc); 5346 5347 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5348 if (sb == NULL) 5349 return (ENOMEM); 5350 5351 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, 5352 M_ZERO | M_WAITOK); 5353 5354 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); 5355 p = buf; 5356 5357 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5358 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], 5359 p[1], p[0]); 5360 } 5361 5362 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); 5363 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5364 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", 5365 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, 5366 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, 5367 (p[1] >> 2) | ((p[2] & 3) << 30), 5368 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, 5369 p[0] & 1); 5370 } 5371 5372 rc = sbuf_finish(sb); 5373 sbuf_delete(sb); 5374 free(buf, M_CXGBE); 5375 return (rc); 5376 } 5377 5378 static int 5379 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) 5380 { 5381 struct adapter *sc = arg1; 5382 u_int i; 5383 struct sbuf *sb; 5384 uint32_t *buf, *p; 5385 int rc; 5386 5387 rc = sysctl_wire_old_buffer(req, 0); 5388 if (rc != 0) 5389 return (rc); 5390 5391 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5392 if (sb == NULL) 5393 return (ENOMEM); 5394 5395 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, 5396 M_ZERO | M_WAITOK); 5397 5398 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); 5399 p = buf; 5400 5401 sbuf_printf(sb, "Cntl ID DataBE Addr Data"); 5402 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5403 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", 5404 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, 5405 p[4], p[3], p[2], p[1], p[0]); 5406 } 5407 5408 sbuf_printf(sb, "\n\nCntl ID Data"); 5409 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5410 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", 5411 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); 5412 } 5413 5414 rc = sbuf_finish(sb); 5415 sbuf_delete(sb); 5416 free(buf, M_CXGBE); 5417 return (rc); 5418 } 5419 5420 static int 5421 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 5422 { 5423 struct adapter *sc = arg1; 5424 struct sbuf *sb; 5425 int rc, i; 5426 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5427 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5428 uint16_t thres[CIM_NUM_IBQ]; 5429 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 5430 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 5431 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 5432 5433 if (is_t4(sc)) { 5434 cim_num_obq = CIM_NUM_OBQ; 5435 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 5436 obq_rdaddr = A_UP_OBQ_0_REALADDR; 5437 } else { 5438 cim_num_obq = CIM_NUM_OBQ_T5; 5439 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 5440 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 5441 } 5442 nq = CIM_NUM_IBQ + cim_num_obq; 5443 5444 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 5445 if (rc == 0) 5446 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 5447 if (rc != 0) 5448 return (rc); 5449 5450 t4_read_cimq_cfg(sc, base, size, thres); 5451 5452 rc = sysctl_wire_old_buffer(req, 0); 5453 if (rc != 0) 5454 return (rc); 5455 5456 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5457 if (sb == NULL) 5458 return (ENOMEM); 5459 5460 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 5461 5462 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 5463 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 5464 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 5465 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5466 G_QUEREMFLITS(p[2]) * 16); 5467 for ( ; i < nq; i++, p += 4, wr += 2) 5468 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 5469 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 5470 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5471 G_QUEREMFLITS(p[2]) * 16); 5472 5473 rc = sbuf_finish(sb); 5474 sbuf_delete(sb); 5475 5476 return (rc); 5477 } 5478 5479 static int 5480 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 5481 { 5482 struct adapter *sc = arg1; 5483 struct sbuf *sb; 5484 int rc; 5485 struct tp_cpl_stats stats; 5486 5487 rc = sysctl_wire_old_buffer(req, 0); 5488 if (rc != 0) 5489 return (rc); 5490 5491 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5492 if (sb == NULL) 5493 return (ENOMEM); 5494 5495 t4_tp_get_cpl_stats(sc, &stats); 5496 5497 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 5498 "channel 3\n"); 5499 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 5500 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 5501 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 5502 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 5503 5504 rc = sbuf_finish(sb); 5505 sbuf_delete(sb); 5506 5507 return (rc); 5508 } 5509 5510 static int 5511 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 5512 { 5513 struct adapter *sc = arg1; 5514 struct sbuf *sb; 5515 int rc; 5516 struct tp_usm_stats stats; 5517 5518 rc = sysctl_wire_old_buffer(req, 0); 5519 if (rc != 0) 5520 return(rc); 5521 5522 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5523 if (sb == NULL) 5524 return (ENOMEM); 5525 5526 t4_get_usm_stats(sc, &stats); 5527 5528 sbuf_printf(sb, "Frames: %u\n", stats.frames); 5529 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 5530 sbuf_printf(sb, "Drops: %u", stats.drops); 5531 5532 rc = sbuf_finish(sb); 5533 sbuf_delete(sb); 5534 5535 return (rc); 5536 } 5537 5538 const char *devlog_level_strings[] = { 5539 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 5540 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 5541 [FW_DEVLOG_LEVEL_ERR] = "ERR", 5542 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 5543 [FW_DEVLOG_LEVEL_INFO] = "INFO", 5544 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 5545 }; 5546 5547 const char *devlog_facility_strings[] = { 5548 [FW_DEVLOG_FACILITY_CORE] = "CORE", 5549 [FW_DEVLOG_FACILITY_CF] = "CF", 5550 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 5551 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 5552 [FW_DEVLOG_FACILITY_RES] = "RES", 5553 [FW_DEVLOG_FACILITY_HW] = "HW", 5554 [FW_DEVLOG_FACILITY_FLR] = "FLR", 5555 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 5556 [FW_DEVLOG_FACILITY_PHY] = "PHY", 5557 [FW_DEVLOG_FACILITY_MAC] = "MAC", 5558 [FW_DEVLOG_FACILITY_PORT] = "PORT", 5559 [FW_DEVLOG_FACILITY_VI] = "VI", 5560 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 5561 [FW_DEVLOG_FACILITY_ACL] = "ACL", 5562 [FW_DEVLOG_FACILITY_TM] = "TM", 5563 [FW_DEVLOG_FACILITY_QFC] = "QFC", 5564 [FW_DEVLOG_FACILITY_DCB] = "DCB", 5565 [FW_DEVLOG_FACILITY_ETH] = "ETH", 5566 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 5567 [FW_DEVLOG_FACILITY_RI] = "RI", 5568 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 5569 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 5570 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 5571 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 5572 }; 5573 5574 static int 5575 sysctl_devlog(SYSCTL_HANDLER_ARGS) 5576 { 5577 struct adapter *sc = arg1; 5578 struct devlog_params *dparams = &sc->params.devlog; 5579 struct fw_devlog_e *buf, *e; 5580 int i, j, rc, nentries, first = 0, m; 5581 struct sbuf *sb; 5582 uint64_t ftstamp = UINT64_MAX; 5583 5584 if (dparams->start == 0) { 5585 dparams->memtype = FW_MEMTYPE_EDC0; 5586 dparams->start = 0x84000; 5587 dparams->size = 32768; 5588 } 5589 5590 nentries = dparams->size / sizeof(struct fw_devlog_e); 5591 5592 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 5593 if (buf == NULL) 5594 return (ENOMEM); 5595 5596 m = fwmtype_to_hwmtype(dparams->memtype); 5597 rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf); 5598 if (rc != 0) 5599 goto done; 5600 5601 for (i = 0; i < nentries; i++) { 5602 e = &buf[i]; 5603 5604 if (e->timestamp == 0) 5605 break; /* end */ 5606 5607 e->timestamp = be64toh(e->timestamp); 5608 e->seqno = be32toh(e->seqno); 5609 for (j = 0; j < 8; j++) 5610 e->params[j] = be32toh(e->params[j]); 5611 5612 if (e->timestamp < ftstamp) { 5613 ftstamp = e->timestamp; 5614 first = i; 5615 } 5616 } 5617 5618 if (buf[first].timestamp == 0) 5619 goto done; /* nothing in the log */ 5620 5621 rc = sysctl_wire_old_buffer(req, 0); 5622 if (rc != 0) 5623 goto done; 5624 5625 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5626 if (sb == NULL) { 5627 rc = ENOMEM; 5628 goto done; 5629 } 5630 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 5631 "Seq#", "Tstamp", "Level", "Facility", "Message"); 5632 5633 i = first; 5634 do { 5635 e = &buf[i]; 5636 if (e->timestamp == 0) 5637 break; /* end */ 5638 5639 sbuf_printf(sb, "%10d %15ju %8s %8s ", 5640 e->seqno, e->timestamp, 5641 (e->level < nitems(devlog_level_strings) ? 5642 devlog_level_strings[e->level] : "UNKNOWN"), 5643 (e->facility < nitems(devlog_facility_strings) ? 5644 devlog_facility_strings[e->facility] : "UNKNOWN")); 5645 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 5646 e->params[2], e->params[3], e->params[4], 5647 e->params[5], e->params[6], e->params[7]); 5648 5649 if (++i == nentries) 5650 i = 0; 5651 } while (i != first); 5652 5653 rc = sbuf_finish(sb); 5654 sbuf_delete(sb); 5655 done: 5656 free(buf, M_CXGBE); 5657 return (rc); 5658 } 5659 5660 static int 5661 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 5662 { 5663 struct adapter *sc = arg1; 5664 struct sbuf *sb; 5665 int rc; 5666 struct tp_fcoe_stats stats[4]; 5667 5668 rc = sysctl_wire_old_buffer(req, 0); 5669 if (rc != 0) 5670 return (rc); 5671 5672 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5673 if (sb == NULL) 5674 return (ENOMEM); 5675 5676 t4_get_fcoe_stats(sc, 0, &stats[0]); 5677 t4_get_fcoe_stats(sc, 1, &stats[1]); 5678 t4_get_fcoe_stats(sc, 2, &stats[2]); 5679 t4_get_fcoe_stats(sc, 3, &stats[3]); 5680 5681 sbuf_printf(sb, " channel 0 channel 1 " 5682 "channel 2 channel 3\n"); 5683 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 5684 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 5685 stats[3].octetsDDP); 5686 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 5687 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 5688 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 5689 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 5690 stats[3].framesDrop); 5691 5692 rc = sbuf_finish(sb); 5693 sbuf_delete(sb); 5694 5695 return (rc); 5696 } 5697 5698 static int 5699 sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 5700 { 5701 struct adapter *sc = arg1; 5702 struct sbuf *sb; 5703 int rc, i; 5704 unsigned int map, kbps, ipg, mode; 5705 unsigned int pace_tab[NTX_SCHED]; 5706 5707 rc = sysctl_wire_old_buffer(req, 0); 5708 if (rc != 0) 5709 return (rc); 5710 5711 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5712 if (sb == NULL) 5713 return (ENOMEM); 5714 5715 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 5716 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 5717 t4_read_pace_tbl(sc, pace_tab); 5718 5719 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 5720 "Class IPG (0.1 ns) Flow IPG (us)"); 5721 5722 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 5723 t4_get_tx_sched(sc, i, &kbps, &ipg); 5724 sbuf_printf(sb, "\n %u %-5s %u ", i, 5725 (mode & (1 << i)) ? "flow" : "class", map & 3); 5726 if (kbps) 5727 sbuf_printf(sb, "%9u ", kbps); 5728 else 5729 sbuf_printf(sb, " disabled "); 5730 5731 if (ipg) 5732 sbuf_printf(sb, "%13u ", ipg); 5733 else 5734 sbuf_printf(sb, " disabled "); 5735 5736 if (pace_tab[i]) 5737 sbuf_printf(sb, "%10u", pace_tab[i]); 5738 else 5739 sbuf_printf(sb, " disabled"); 5740 } 5741 5742 rc = sbuf_finish(sb); 5743 sbuf_delete(sb); 5744 5745 return (rc); 5746 } 5747 5748 static int 5749 sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 5750 { 5751 struct adapter *sc = arg1; 5752 struct sbuf *sb; 5753 int rc, i, j; 5754 uint64_t *p0, *p1; 5755 struct lb_port_stats s[2]; 5756 static const char *stat_name[] = { 5757 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 5758 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 5759 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 5760 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 5761 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 5762 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 5763 "BG2FramesTrunc:", "BG3FramesTrunc:" 5764 }; 5765 5766 rc = sysctl_wire_old_buffer(req, 0); 5767 if (rc != 0) 5768 return (rc); 5769 5770 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5771 if (sb == NULL) 5772 return (ENOMEM); 5773 5774 memset(s, 0, sizeof(s)); 5775 5776 for (i = 0; i < 4; i += 2) { 5777 t4_get_lb_stats(sc, i, &s[0]); 5778 t4_get_lb_stats(sc, i + 1, &s[1]); 5779 5780 p0 = &s[0].octets; 5781 p1 = &s[1].octets; 5782 sbuf_printf(sb, "%s Loopback %u" 5783 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 5784 5785 for (j = 0; j < nitems(stat_name); j++) 5786 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 5787 *p0++, *p1++); 5788 } 5789 5790 rc = sbuf_finish(sb); 5791 sbuf_delete(sb); 5792 5793 return (rc); 5794 } 5795 5796 static int 5797 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) 5798 { 5799 int rc = 0; 5800 struct port_info *pi = arg1; 5801 struct sbuf *sb; 5802 static const char *linkdnreasons[] = { 5803 "non-specific", "remote fault", "autoneg failed", "reserved3", 5804 "PHY overheated", "unknown", "rx los", "reserved7" 5805 }; 5806 5807 rc = sysctl_wire_old_buffer(req, 0); 5808 if (rc != 0) 5809 return(rc); 5810 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); 5811 if (sb == NULL) 5812 return (ENOMEM); 5813 5814 if (pi->linkdnrc < 0) 5815 sbuf_printf(sb, "n/a"); 5816 else if (pi->linkdnrc < nitems(linkdnreasons)) 5817 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]); 5818 else 5819 sbuf_printf(sb, "%d", pi->linkdnrc); 5820 5821 rc = sbuf_finish(sb); 5822 sbuf_delete(sb); 5823 5824 return (rc); 5825 } 5826 5827 struct mem_desc { 5828 unsigned int base; 5829 unsigned int limit; 5830 unsigned int idx; 5831 }; 5832 5833 static int 5834 mem_desc_cmp(const void *a, const void *b) 5835 { 5836 return ((const struct mem_desc *)a)->base - 5837 ((const struct mem_desc *)b)->base; 5838 } 5839 5840 static void 5841 mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 5842 unsigned int to) 5843 { 5844 unsigned int size; 5845 5846 size = to - from + 1; 5847 if (size == 0) 5848 return; 5849 5850 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 5851 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 5852 } 5853 5854 static int 5855 sysctl_meminfo(SYSCTL_HANDLER_ARGS) 5856 { 5857 struct adapter *sc = arg1; 5858 struct sbuf *sb; 5859 int rc, i, n; 5860 uint32_t lo, hi, used, alloc; 5861 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 5862 static const char *region[] = { 5863 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 5864 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 5865 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 5866 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 5867 "RQUDP region:", "PBL region:", "TXPBL region:", 5868 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 5869 "On-chip queues:" 5870 }; 5871 struct mem_desc avail[4]; 5872 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 5873 struct mem_desc *md = mem; 5874 5875 rc = sysctl_wire_old_buffer(req, 0); 5876 if (rc != 0) 5877 return (rc); 5878 5879 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5880 if (sb == NULL) 5881 return (ENOMEM); 5882 5883 for (i = 0; i < nitems(mem); i++) { 5884 mem[i].limit = 0; 5885 mem[i].idx = i; 5886 } 5887 5888 /* Find and sort the populated memory ranges */ 5889 i = 0; 5890 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 5891 if (lo & F_EDRAM0_ENABLE) { 5892 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 5893 avail[i].base = G_EDRAM0_BASE(hi) << 20; 5894 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 5895 avail[i].idx = 0; 5896 i++; 5897 } 5898 if (lo & F_EDRAM1_ENABLE) { 5899 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 5900 avail[i].base = G_EDRAM1_BASE(hi) << 20; 5901 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 5902 avail[i].idx = 1; 5903 i++; 5904 } 5905 if (lo & F_EXT_MEM_ENABLE) { 5906 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 5907 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 5908 avail[i].limit = avail[i].base + 5909 (G_EXT_MEM_SIZE(hi) << 20); 5910 avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */ 5911 i++; 5912 } 5913 if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) { 5914 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 5915 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 5916 avail[i].limit = avail[i].base + 5917 (G_EXT_MEM1_SIZE(hi) << 20); 5918 avail[i].idx = 4; 5919 i++; 5920 } 5921 if (!i) /* no memory available */ 5922 return 0; 5923 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 5924 5925 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 5926 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 5927 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 5928 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 5929 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 5930 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 5931 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 5932 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 5933 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 5934 5935 /* the next few have explicit upper bounds */ 5936 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 5937 md->limit = md->base - 1 + 5938 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 5939 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 5940 md++; 5941 5942 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 5943 md->limit = md->base - 1 + 5944 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 5945 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 5946 md++; 5947 5948 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 5949 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 5950 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 5951 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 5952 } else { 5953 md->base = 0; 5954 md->idx = nitems(region); /* hide it */ 5955 } 5956 md++; 5957 5958 #define ulp_region(reg) \ 5959 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 5960 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 5961 5962 ulp_region(RX_ISCSI); 5963 ulp_region(RX_TDDP); 5964 ulp_region(TX_TPT); 5965 ulp_region(RX_STAG); 5966 ulp_region(RX_RQ); 5967 ulp_region(RX_RQUDP); 5968 ulp_region(RX_PBL); 5969 ulp_region(TX_PBL); 5970 #undef ulp_region 5971 5972 md->base = 0; 5973 md->idx = nitems(region); 5974 if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) { 5975 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); 5976 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc, 5977 A_SGE_DBVFIFO_SIZE))) << 2) - 1; 5978 } 5979 md++; 5980 5981 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 5982 md->limit = md->base + sc->tids.ntids - 1; 5983 md++; 5984 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 5985 md->limit = md->base + sc->tids.ntids - 1; 5986 md++; 5987 5988 md->base = sc->vres.ocq.start; 5989 if (sc->vres.ocq.size) 5990 md->limit = md->base + sc->vres.ocq.size - 1; 5991 else 5992 md->idx = nitems(region); /* hide it */ 5993 md++; 5994 5995 /* add any address-space holes, there can be up to 3 */ 5996 for (n = 0; n < i - 1; n++) 5997 if (avail[n].limit < avail[n + 1].base) 5998 (md++)->base = avail[n].limit; 5999 if (avail[n].limit) 6000 (md++)->base = avail[n].limit; 6001 6002 n = md - mem; 6003 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 6004 6005 for (lo = 0; lo < i; lo++) 6006 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 6007 avail[lo].limit - 1); 6008 6009 sbuf_printf(sb, "\n"); 6010 for (i = 0; i < n; i++) { 6011 if (mem[i].idx >= nitems(region)) 6012 continue; /* skip holes */ 6013 if (!mem[i].limit) 6014 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 6015 mem_region_show(sb, region[mem[i].idx], mem[i].base, 6016 mem[i].limit); 6017 } 6018 6019 sbuf_printf(sb, "\n"); 6020 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 6021 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 6022 mem_region_show(sb, "uP RAM:", lo, hi); 6023 6024 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 6025 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 6026 mem_region_show(sb, "uP Extmem2:", lo, hi); 6027 6028 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 6029 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 6030 G_PMRXMAXPAGE(lo), 6031 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 6032 (lo & F_PMRXNUMCHN) ? 2 : 1); 6033 6034 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 6035 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 6036 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 6037 G_PMTXMAXPAGE(lo), 6038 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 6039 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 6040 sbuf_printf(sb, "%u p-structs\n", 6041 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 6042 6043 for (i = 0; i < 4; i++) { 6044 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 6045 if (is_t4(sc)) { 6046 used = G_USED(lo); 6047 alloc = G_ALLOC(lo); 6048 } else { 6049 used = G_T5_USED(lo); 6050 alloc = G_T5_ALLOC(lo); 6051 } 6052 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 6053 i, used, alloc); 6054 } 6055 for (i = 0; i < 4; i++) { 6056 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 6057 if (is_t4(sc)) { 6058 used = G_USED(lo); 6059 alloc = G_ALLOC(lo); 6060 } else { 6061 used = G_T5_USED(lo); 6062 alloc = G_T5_ALLOC(lo); 6063 } 6064 sbuf_printf(sb, 6065 "\nLoopback %d using %u pages out of %u allocated", 6066 i, used, alloc); 6067 } 6068 6069 rc = sbuf_finish(sb); 6070 sbuf_delete(sb); 6071 6072 return (rc); 6073 } 6074 6075 static inline void 6076 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) 6077 { 6078 *mask = x | y; 6079 y = htobe64(y); 6080 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); 6081 } 6082 6083 static int 6084 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) 6085 { 6086 struct adapter *sc = arg1; 6087 struct sbuf *sb; 6088 int rc, i, n; 6089 6090 rc = sysctl_wire_old_buffer(req, 0); 6091 if (rc != 0) 6092 return (rc); 6093 6094 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6095 if (sb == NULL) 6096 return (ENOMEM); 6097 6098 sbuf_printf(sb, 6099 "Idx Ethernet address Mask Vld Ports PF" 6100 " VF Replication P0 P1 P2 P3 ML"); 6101 n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES : 6102 NUM_MPS_T5_CLS_SRAM_L_INSTANCES; 6103 for (i = 0; i < n; i++) { 6104 uint64_t tcamx, tcamy, mask; 6105 uint32_t cls_lo, cls_hi; 6106 uint8_t addr[ETHER_ADDR_LEN]; 6107 6108 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); 6109 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); 6110 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 6111 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 6112 6113 if (tcamx & tcamy) 6114 continue; 6115 6116 tcamxy2valmask(tcamx, tcamy, addr, &mask); 6117 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" 6118 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], 6119 addr[3], addr[4], addr[5], (uintmax_t)mask, 6120 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', 6121 G_PORTMAP(cls_hi), G_PF(cls_lo), 6122 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); 6123 6124 if (cls_lo & F_REPLICATE) { 6125 struct fw_ldst_cmd ldst_cmd; 6126 6127 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 6128 ldst_cmd.op_to_addrspace = 6129 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 6130 F_FW_CMD_REQUEST | F_FW_CMD_READ | 6131 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 6132 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 6133 ldst_cmd.u.mps.fid_ctl = 6134 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 6135 V_FW_LDST_CMD_CTL(i)); 6136 6137 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 6138 "t4mps"); 6139 if (rc) 6140 break; 6141 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 6142 sizeof(ldst_cmd), &ldst_cmd); 6143 end_synchronized_op(sc, 0); 6144 6145 if (rc != 0) { 6146 sbuf_printf(sb, 6147 " ------------ error %3u ------------", rc); 6148 rc = 0; 6149 } else { 6150 sbuf_printf(sb, " %08x %08x %08x %08x", 6151 be32toh(ldst_cmd.u.mps.rplc127_96), 6152 be32toh(ldst_cmd.u.mps.rplc95_64), 6153 be32toh(ldst_cmd.u.mps.rplc63_32), 6154 be32toh(ldst_cmd.u.mps.rplc31_0)); 6155 } 6156 } else 6157 sbuf_printf(sb, "%36s", ""); 6158 6159 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), 6160 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), 6161 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); 6162 } 6163 6164 if (rc) 6165 (void) sbuf_finish(sb); 6166 else 6167 rc = sbuf_finish(sb); 6168 sbuf_delete(sb); 6169 6170 return (rc); 6171 } 6172 6173 static int 6174 sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 6175 { 6176 struct adapter *sc = arg1; 6177 struct sbuf *sb; 6178 int rc; 6179 uint16_t mtus[NMTUS]; 6180 6181 rc = sysctl_wire_old_buffer(req, 0); 6182 if (rc != 0) 6183 return (rc); 6184 6185 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6186 if (sb == NULL) 6187 return (ENOMEM); 6188 6189 t4_read_mtu_tbl(sc, mtus, NULL); 6190 6191 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 6192 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 6193 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 6194 mtus[14], mtus[15]); 6195 6196 rc = sbuf_finish(sb); 6197 sbuf_delete(sb); 6198 6199 return (rc); 6200 } 6201 6202 static int 6203 sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 6204 { 6205 struct adapter *sc = arg1; 6206 struct sbuf *sb; 6207 int rc, i; 6208 uint32_t cnt[PM_NSTATS]; 6209 uint64_t cyc[PM_NSTATS]; 6210 static const char *rx_stats[] = { 6211 "Read:", "Write bypass:", "Write mem:", "Flush:" 6212 }; 6213 static const char *tx_stats[] = { 6214 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:" 6215 }; 6216 6217 rc = sysctl_wire_old_buffer(req, 0); 6218 if (rc != 0) 6219 return (rc); 6220 6221 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6222 if (sb == NULL) 6223 return (ENOMEM); 6224 6225 t4_pmtx_get_stats(sc, cnt, cyc); 6226 sbuf_printf(sb, " Tx pcmds Tx bytes"); 6227 for (i = 0; i < ARRAY_SIZE(tx_stats); i++) 6228 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i], 6229 cyc[i]); 6230 6231 t4_pmrx_get_stats(sc, cnt, cyc); 6232 sbuf_printf(sb, "\n Rx pcmds Rx bytes"); 6233 for (i = 0; i < ARRAY_SIZE(rx_stats); i++) 6234 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i], 6235 cyc[i]); 6236 6237 rc = sbuf_finish(sb); 6238 sbuf_delete(sb); 6239 6240 return (rc); 6241 } 6242 6243 static int 6244 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 6245 { 6246 struct adapter *sc = arg1; 6247 struct sbuf *sb; 6248 int rc; 6249 struct tp_rdma_stats stats; 6250 6251 rc = sysctl_wire_old_buffer(req, 0); 6252 if (rc != 0) 6253 return (rc); 6254 6255 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6256 if (sb == NULL) 6257 return (ENOMEM); 6258 6259 t4_tp_get_rdma_stats(sc, &stats); 6260 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 6261 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 6262 6263 rc = sbuf_finish(sb); 6264 sbuf_delete(sb); 6265 6266 return (rc); 6267 } 6268 6269 static int 6270 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 6271 { 6272 struct adapter *sc = arg1; 6273 struct sbuf *sb; 6274 int rc; 6275 struct tp_tcp_stats v4, v6; 6276 6277 rc = sysctl_wire_old_buffer(req, 0); 6278 if (rc != 0) 6279 return (rc); 6280 6281 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6282 if (sb == NULL) 6283 return (ENOMEM); 6284 6285 t4_tp_get_tcp_stats(sc, &v4, &v6); 6286 sbuf_printf(sb, 6287 " IP IPv6\n"); 6288 sbuf_printf(sb, "OutRsts: %20u %20u\n", 6289 v4.tcpOutRsts, v6.tcpOutRsts); 6290 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 6291 v4.tcpInSegs, v6.tcpInSegs); 6292 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 6293 v4.tcpOutSegs, v6.tcpOutSegs); 6294 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 6295 v4.tcpRetransSegs, v6.tcpRetransSegs); 6296 6297 rc = sbuf_finish(sb); 6298 sbuf_delete(sb); 6299 6300 return (rc); 6301 } 6302 6303 static int 6304 sysctl_tids(SYSCTL_HANDLER_ARGS) 6305 { 6306 struct adapter *sc = arg1; 6307 struct sbuf *sb; 6308 int rc; 6309 struct tid_info *t = &sc->tids; 6310 6311 rc = sysctl_wire_old_buffer(req, 0); 6312 if (rc != 0) 6313 return (rc); 6314 6315 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6316 if (sb == NULL) 6317 return (ENOMEM); 6318 6319 if (t->natids) { 6320 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 6321 t->atids_in_use); 6322 } 6323 6324 if (t->ntids) { 6325 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6326 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 6327 6328 if (b) { 6329 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 6330 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6331 t->ntids - 1); 6332 } else { 6333 sbuf_printf(sb, "TID range: %u-%u", 6334 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6335 t->ntids - 1); 6336 } 6337 } else 6338 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 6339 sbuf_printf(sb, ", in use: %u\n", 6340 atomic_load_acq_int(&t->tids_in_use)); 6341 } 6342 6343 if (t->nstids) { 6344 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 6345 t->stid_base + t->nstids - 1, t->stids_in_use); 6346 } 6347 6348 if (t->nftids) { 6349 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 6350 t->ftid_base + t->nftids - 1); 6351 } 6352 6353 if (t->netids) { 6354 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base, 6355 t->etid_base + t->netids - 1); 6356 } 6357 6358 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 6359 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 6360 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 6361 6362 rc = sbuf_finish(sb); 6363 sbuf_delete(sb); 6364 6365 return (rc); 6366 } 6367 6368 static int 6369 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 6370 { 6371 struct adapter *sc = arg1; 6372 struct sbuf *sb; 6373 int rc; 6374 struct tp_err_stats stats; 6375 6376 rc = sysctl_wire_old_buffer(req, 0); 6377 if (rc != 0) 6378 return (rc); 6379 6380 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6381 if (sb == NULL) 6382 return (ENOMEM); 6383 6384 t4_tp_get_err_stats(sc, &stats); 6385 6386 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6387 "channel 3\n"); 6388 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 6389 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 6390 stats.macInErrs[3]); 6391 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 6392 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 6393 stats.hdrInErrs[3]); 6394 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 6395 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 6396 stats.tcpInErrs[3]); 6397 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 6398 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 6399 stats.tcp6InErrs[3]); 6400 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 6401 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 6402 stats.tnlCongDrops[3]); 6403 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 6404 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 6405 stats.tnlTxDrops[3]); 6406 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 6407 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 6408 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 6409 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 6410 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 6411 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 6412 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 6413 stats.ofldNoNeigh, stats.ofldCongDefer); 6414 6415 rc = sbuf_finish(sb); 6416 sbuf_delete(sb); 6417 6418 return (rc); 6419 } 6420 6421 struct field_desc { 6422 const char *name; 6423 u_int start; 6424 u_int width; 6425 }; 6426 6427 static void 6428 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) 6429 { 6430 char buf[32]; 6431 int line_size = 0; 6432 6433 while (f->name) { 6434 uint64_t mask = (1ULL << f->width) - 1; 6435 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, 6436 ((uintmax_t)v >> f->start) & mask); 6437 6438 if (line_size + len >= 79) { 6439 line_size = 8; 6440 sbuf_printf(sb, "\n "); 6441 } 6442 sbuf_printf(sb, "%s ", buf); 6443 line_size += len + 1; 6444 f++; 6445 } 6446 sbuf_printf(sb, "\n"); 6447 } 6448 6449 static struct field_desc tp_la0[] = { 6450 { "RcfOpCodeOut", 60, 4 }, 6451 { "State", 56, 4 }, 6452 { "WcfState", 52, 4 }, 6453 { "RcfOpcSrcOut", 50, 2 }, 6454 { "CRxError", 49, 1 }, 6455 { "ERxError", 48, 1 }, 6456 { "SanityFailed", 47, 1 }, 6457 { "SpuriousMsg", 46, 1 }, 6458 { "FlushInputMsg", 45, 1 }, 6459 { "FlushInputCpl", 44, 1 }, 6460 { "RssUpBit", 43, 1 }, 6461 { "RssFilterHit", 42, 1 }, 6462 { "Tid", 32, 10 }, 6463 { "InitTcb", 31, 1 }, 6464 { "LineNumber", 24, 7 }, 6465 { "Emsg", 23, 1 }, 6466 { "EdataOut", 22, 1 }, 6467 { "Cmsg", 21, 1 }, 6468 { "CdataOut", 20, 1 }, 6469 { "EreadPdu", 19, 1 }, 6470 { "CreadPdu", 18, 1 }, 6471 { "TunnelPkt", 17, 1 }, 6472 { "RcfPeerFin", 16, 1 }, 6473 { "RcfReasonOut", 12, 4 }, 6474 { "TxCchannel", 10, 2 }, 6475 { "RcfTxChannel", 8, 2 }, 6476 { "RxEchannel", 6, 2 }, 6477 { "RcfRxChannel", 5, 1 }, 6478 { "RcfDataOutSrdy", 4, 1 }, 6479 { "RxDvld", 3, 1 }, 6480 { "RxOoDvld", 2, 1 }, 6481 { "RxCongestion", 1, 1 }, 6482 { "TxCongestion", 0, 1 }, 6483 { NULL } 6484 }; 6485 6486 static struct field_desc tp_la1[] = { 6487 { "CplCmdIn", 56, 8 }, 6488 { "CplCmdOut", 48, 8 }, 6489 { "ESynOut", 47, 1 }, 6490 { "EAckOut", 46, 1 }, 6491 { "EFinOut", 45, 1 }, 6492 { "ERstOut", 44, 1 }, 6493 { "SynIn", 43, 1 }, 6494 { "AckIn", 42, 1 }, 6495 { "FinIn", 41, 1 }, 6496 { "RstIn", 40, 1 }, 6497 { "DataIn", 39, 1 }, 6498 { "DataInVld", 38, 1 }, 6499 { "PadIn", 37, 1 }, 6500 { "RxBufEmpty", 36, 1 }, 6501 { "RxDdp", 35, 1 }, 6502 { "RxFbCongestion", 34, 1 }, 6503 { "TxFbCongestion", 33, 1 }, 6504 { "TxPktSumSrdy", 32, 1 }, 6505 { "RcfUlpType", 28, 4 }, 6506 { "Eread", 27, 1 }, 6507 { "Ebypass", 26, 1 }, 6508 { "Esave", 25, 1 }, 6509 { "Static0", 24, 1 }, 6510 { "Cread", 23, 1 }, 6511 { "Cbypass", 22, 1 }, 6512 { "Csave", 21, 1 }, 6513 { "CPktOut", 20, 1 }, 6514 { "RxPagePoolFull", 18, 2 }, 6515 { "RxLpbkPkt", 17, 1 }, 6516 { "TxLpbkPkt", 16, 1 }, 6517 { "RxVfValid", 15, 1 }, 6518 { "SynLearned", 14, 1 }, 6519 { "SetDelEntry", 13, 1 }, 6520 { "SetInvEntry", 12, 1 }, 6521 { "CpcmdDvld", 11, 1 }, 6522 { "CpcmdSave", 10, 1 }, 6523 { "RxPstructsFull", 8, 2 }, 6524 { "EpcmdDvld", 7, 1 }, 6525 { "EpcmdFlush", 6, 1 }, 6526 { "EpcmdTrimPrefix", 5, 1 }, 6527 { "EpcmdTrimPostfix", 4, 1 }, 6528 { "ERssIp4Pkt", 3, 1 }, 6529 { "ERssIp6Pkt", 2, 1 }, 6530 { "ERssTcpUdpPkt", 1, 1 }, 6531 { "ERssFceFipPkt", 0, 1 }, 6532 { NULL } 6533 }; 6534 6535 static struct field_desc tp_la2[] = { 6536 { "CplCmdIn", 56, 8 }, 6537 { "MpsVfVld", 55, 1 }, 6538 { "MpsPf", 52, 3 }, 6539 { "MpsVf", 44, 8 }, 6540 { "SynIn", 43, 1 }, 6541 { "AckIn", 42, 1 }, 6542 { "FinIn", 41, 1 }, 6543 { "RstIn", 40, 1 }, 6544 { "DataIn", 39, 1 }, 6545 { "DataInVld", 38, 1 }, 6546 { "PadIn", 37, 1 }, 6547 { "RxBufEmpty", 36, 1 }, 6548 { "RxDdp", 35, 1 }, 6549 { "RxFbCongestion", 34, 1 }, 6550 { "TxFbCongestion", 33, 1 }, 6551 { "TxPktSumSrdy", 32, 1 }, 6552 { "RcfUlpType", 28, 4 }, 6553 { "Eread", 27, 1 }, 6554 { "Ebypass", 26, 1 }, 6555 { "Esave", 25, 1 }, 6556 { "Static0", 24, 1 }, 6557 { "Cread", 23, 1 }, 6558 { "Cbypass", 22, 1 }, 6559 { "Csave", 21, 1 }, 6560 { "CPktOut", 20, 1 }, 6561 { "RxPagePoolFull", 18, 2 }, 6562 { "RxLpbkPkt", 17, 1 }, 6563 { "TxLpbkPkt", 16, 1 }, 6564 { "RxVfValid", 15, 1 }, 6565 { "SynLearned", 14, 1 }, 6566 { "SetDelEntry", 13, 1 }, 6567 { "SetInvEntry", 12, 1 }, 6568 { "CpcmdDvld", 11, 1 }, 6569 { "CpcmdSave", 10, 1 }, 6570 { "RxPstructsFull", 8, 2 }, 6571 { "EpcmdDvld", 7, 1 }, 6572 { "EpcmdFlush", 6, 1 }, 6573 { "EpcmdTrimPrefix", 5, 1 }, 6574 { "EpcmdTrimPostfix", 4, 1 }, 6575 { "ERssIp4Pkt", 3, 1 }, 6576 { "ERssIp6Pkt", 2, 1 }, 6577 { "ERssTcpUdpPkt", 1, 1 }, 6578 { "ERssFceFipPkt", 0, 1 }, 6579 { NULL } 6580 }; 6581 6582 static void 6583 tp_la_show(struct sbuf *sb, uint64_t *p, int idx) 6584 { 6585 6586 field_desc_show(sb, *p, tp_la0); 6587 } 6588 6589 static void 6590 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) 6591 { 6592 6593 if (idx) 6594 sbuf_printf(sb, "\n"); 6595 field_desc_show(sb, p[0], tp_la0); 6596 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6597 field_desc_show(sb, p[1], tp_la0); 6598 } 6599 6600 static void 6601 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) 6602 { 6603 6604 if (idx) 6605 sbuf_printf(sb, "\n"); 6606 field_desc_show(sb, p[0], tp_la0); 6607 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6608 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); 6609 } 6610 6611 static int 6612 sysctl_tp_la(SYSCTL_HANDLER_ARGS) 6613 { 6614 struct adapter *sc = arg1; 6615 struct sbuf *sb; 6616 uint64_t *buf, *p; 6617 int rc; 6618 u_int i, inc; 6619 void (*show_func)(struct sbuf *, uint64_t *, int); 6620 6621 rc = sysctl_wire_old_buffer(req, 0); 6622 if (rc != 0) 6623 return (rc); 6624 6625 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6626 if (sb == NULL) 6627 return (ENOMEM); 6628 6629 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); 6630 6631 t4_tp_read_la(sc, buf, NULL); 6632 p = buf; 6633 6634 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { 6635 case 2: 6636 inc = 2; 6637 show_func = tp_la_show2; 6638 break; 6639 case 3: 6640 inc = 2; 6641 show_func = tp_la_show3; 6642 break; 6643 default: 6644 inc = 1; 6645 show_func = tp_la_show; 6646 } 6647 6648 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) 6649 (*show_func)(sb, p, i); 6650 6651 rc = sbuf_finish(sb); 6652 sbuf_delete(sb); 6653 free(buf, M_CXGBE); 6654 return (rc); 6655 } 6656 6657 static int 6658 sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 6659 { 6660 struct adapter *sc = arg1; 6661 struct sbuf *sb; 6662 int rc; 6663 u64 nrate[NCHAN], orate[NCHAN]; 6664 6665 rc = sysctl_wire_old_buffer(req, 0); 6666 if (rc != 0) 6667 return (rc); 6668 6669 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6670 if (sb == NULL) 6671 return (ENOMEM); 6672 6673 t4_get_chan_txrate(sc, nrate, orate); 6674 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6675 "channel 3\n"); 6676 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 6677 nrate[0], nrate[1], nrate[2], nrate[3]); 6678 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 6679 orate[0], orate[1], orate[2], orate[3]); 6680 6681 rc = sbuf_finish(sb); 6682 sbuf_delete(sb); 6683 6684 return (rc); 6685 } 6686 6687 static int 6688 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) 6689 { 6690 struct adapter *sc = arg1; 6691 struct sbuf *sb; 6692 uint32_t *buf, *p; 6693 int rc, i; 6694 6695 rc = sysctl_wire_old_buffer(req, 0); 6696 if (rc != 0) 6697 return (rc); 6698 6699 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6700 if (sb == NULL) 6701 return (ENOMEM); 6702 6703 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, 6704 M_ZERO | M_WAITOK); 6705 6706 t4_ulprx_read_la(sc, buf); 6707 p = buf; 6708 6709 sbuf_printf(sb, " Pcmd Type Message" 6710 " Data"); 6711 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { 6712 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", 6713 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); 6714 } 6715 6716 rc = sbuf_finish(sb); 6717 sbuf_delete(sb); 6718 free(buf, M_CXGBE); 6719 return (rc); 6720 } 6721 6722 static int 6723 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 6724 { 6725 struct adapter *sc = arg1; 6726 struct sbuf *sb; 6727 int rc, v; 6728 6729 rc = sysctl_wire_old_buffer(req, 0); 6730 if (rc != 0) 6731 return (rc); 6732 6733 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6734 if (sb == NULL) 6735 return (ENOMEM); 6736 6737 v = t4_read_reg(sc, A_SGE_STAT_CFG); 6738 if (G_STATSOURCE_T5(v) == 7) { 6739 if (G_STATMODE(v) == 0) { 6740 sbuf_printf(sb, "total %d, incomplete %d", 6741 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6742 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6743 } else if (G_STATMODE(v) == 1) { 6744 sbuf_printf(sb, "total %d, data overflow %d", 6745 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6746 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6747 } 6748 } 6749 rc = sbuf_finish(sb); 6750 sbuf_delete(sb); 6751 6752 return (rc); 6753 } 6754 #endif 6755 6756 static inline void 6757 txq_start(struct ifnet *ifp, struct sge_txq *txq) 6758 { 6759 struct buf_ring *br; 6760 struct mbuf *m; 6761 6762 TXQ_LOCK_ASSERT_OWNED(txq); 6763 6764 br = txq->br; 6765 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 6766 if (m) 6767 t4_eth_tx(ifp, txq, m); 6768 } 6769 6770 void 6771 t4_tx_callout(void *arg) 6772 { 6773 struct sge_eq *eq = arg; 6774 struct adapter *sc; 6775 6776 if (EQ_TRYLOCK(eq) == 0) 6777 goto reschedule; 6778 6779 if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) { 6780 EQ_UNLOCK(eq); 6781 reschedule: 6782 if (__predict_true(!(eq->flags && EQ_DOOMED))) 6783 callout_schedule(&eq->tx_callout, 1); 6784 return; 6785 } 6786 6787 EQ_LOCK_ASSERT_OWNED(eq); 6788 6789 if (__predict_true((eq->flags & EQ_DOOMED) == 0)) { 6790 6791 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 6792 struct sge_txq *txq = arg; 6793 struct port_info *pi = txq->ifp->if_softc; 6794 6795 sc = pi->adapter; 6796 } else { 6797 struct sge_wrq *wrq = arg; 6798 6799 sc = wrq->adapter; 6800 } 6801 6802 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task); 6803 } 6804 6805 EQ_UNLOCK(eq); 6806 } 6807 6808 void 6809 t4_tx_task(void *arg, int count) 6810 { 6811 struct sge_eq *eq = arg; 6812 6813 EQ_LOCK(eq); 6814 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 6815 struct sge_txq *txq = arg; 6816 txq_start(txq->ifp, txq); 6817 } else { 6818 struct sge_wrq *wrq = arg; 6819 t4_wrq_tx_locked(wrq->adapter, wrq, NULL); 6820 } 6821 EQ_UNLOCK(eq); 6822 } 6823 6824 static uint32_t 6825 fconf_to_mode(uint32_t fconf) 6826 { 6827 uint32_t mode; 6828 6829 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 6830 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 6831 6832 if (fconf & F_FRAGMENTATION) 6833 mode |= T4_FILTER_IP_FRAGMENT; 6834 6835 if (fconf & F_MPSHITTYPE) 6836 mode |= T4_FILTER_MPS_HIT_TYPE; 6837 6838 if (fconf & F_MACMATCH) 6839 mode |= T4_FILTER_MAC_IDX; 6840 6841 if (fconf & F_ETHERTYPE) 6842 mode |= T4_FILTER_ETH_TYPE; 6843 6844 if (fconf & F_PROTOCOL) 6845 mode |= T4_FILTER_IP_PROTO; 6846 6847 if (fconf & F_TOS) 6848 mode |= T4_FILTER_IP_TOS; 6849 6850 if (fconf & F_VLAN) 6851 mode |= T4_FILTER_VLAN; 6852 6853 if (fconf & F_VNIC_ID) 6854 mode |= T4_FILTER_VNIC; 6855 6856 if (fconf & F_PORT) 6857 mode |= T4_FILTER_PORT; 6858 6859 if (fconf & F_FCOE) 6860 mode |= T4_FILTER_FCoE; 6861 6862 return (mode); 6863 } 6864 6865 static uint32_t 6866 mode_to_fconf(uint32_t mode) 6867 { 6868 uint32_t fconf = 0; 6869 6870 if (mode & T4_FILTER_IP_FRAGMENT) 6871 fconf |= F_FRAGMENTATION; 6872 6873 if (mode & T4_FILTER_MPS_HIT_TYPE) 6874 fconf |= F_MPSHITTYPE; 6875 6876 if (mode & T4_FILTER_MAC_IDX) 6877 fconf |= F_MACMATCH; 6878 6879 if (mode & T4_FILTER_ETH_TYPE) 6880 fconf |= F_ETHERTYPE; 6881 6882 if (mode & T4_FILTER_IP_PROTO) 6883 fconf |= F_PROTOCOL; 6884 6885 if (mode & T4_FILTER_IP_TOS) 6886 fconf |= F_TOS; 6887 6888 if (mode & T4_FILTER_VLAN) 6889 fconf |= F_VLAN; 6890 6891 if (mode & T4_FILTER_VNIC) 6892 fconf |= F_VNIC_ID; 6893 6894 if (mode & T4_FILTER_PORT) 6895 fconf |= F_PORT; 6896 6897 if (mode & T4_FILTER_FCoE) 6898 fconf |= F_FCOE; 6899 6900 return (fconf); 6901 } 6902 6903 static uint32_t 6904 fspec_to_fconf(struct t4_filter_specification *fs) 6905 { 6906 uint32_t fconf = 0; 6907 6908 if (fs->val.frag || fs->mask.frag) 6909 fconf |= F_FRAGMENTATION; 6910 6911 if (fs->val.matchtype || fs->mask.matchtype) 6912 fconf |= F_MPSHITTYPE; 6913 6914 if (fs->val.macidx || fs->mask.macidx) 6915 fconf |= F_MACMATCH; 6916 6917 if (fs->val.ethtype || fs->mask.ethtype) 6918 fconf |= F_ETHERTYPE; 6919 6920 if (fs->val.proto || fs->mask.proto) 6921 fconf |= F_PROTOCOL; 6922 6923 if (fs->val.tos || fs->mask.tos) 6924 fconf |= F_TOS; 6925 6926 if (fs->val.vlan_vld || fs->mask.vlan_vld) 6927 fconf |= F_VLAN; 6928 6929 if (fs->val.vnic_vld || fs->mask.vnic_vld) 6930 fconf |= F_VNIC_ID; 6931 6932 if (fs->val.iport || fs->mask.iport) 6933 fconf |= F_PORT; 6934 6935 if (fs->val.fcoe || fs->mask.fcoe) 6936 fconf |= F_FCOE; 6937 6938 return (fconf); 6939 } 6940 6941 static int 6942 get_filter_mode(struct adapter *sc, uint32_t *mode) 6943 { 6944 int rc; 6945 uint32_t fconf; 6946 6947 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 6948 "t4getfm"); 6949 if (rc) 6950 return (rc); 6951 6952 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 6953 A_TP_VLAN_PRI_MAP); 6954 6955 if (sc->params.tp.vlan_pri_map != fconf) { 6956 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 6957 device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map, 6958 fconf); 6959 sc->params.tp.vlan_pri_map = fconf; 6960 } 6961 6962 *mode = fconf_to_mode(sc->params.tp.vlan_pri_map); 6963 6964 end_synchronized_op(sc, LOCK_HELD); 6965 return (0); 6966 } 6967 6968 static int 6969 set_filter_mode(struct adapter *sc, uint32_t mode) 6970 { 6971 uint32_t fconf; 6972 int rc; 6973 6974 fconf = mode_to_fconf(mode); 6975 6976 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 6977 "t4setfm"); 6978 if (rc) 6979 return (rc); 6980 6981 if (sc->tids.ftids_in_use > 0) { 6982 rc = EBUSY; 6983 goto done; 6984 } 6985 6986 #ifdef TCP_OFFLOAD 6987 if (sc->offload_map) { 6988 rc = EBUSY; 6989 goto done; 6990 } 6991 #endif 6992 6993 #ifdef notyet 6994 rc = -t4_set_filter_mode(sc, fconf); 6995 if (rc == 0) 6996 sc->filter_mode = fconf; 6997 #else 6998 rc = ENOTSUP; 6999 #endif 7000 7001 done: 7002 end_synchronized_op(sc, LOCK_HELD); 7003 return (rc); 7004 } 7005 7006 static inline uint64_t 7007 get_filter_hits(struct adapter *sc, uint32_t fid) 7008 { 7009 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 7010 uint64_t hits; 7011 7012 memwin_info(sc, 0, &mw_base, NULL); 7013 off = position_memwin(sc, 0, 7014 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 7015 if (is_t4(sc)) { 7016 hits = t4_read_reg64(sc, mw_base + off + 16); 7017 hits = be64toh(hits); 7018 } else { 7019 hits = t4_read_reg(sc, mw_base + off + 24); 7020 hits = be32toh(hits); 7021 } 7022 7023 return (hits); 7024 } 7025 7026 static int 7027 get_filter(struct adapter *sc, struct t4_filter *t) 7028 { 7029 int i, rc, nfilters = sc->tids.nftids; 7030 struct filter_entry *f; 7031 7032 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7033 "t4getf"); 7034 if (rc) 7035 return (rc); 7036 7037 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 7038 t->idx >= nfilters) { 7039 t->idx = 0xffffffff; 7040 goto done; 7041 } 7042 7043 f = &sc->tids.ftid_tab[t->idx]; 7044 for (i = t->idx; i < nfilters; i++, f++) { 7045 if (f->valid) { 7046 t->idx = i; 7047 t->l2tidx = f->l2t ? f->l2t->idx : 0; 7048 t->smtidx = f->smtidx; 7049 if (f->fs.hitcnts) 7050 t->hits = get_filter_hits(sc, t->idx); 7051 else 7052 t->hits = UINT64_MAX; 7053 t->fs = f->fs; 7054 7055 goto done; 7056 } 7057 } 7058 7059 t->idx = 0xffffffff; 7060 done: 7061 end_synchronized_op(sc, LOCK_HELD); 7062 return (0); 7063 } 7064 7065 static int 7066 set_filter(struct adapter *sc, struct t4_filter *t) 7067 { 7068 unsigned int nfilters, nports; 7069 struct filter_entry *f; 7070 int i, rc; 7071 7072 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 7073 if (rc) 7074 return (rc); 7075 7076 nfilters = sc->tids.nftids; 7077 nports = sc->params.nports; 7078 7079 if (nfilters == 0) { 7080 rc = ENOTSUP; 7081 goto done; 7082 } 7083 7084 if (!(sc->flags & FULL_INIT_DONE)) { 7085 rc = EAGAIN; 7086 goto done; 7087 } 7088 7089 if (t->idx >= nfilters) { 7090 rc = EINVAL; 7091 goto done; 7092 } 7093 7094 /* Validate against the global filter mode */ 7095 if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) != 7096 sc->params.tp.vlan_pri_map) { 7097 rc = E2BIG; 7098 goto done; 7099 } 7100 7101 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 7102 rc = EINVAL; 7103 goto done; 7104 } 7105 7106 if (t->fs.val.iport >= nports) { 7107 rc = EINVAL; 7108 goto done; 7109 } 7110 7111 /* Can't specify an iq if not steering to it */ 7112 if (!t->fs.dirsteer && t->fs.iq) { 7113 rc = EINVAL; 7114 goto done; 7115 } 7116 7117 /* IPv6 filter idx must be 4 aligned */ 7118 if (t->fs.type == 1 && 7119 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 7120 rc = EINVAL; 7121 goto done; 7122 } 7123 7124 if (sc->tids.ftid_tab == NULL) { 7125 KASSERT(sc->tids.ftids_in_use == 0, 7126 ("%s: no memory allocated but filters_in_use > 0", 7127 __func__)); 7128 7129 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 7130 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 7131 if (sc->tids.ftid_tab == NULL) { 7132 rc = ENOMEM; 7133 goto done; 7134 } 7135 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 7136 } 7137 7138 for (i = 0; i < 4; i++) { 7139 f = &sc->tids.ftid_tab[t->idx + i]; 7140 7141 if (f->pending || f->valid) { 7142 rc = EBUSY; 7143 goto done; 7144 } 7145 if (f->locked) { 7146 rc = EPERM; 7147 goto done; 7148 } 7149 7150 if (t->fs.type == 0) 7151 break; 7152 } 7153 7154 f = &sc->tids.ftid_tab[t->idx]; 7155 f->fs = t->fs; 7156 7157 rc = set_filter_wr(sc, t->idx); 7158 done: 7159 end_synchronized_op(sc, 0); 7160 7161 if (rc == 0) { 7162 mtx_lock(&sc->tids.ftid_lock); 7163 for (;;) { 7164 if (f->pending == 0) { 7165 rc = f->valid ? 0 : EIO; 7166 break; 7167 } 7168 7169 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7170 PCATCH, "t4setfw", 0)) { 7171 rc = EINPROGRESS; 7172 break; 7173 } 7174 } 7175 mtx_unlock(&sc->tids.ftid_lock); 7176 } 7177 return (rc); 7178 } 7179 7180 static int 7181 del_filter(struct adapter *sc, struct t4_filter *t) 7182 { 7183 unsigned int nfilters; 7184 struct filter_entry *f; 7185 int rc; 7186 7187 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 7188 if (rc) 7189 return (rc); 7190 7191 nfilters = sc->tids.nftids; 7192 7193 if (nfilters == 0) { 7194 rc = ENOTSUP; 7195 goto done; 7196 } 7197 7198 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 7199 t->idx >= nfilters) { 7200 rc = EINVAL; 7201 goto done; 7202 } 7203 7204 if (!(sc->flags & FULL_INIT_DONE)) { 7205 rc = EAGAIN; 7206 goto done; 7207 } 7208 7209 f = &sc->tids.ftid_tab[t->idx]; 7210 7211 if (f->pending) { 7212 rc = EBUSY; 7213 goto done; 7214 } 7215 if (f->locked) { 7216 rc = EPERM; 7217 goto done; 7218 } 7219 7220 if (f->valid) { 7221 t->fs = f->fs; /* extra info for the caller */ 7222 rc = del_filter_wr(sc, t->idx); 7223 } 7224 7225 done: 7226 end_synchronized_op(sc, 0); 7227 7228 if (rc == 0) { 7229 mtx_lock(&sc->tids.ftid_lock); 7230 for (;;) { 7231 if (f->pending == 0) { 7232 rc = f->valid ? EIO : 0; 7233 break; 7234 } 7235 7236 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7237 PCATCH, "t4delfw", 0)) { 7238 rc = EINPROGRESS; 7239 break; 7240 } 7241 } 7242 mtx_unlock(&sc->tids.ftid_lock); 7243 } 7244 7245 return (rc); 7246 } 7247 7248 static void 7249 clear_filter(struct filter_entry *f) 7250 { 7251 if (f->l2t) 7252 t4_l2t_release(f->l2t); 7253 7254 bzero(f, sizeof (*f)); 7255 } 7256 7257 static int 7258 set_filter_wr(struct adapter *sc, int fidx) 7259 { 7260 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7261 struct wrqe *wr; 7262 struct fw_filter_wr *fwr; 7263 unsigned int ftid; 7264 7265 ASSERT_SYNCHRONIZED_OP(sc); 7266 7267 if (f->fs.newdmac || f->fs.newvlan) { 7268 /* This filter needs an L2T entry; allocate one. */ 7269 f->l2t = t4_l2t_alloc_switching(sc->l2t); 7270 if (f->l2t == NULL) 7271 return (EAGAIN); 7272 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 7273 f->fs.dmac)) { 7274 t4_l2t_release(f->l2t); 7275 f->l2t = NULL; 7276 return (ENOMEM); 7277 } 7278 } 7279 7280 ftid = sc->tids.ftid_base + fidx; 7281 7282 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 7283 if (wr == NULL) 7284 return (ENOMEM); 7285 7286 fwr = wrtod(wr); 7287 bzero(fwr, sizeof (*fwr)); 7288 7289 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 7290 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 7291 fwr->tid_to_iq = 7292 htobe32(V_FW_FILTER_WR_TID(ftid) | 7293 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 7294 V_FW_FILTER_WR_NOREPLY(0) | 7295 V_FW_FILTER_WR_IQ(f->fs.iq)); 7296 fwr->del_filter_to_l2tix = 7297 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 7298 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 7299 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 7300 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 7301 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 7302 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 7303 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 7304 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 7305 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 7306 f->fs.newvlan == VLAN_REWRITE) | 7307 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 7308 f->fs.newvlan == VLAN_REWRITE) | 7309 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 7310 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 7311 V_FW_FILTER_WR_PRIO(f->fs.prio) | 7312 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 7313 fwr->ethtype = htobe16(f->fs.val.ethtype); 7314 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 7315 fwr->frag_to_ovlan_vldm = 7316 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 7317 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 7318 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 7319 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 7320 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 7321 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 7322 fwr->smac_sel = 0; 7323 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 7324 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 7325 fwr->maci_to_matchtypem = 7326 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 7327 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 7328 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 7329 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 7330 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 7331 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 7332 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 7333 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 7334 fwr->ptcl = f->fs.val.proto; 7335 fwr->ptclm = f->fs.mask.proto; 7336 fwr->ttyp = f->fs.val.tos; 7337 fwr->ttypm = f->fs.mask.tos; 7338 fwr->ivlan = htobe16(f->fs.val.vlan); 7339 fwr->ivlanm = htobe16(f->fs.mask.vlan); 7340 fwr->ovlan = htobe16(f->fs.val.vnic); 7341 fwr->ovlanm = htobe16(f->fs.mask.vnic); 7342 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 7343 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 7344 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 7345 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 7346 fwr->lp = htobe16(f->fs.val.dport); 7347 fwr->lpm = htobe16(f->fs.mask.dport); 7348 fwr->fp = htobe16(f->fs.val.sport); 7349 fwr->fpm = htobe16(f->fs.mask.sport); 7350 if (f->fs.newsmac) 7351 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 7352 7353 f->pending = 1; 7354 sc->tids.ftids_in_use++; 7355 7356 t4_wrq_tx(sc, wr); 7357 return (0); 7358 } 7359 7360 static int 7361 del_filter_wr(struct adapter *sc, int fidx) 7362 { 7363 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7364 struct wrqe *wr; 7365 struct fw_filter_wr *fwr; 7366 unsigned int ftid; 7367 7368 ftid = sc->tids.ftid_base + fidx; 7369 7370 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 7371 if (wr == NULL) 7372 return (ENOMEM); 7373 fwr = wrtod(wr); 7374 bzero(fwr, sizeof (*fwr)); 7375 7376 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 7377 7378 f->pending = 1; 7379 t4_wrq_tx(sc, wr); 7380 return (0); 7381 } 7382 7383 int 7384 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 7385 { 7386 struct adapter *sc = iq->adapter; 7387 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 7388 unsigned int idx = GET_TID(rpl); 7389 unsigned int rc; 7390 struct filter_entry *f; 7391 7392 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 7393 rss->opcode)); 7394 7395 if (is_ftid(sc, idx)) { 7396 7397 idx -= sc->tids.ftid_base; 7398 f = &sc->tids.ftid_tab[idx]; 7399 rc = G_COOKIE(rpl->cookie); 7400 7401 mtx_lock(&sc->tids.ftid_lock); 7402 if (rc == FW_FILTER_WR_FLT_ADDED) { 7403 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 7404 __func__, idx)); 7405 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 7406 f->pending = 0; /* asynchronous setup completed */ 7407 f->valid = 1; 7408 } else { 7409 if (rc != FW_FILTER_WR_FLT_DELETED) { 7410 /* Add or delete failed, display an error */ 7411 log(LOG_ERR, 7412 "filter %u setup failed with error %u\n", 7413 idx, rc); 7414 } 7415 7416 clear_filter(f); 7417 sc->tids.ftids_in_use--; 7418 } 7419 wakeup(&sc->tids.ftid_tab); 7420 mtx_unlock(&sc->tids.ftid_lock); 7421 } 7422 7423 return (0); 7424 } 7425 7426 static int 7427 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 7428 { 7429 int rc; 7430 7431 if (cntxt->cid > M_CTXTQID) 7432 return (EINVAL); 7433 7434 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 7435 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 7436 return (EINVAL); 7437 7438 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 7439 if (rc) 7440 return (rc); 7441 7442 if (sc->flags & FW_OK) { 7443 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 7444 &cntxt->data[0]); 7445 if (rc == 0) 7446 goto done; 7447 } 7448 7449 /* 7450 * Read via firmware failed or wasn't even attempted. Read directly via 7451 * the backdoor. 7452 */ 7453 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 7454 done: 7455 end_synchronized_op(sc, 0); 7456 return (rc); 7457 } 7458 7459 static int 7460 load_fw(struct adapter *sc, struct t4_data *fw) 7461 { 7462 int rc; 7463 uint8_t *fw_data; 7464 7465 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 7466 if (rc) 7467 return (rc); 7468 7469 if (sc->flags & FULL_INIT_DONE) { 7470 rc = EBUSY; 7471 goto done; 7472 } 7473 7474 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 7475 if (fw_data == NULL) { 7476 rc = ENOMEM; 7477 goto done; 7478 } 7479 7480 rc = copyin(fw->data, fw_data, fw->len); 7481 if (rc == 0) 7482 rc = -t4_load_fw(sc, fw_data, fw->len); 7483 7484 free(fw_data, M_CXGBE); 7485 done: 7486 end_synchronized_op(sc, 0); 7487 return (rc); 7488 } 7489 7490 static int 7491 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 7492 { 7493 uint32_t addr, off, remaining, i, n; 7494 uint32_t *buf, *b; 7495 uint32_t mw_base, mw_aperture; 7496 int rc; 7497 uint8_t *dst; 7498 7499 rc = validate_mem_range(sc, mr->addr, mr->len); 7500 if (rc != 0) 7501 return (rc); 7502 7503 memwin_info(sc, win, &mw_base, &mw_aperture); 7504 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 7505 addr = mr->addr; 7506 remaining = mr->len; 7507 dst = (void *)mr->data; 7508 7509 while (remaining) { 7510 off = position_memwin(sc, win, addr); 7511 7512 /* number of bytes that we'll copy in the inner loop */ 7513 n = min(remaining, mw_aperture - off); 7514 for (i = 0; i < n; i += 4) 7515 *b++ = t4_read_reg(sc, mw_base + off + i); 7516 7517 rc = copyout(buf, dst, n); 7518 if (rc != 0) 7519 break; 7520 7521 b = buf; 7522 dst += n; 7523 remaining -= n; 7524 addr += n; 7525 } 7526 7527 free(buf, M_CXGBE); 7528 return (rc); 7529 } 7530 7531 static int 7532 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 7533 { 7534 int rc; 7535 7536 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 7537 return (EINVAL); 7538 7539 if (i2cd->len > sizeof(i2cd->data)) 7540 return (EFBIG); 7541 7542 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 7543 if (rc) 7544 return (rc); 7545 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 7546 i2cd->offset, i2cd->len, &i2cd->data[0]); 7547 end_synchronized_op(sc, 0); 7548 7549 return (rc); 7550 } 7551 7552 static int 7553 in_range(int val, int lo, int hi) 7554 { 7555 7556 return (val < 0 || (val <= hi && val >= lo)); 7557 } 7558 7559 static int 7560 set_sched_class(struct adapter *sc, struct t4_sched_params *p) 7561 { 7562 int fw_subcmd, fw_type, rc; 7563 7564 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc"); 7565 if (rc) 7566 return (rc); 7567 7568 if (!(sc->flags & FULL_INIT_DONE)) { 7569 rc = EAGAIN; 7570 goto done; 7571 } 7572 7573 /* 7574 * Translate the cxgbetool parameters into T4 firmware parameters. (The 7575 * sub-command and type are in common locations.) 7576 */ 7577 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG) 7578 fw_subcmd = FW_SCHED_SC_CONFIG; 7579 else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS) 7580 fw_subcmd = FW_SCHED_SC_PARAMS; 7581 else { 7582 rc = EINVAL; 7583 goto done; 7584 } 7585 if (p->type == SCHED_CLASS_TYPE_PACKET) 7586 fw_type = FW_SCHED_TYPE_PKTSCHED; 7587 else { 7588 rc = EINVAL; 7589 goto done; 7590 } 7591 7592 if (fw_subcmd == FW_SCHED_SC_CONFIG) { 7593 /* Vet our parameters ..*/ 7594 if (p->u.config.minmax < 0) { 7595 rc = EINVAL; 7596 goto done; 7597 } 7598 7599 /* And pass the request to the firmware ...*/ 7600 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax); 7601 goto done; 7602 } 7603 7604 if (fw_subcmd == FW_SCHED_SC_PARAMS) { 7605 int fw_level; 7606 int fw_mode; 7607 int fw_rateunit; 7608 int fw_ratemode; 7609 7610 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL) 7611 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL; 7612 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) 7613 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR; 7614 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) 7615 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL; 7616 else { 7617 rc = EINVAL; 7618 goto done; 7619 } 7620 7621 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS) 7622 fw_mode = FW_SCHED_PARAMS_MODE_CLASS; 7623 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW) 7624 fw_mode = FW_SCHED_PARAMS_MODE_FLOW; 7625 else { 7626 rc = EINVAL; 7627 goto done; 7628 } 7629 7630 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS) 7631 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 7632 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS) 7633 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE; 7634 else { 7635 rc = EINVAL; 7636 goto done; 7637 } 7638 7639 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL) 7640 fw_ratemode = FW_SCHED_PARAMS_RATE_REL; 7641 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS) 7642 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS; 7643 else { 7644 rc = EINVAL; 7645 goto done; 7646 } 7647 7648 /* Vet our parameters ... */ 7649 if (!in_range(p->u.params.channel, 0, 3) || 7650 !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) || 7651 !in_range(p->u.params.minrate, 0, 10000000) || 7652 !in_range(p->u.params.maxrate, 0, 10000000) || 7653 !in_range(p->u.params.weight, 0, 100)) { 7654 rc = ERANGE; 7655 goto done; 7656 } 7657 7658 /* 7659 * Translate any unset parameters into the firmware's 7660 * nomenclature and/or fail the call if the parameters 7661 * are required ... 7662 */ 7663 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 || 7664 p->u.params.channel < 0 || p->u.params.cl < 0) { 7665 rc = EINVAL; 7666 goto done; 7667 } 7668 if (p->u.params.minrate < 0) 7669 p->u.params.minrate = 0; 7670 if (p->u.params.maxrate < 0) { 7671 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7672 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7673 rc = EINVAL; 7674 goto done; 7675 } else 7676 p->u.params.maxrate = 0; 7677 } 7678 if (p->u.params.weight < 0) { 7679 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) { 7680 rc = EINVAL; 7681 goto done; 7682 } else 7683 p->u.params.weight = 0; 7684 } 7685 if (p->u.params.pktsize < 0) { 7686 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7687 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7688 rc = EINVAL; 7689 goto done; 7690 } else 7691 p->u.params.pktsize = 0; 7692 } 7693 7694 /* See what the firmware thinks of the request ... */ 7695 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode, 7696 fw_rateunit, fw_ratemode, p->u.params.channel, 7697 p->u.params.cl, p->u.params.minrate, p->u.params.maxrate, 7698 p->u.params.weight, p->u.params.pktsize); 7699 goto done; 7700 } 7701 7702 rc = EINVAL; 7703 done: 7704 end_synchronized_op(sc, 0); 7705 return (rc); 7706 } 7707 7708 static int 7709 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p) 7710 { 7711 struct port_info *pi = NULL; 7712 struct sge_txq *txq; 7713 uint32_t fw_mnem, fw_queue, fw_class; 7714 int i, rc; 7715 7716 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq"); 7717 if (rc) 7718 return (rc); 7719 7720 if (!(sc->flags & FULL_INIT_DONE)) { 7721 rc = EAGAIN; 7722 goto done; 7723 } 7724 7725 if (p->port >= sc->params.nports) { 7726 rc = EINVAL; 7727 goto done; 7728 } 7729 7730 pi = sc->port[p->port]; 7731 if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) { 7732 rc = EINVAL; 7733 goto done; 7734 } 7735 7736 /* 7737 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX 7738 * Scheduling Class in this case). 7739 */ 7740 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 7741 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH)); 7742 fw_class = p->cl < 0 ? 0xffffffff : p->cl; 7743 7744 /* 7745 * If op.queue is non-negative, then we're only changing the scheduling 7746 * on a single specified TX queue. 7747 */ 7748 if (p->queue >= 0) { 7749 txq = &sc->sge.txq[pi->first_txq + p->queue]; 7750 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7751 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7752 &fw_class); 7753 goto done; 7754 } 7755 7756 /* 7757 * Change the scheduling on all the TX queues for the 7758 * interface. 7759 */ 7760 for_each_txq(pi, i, txq) { 7761 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7762 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7763 &fw_class); 7764 if (rc) 7765 goto done; 7766 } 7767 7768 rc = 0; 7769 done: 7770 end_synchronized_op(sc, 0); 7771 return (rc); 7772 } 7773 7774 int 7775 t4_os_find_pci_capability(struct adapter *sc, int cap) 7776 { 7777 int i; 7778 7779 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 7780 } 7781 7782 int 7783 t4_os_pci_save_state(struct adapter *sc) 7784 { 7785 device_t dev; 7786 struct pci_devinfo *dinfo; 7787 7788 dev = sc->dev; 7789 dinfo = device_get_ivars(dev); 7790 7791 pci_cfg_save(dev, dinfo, 0); 7792 return (0); 7793 } 7794 7795 int 7796 t4_os_pci_restore_state(struct adapter *sc) 7797 { 7798 device_t dev; 7799 struct pci_devinfo *dinfo; 7800 7801 dev = sc->dev; 7802 dinfo = device_get_ivars(dev); 7803 7804 pci_cfg_restore(dev, dinfo); 7805 return (0); 7806 } 7807 7808 void 7809 t4_os_portmod_changed(const struct adapter *sc, int idx) 7810 { 7811 struct port_info *pi = sc->port[idx]; 7812 static const char *mod_str[] = { 7813 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 7814 }; 7815 7816 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 7817 if_printf(pi->ifp, "transceiver unplugged.\n"); 7818 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 7819 if_printf(pi->ifp, "unknown transceiver inserted.\n"); 7820 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 7821 if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 7822 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 7823 if_printf(pi->ifp, "%s transceiver inserted.\n", 7824 mod_str[pi->mod_type]); 7825 } else { 7826 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 7827 pi->mod_type); 7828 } 7829 } 7830 7831 void 7832 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason) 7833 { 7834 struct port_info *pi = sc->port[idx]; 7835 struct ifnet *ifp = pi->ifp; 7836 7837 if (link_stat) { 7838 pi->linkdnrc = -1; 7839 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 7840 if_link_state_change(ifp, LINK_STATE_UP); 7841 } else { 7842 if (reason >= 0) 7843 pi->linkdnrc = reason; 7844 if_link_state_change(ifp, LINK_STATE_DOWN); 7845 } 7846 } 7847 7848 void 7849 t4_iterate(void (*func)(struct adapter *, void *), void *arg) 7850 { 7851 struct adapter *sc; 7852 7853 sx_slock(&t4_list_lock); 7854 SLIST_FOREACH(sc, &t4_list, link) { 7855 /* 7856 * func should not make any assumptions about what state sc is 7857 * in - the only guarantee is that sc->sc_lock is a valid lock. 7858 */ 7859 func(sc, arg); 7860 } 7861 sx_sunlock(&t4_list_lock); 7862 } 7863 7864 static int 7865 t4_open(struct cdev *dev, int flags, int type, struct thread *td) 7866 { 7867 return (0); 7868 } 7869 7870 static int 7871 t4_close(struct cdev *dev, int flags, int type, struct thread *td) 7872 { 7873 return (0); 7874 } 7875 7876 static int 7877 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 7878 struct thread *td) 7879 { 7880 int rc; 7881 struct adapter *sc = dev->si_drv1; 7882 7883 rc = priv_check(td, PRIV_DRIVER); 7884 if (rc != 0) 7885 return (rc); 7886 7887 switch (cmd) { 7888 case CHELSIO_T4_GETREG: { 7889 struct t4_reg *edata = (struct t4_reg *)data; 7890 7891 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 7892 return (EFAULT); 7893 7894 if (edata->size == 4) 7895 edata->val = t4_read_reg(sc, edata->addr); 7896 else if (edata->size == 8) 7897 edata->val = t4_read_reg64(sc, edata->addr); 7898 else 7899 return (EINVAL); 7900 7901 break; 7902 } 7903 case CHELSIO_T4_SETREG: { 7904 struct t4_reg *edata = (struct t4_reg *)data; 7905 7906 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 7907 return (EFAULT); 7908 7909 if (edata->size == 4) { 7910 if (edata->val & 0xffffffff00000000) 7911 return (EINVAL); 7912 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 7913 } else if (edata->size == 8) 7914 t4_write_reg64(sc, edata->addr, edata->val); 7915 else 7916 return (EINVAL); 7917 break; 7918 } 7919 case CHELSIO_T4_REGDUMP: { 7920 struct t4_regdump *regs = (struct t4_regdump *)data; 7921 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 7922 uint8_t *buf; 7923 7924 if (regs->len < reglen) { 7925 regs->len = reglen; /* hint to the caller */ 7926 return (ENOBUFS); 7927 } 7928 7929 regs->len = reglen; 7930 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 7931 t4_get_regs(sc, regs, buf); 7932 rc = copyout(buf, regs->data, reglen); 7933 free(buf, M_CXGBE); 7934 break; 7935 } 7936 case CHELSIO_T4_GET_FILTER_MODE: 7937 rc = get_filter_mode(sc, (uint32_t *)data); 7938 break; 7939 case CHELSIO_T4_SET_FILTER_MODE: 7940 rc = set_filter_mode(sc, *(uint32_t *)data); 7941 break; 7942 case CHELSIO_T4_GET_FILTER: 7943 rc = get_filter(sc, (struct t4_filter *)data); 7944 break; 7945 case CHELSIO_T4_SET_FILTER: 7946 rc = set_filter(sc, (struct t4_filter *)data); 7947 break; 7948 case CHELSIO_T4_DEL_FILTER: 7949 rc = del_filter(sc, (struct t4_filter *)data); 7950 break; 7951 case CHELSIO_T4_GET_SGE_CONTEXT: 7952 rc = get_sge_context(sc, (struct t4_sge_context *)data); 7953 break; 7954 case CHELSIO_T4_LOAD_FW: 7955 rc = load_fw(sc, (struct t4_data *)data); 7956 break; 7957 case CHELSIO_T4_GET_MEM: 7958 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 7959 break; 7960 case CHELSIO_T4_GET_I2C: 7961 rc = read_i2c(sc, (struct t4_i2c_data *)data); 7962 break; 7963 case CHELSIO_T4_CLEAR_STATS: { 7964 int i; 7965 u_int port_id = *(uint32_t *)data; 7966 struct port_info *pi; 7967 7968 if (port_id >= sc->params.nports) 7969 return (EINVAL); 7970 pi = sc->port[port_id]; 7971 7972 /* MAC stats */ 7973 t4_clr_port_stats(sc, pi->tx_chan); 7974 7975 if (pi->flags & PORT_INIT_DONE) { 7976 struct sge_rxq *rxq; 7977 struct sge_txq *txq; 7978 struct sge_wrq *wrq; 7979 7980 for_each_rxq(pi, i, rxq) { 7981 #if defined(INET) || defined(INET6) 7982 rxq->lro.lro_queued = 0; 7983 rxq->lro.lro_flushed = 0; 7984 #endif 7985 rxq->rxcsum = 0; 7986 rxq->vlan_extraction = 0; 7987 } 7988 7989 for_each_txq(pi, i, txq) { 7990 txq->txcsum = 0; 7991 txq->tso_wrs = 0; 7992 txq->vlan_insertion = 0; 7993 txq->imm_wrs = 0; 7994 txq->sgl_wrs = 0; 7995 txq->txpkt_wrs = 0; 7996 txq->txpkts_wrs = 0; 7997 txq->txpkts_pkts = 0; 7998 txq->br->br_drops = 0; 7999 txq->no_dmamap = 0; 8000 txq->no_desc = 0; 8001 } 8002 8003 #ifdef TCP_OFFLOAD 8004 /* nothing to clear for each ofld_rxq */ 8005 8006 for_each_ofld_txq(pi, i, wrq) { 8007 wrq->tx_wrs = 0; 8008 wrq->no_desc = 0; 8009 } 8010 #endif 8011 wrq = &sc->sge.ctrlq[pi->port_id]; 8012 wrq->tx_wrs = 0; 8013 wrq->no_desc = 0; 8014 } 8015 break; 8016 } 8017 case CHELSIO_T4_SCHED_CLASS: 8018 rc = set_sched_class(sc, (struct t4_sched_params *)data); 8019 break; 8020 case CHELSIO_T4_SCHED_QUEUE: 8021 rc = set_sched_queue(sc, (struct t4_sched_queue *)data); 8022 break; 8023 case CHELSIO_T4_GET_TRACER: 8024 rc = t4_get_tracer(sc, (struct t4_tracer *)data); 8025 break; 8026 case CHELSIO_T4_SET_TRACER: 8027 rc = t4_set_tracer(sc, (struct t4_tracer *)data); 8028 break; 8029 default: 8030 rc = EINVAL; 8031 } 8032 8033 return (rc); 8034 } 8035 8036 #ifdef TCP_OFFLOAD 8037 static int 8038 toe_capability(struct port_info *pi, int enable) 8039 { 8040 int rc; 8041 struct adapter *sc = pi->adapter; 8042 8043 ASSERT_SYNCHRONIZED_OP(sc); 8044 8045 if (!is_offload(sc)) 8046 return (ENODEV); 8047 8048 if (enable) { 8049 if (!(sc->flags & FULL_INIT_DONE)) { 8050 rc = cxgbe_init_synchronized(pi); 8051 if (rc) 8052 return (rc); 8053 } 8054 8055 if (isset(&sc->offload_map, pi->port_id)) 8056 return (0); 8057 8058 if (!(sc->flags & TOM_INIT_DONE)) { 8059 rc = t4_activate_uld(sc, ULD_TOM); 8060 if (rc == EAGAIN) { 8061 log(LOG_WARNING, 8062 "You must kldload t4_tom.ko before trying " 8063 "to enable TOE on a cxgbe interface.\n"); 8064 } 8065 if (rc != 0) 8066 return (rc); 8067 KASSERT(sc->tom_softc != NULL, 8068 ("%s: TOM activated but softc NULL", __func__)); 8069 KASSERT(sc->flags & TOM_INIT_DONE, 8070 ("%s: TOM activated but flag not set", __func__)); 8071 } 8072 8073 setbit(&sc->offload_map, pi->port_id); 8074 } else { 8075 if (!isset(&sc->offload_map, pi->port_id)) 8076 return (0); 8077 8078 KASSERT(sc->flags & TOM_INIT_DONE, 8079 ("%s: TOM never initialized?", __func__)); 8080 clrbit(&sc->offload_map, pi->port_id); 8081 } 8082 8083 return (0); 8084 } 8085 8086 /* 8087 * Add an upper layer driver to the global list. 8088 */ 8089 int 8090 t4_register_uld(struct uld_info *ui) 8091 { 8092 int rc = 0; 8093 struct uld_info *u; 8094 8095 sx_xlock(&t4_uld_list_lock); 8096 SLIST_FOREACH(u, &t4_uld_list, link) { 8097 if (u->uld_id == ui->uld_id) { 8098 rc = EEXIST; 8099 goto done; 8100 } 8101 } 8102 8103 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 8104 ui->refcount = 0; 8105 done: 8106 sx_xunlock(&t4_uld_list_lock); 8107 return (rc); 8108 } 8109 8110 int 8111 t4_unregister_uld(struct uld_info *ui) 8112 { 8113 int rc = EINVAL; 8114 struct uld_info *u; 8115 8116 sx_xlock(&t4_uld_list_lock); 8117 8118 SLIST_FOREACH(u, &t4_uld_list, link) { 8119 if (u == ui) { 8120 if (ui->refcount > 0) { 8121 rc = EBUSY; 8122 goto done; 8123 } 8124 8125 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 8126 rc = 0; 8127 goto done; 8128 } 8129 } 8130 done: 8131 sx_xunlock(&t4_uld_list_lock); 8132 return (rc); 8133 } 8134 8135 int 8136 t4_activate_uld(struct adapter *sc, int id) 8137 { 8138 int rc = EAGAIN; 8139 struct uld_info *ui; 8140 8141 ASSERT_SYNCHRONIZED_OP(sc); 8142 8143 sx_slock(&t4_uld_list_lock); 8144 8145 SLIST_FOREACH(ui, &t4_uld_list, link) { 8146 if (ui->uld_id == id) { 8147 rc = ui->activate(sc); 8148 if (rc == 0) 8149 ui->refcount++; 8150 goto done; 8151 } 8152 } 8153 done: 8154 sx_sunlock(&t4_uld_list_lock); 8155 8156 return (rc); 8157 } 8158 8159 int 8160 t4_deactivate_uld(struct adapter *sc, int id) 8161 { 8162 int rc = EINVAL; 8163 struct uld_info *ui; 8164 8165 ASSERT_SYNCHRONIZED_OP(sc); 8166 8167 sx_slock(&t4_uld_list_lock); 8168 8169 SLIST_FOREACH(ui, &t4_uld_list, link) { 8170 if (ui->uld_id == id) { 8171 rc = ui->deactivate(sc); 8172 if (rc == 0) 8173 ui->refcount--; 8174 goto done; 8175 } 8176 } 8177 done: 8178 sx_sunlock(&t4_uld_list_lock); 8179 8180 return (rc); 8181 } 8182 #endif 8183 8184 /* 8185 * Come up with reasonable defaults for some of the tunables, provided they're 8186 * not set by the user (in which case we'll use the values as is). 8187 */ 8188 static void 8189 tweak_tunables(void) 8190 { 8191 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 8192 8193 if (t4_ntxq10g < 1) 8194 t4_ntxq10g = min(nc, NTXQ_10G); 8195 8196 if (t4_ntxq1g < 1) 8197 t4_ntxq1g = min(nc, NTXQ_1G); 8198 8199 if (t4_nrxq10g < 1) 8200 t4_nrxq10g = min(nc, NRXQ_10G); 8201 8202 if (t4_nrxq1g < 1) 8203 t4_nrxq1g = min(nc, NRXQ_1G); 8204 8205 #ifdef TCP_OFFLOAD 8206 if (t4_nofldtxq10g < 1) 8207 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 8208 8209 if (t4_nofldtxq1g < 1) 8210 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 8211 8212 if (t4_nofldrxq10g < 1) 8213 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 8214 8215 if (t4_nofldrxq1g < 1) 8216 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 8217 8218 if (t4_toecaps_allowed == -1) 8219 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 8220 #else 8221 if (t4_toecaps_allowed == -1) 8222 t4_toecaps_allowed = 0; 8223 #endif 8224 8225 #ifdef DEV_NETMAP 8226 if (t4_nnmtxq10g < 1) 8227 t4_nnmtxq10g = min(nc, NNMTXQ_10G); 8228 8229 if (t4_nnmtxq1g < 1) 8230 t4_nnmtxq1g = min(nc, NNMTXQ_1G); 8231 8232 if (t4_nnmrxq10g < 1) 8233 t4_nnmrxq10g = min(nc, NNMRXQ_10G); 8234 8235 if (t4_nnmrxq1g < 1) 8236 t4_nnmrxq1g = min(nc, NNMRXQ_1G); 8237 #endif 8238 8239 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 8240 t4_tmr_idx_10g = TMR_IDX_10G; 8241 8242 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 8243 t4_pktc_idx_10g = PKTC_IDX_10G; 8244 8245 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 8246 t4_tmr_idx_1g = TMR_IDX_1G; 8247 8248 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 8249 t4_pktc_idx_1g = PKTC_IDX_1G; 8250 8251 if (t4_qsize_txq < 128) 8252 t4_qsize_txq = 128; 8253 8254 if (t4_qsize_rxq < 128) 8255 t4_qsize_rxq = 128; 8256 while (t4_qsize_rxq & 7) 8257 t4_qsize_rxq++; 8258 8259 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 8260 } 8261 8262 static int 8263 mod_event(module_t mod, int cmd, void *arg) 8264 { 8265 int rc = 0; 8266 static int loaded = 0; 8267 8268 switch (cmd) { 8269 case MOD_LOAD: 8270 if (atomic_fetchadd_int(&loaded, 1)) 8271 break; 8272 t4_sge_modload(); 8273 sx_init(&t4_list_lock, "T4/T5 adapters"); 8274 SLIST_INIT(&t4_list); 8275 #ifdef TCP_OFFLOAD 8276 sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); 8277 SLIST_INIT(&t4_uld_list); 8278 #endif 8279 t4_tracer_modload(); 8280 tweak_tunables(); 8281 break; 8282 8283 case MOD_UNLOAD: 8284 if (atomic_fetchadd_int(&loaded, -1) > 1) 8285 break; 8286 t4_tracer_modunload(); 8287 #ifdef TCP_OFFLOAD 8288 sx_slock(&t4_uld_list_lock); 8289 if (!SLIST_EMPTY(&t4_uld_list)) { 8290 rc = EBUSY; 8291 sx_sunlock(&t4_uld_list_lock); 8292 break; 8293 } 8294 sx_sunlock(&t4_uld_list_lock); 8295 sx_destroy(&t4_uld_list_lock); 8296 #endif 8297 sx_slock(&t4_list_lock); 8298 if (!SLIST_EMPTY(&t4_list)) { 8299 rc = EBUSY; 8300 sx_sunlock(&t4_list_lock); 8301 break; 8302 } 8303 sx_sunlock(&t4_list_lock); 8304 sx_destroy(&t4_list_lock); 8305 break; 8306 } 8307 8308 return (rc); 8309 } 8310 8311 static devclass_t t4_devclass, t5_devclass; 8312 static devclass_t cxgbe_devclass, cxl_devclass; 8313 8314 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 8315 MODULE_VERSION(t4nex, 1); 8316 MODULE_DEPEND(t4nex, firmware, 1, 1, 1); 8317 8318 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 8319 MODULE_VERSION(t5nex, 1); 8320 MODULE_DEPEND(t5nex, firmware, 1, 1, 1); 8321 8322 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 8323 MODULE_VERSION(cxgbe, 1); 8324 8325 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 8326 MODULE_VERSION(cxl, 1); 8327