1 /*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_inet.h" 32 #include "opt_inet6.h" 33 34 #include <sys/param.h> 35 #include <sys/conf.h> 36 #include <sys/priv.h> 37 #include <sys/kernel.h> 38 #include <sys/bus.h> 39 #include <sys/module.h> 40 #include <sys/malloc.h> 41 #include <sys/queue.h> 42 #include <sys/taskqueue.h> 43 #include <sys/pciio.h> 44 #include <dev/pci/pcireg.h> 45 #include <dev/pci/pcivar.h> 46 #include <dev/pci/pci_private.h> 47 #include <sys/firmware.h> 48 #include <sys/sbuf.h> 49 #include <sys/smp.h> 50 #include <sys/socket.h> 51 #include <sys/sockio.h> 52 #include <sys/sysctl.h> 53 #include <net/ethernet.h> 54 #include <net/if.h> 55 #include <net/if_types.h> 56 #include <net/if_dl.h> 57 #include <net/if_vlan_var.h> 58 #if defined(__i386__) || defined(__amd64__) 59 #include <vm/vm.h> 60 #include <vm/pmap.h> 61 #endif 62 63 #include "common/common.h" 64 #include "common/t4_msg.h" 65 #include "common/t4_regs.h" 66 #include "common/t4_regs_values.h" 67 #include "t4_ioctl.h" 68 #include "t4_l2t.h" 69 70 /* T4 bus driver interface */ 71 static int t4_probe(device_t); 72 static int t4_attach(device_t); 73 static int t4_detach(device_t); 74 static device_method_t t4_methods[] = { 75 DEVMETHOD(device_probe, t4_probe), 76 DEVMETHOD(device_attach, t4_attach), 77 DEVMETHOD(device_detach, t4_detach), 78 79 DEVMETHOD_END 80 }; 81 static driver_t t4_driver = { 82 "t4nex", 83 t4_methods, 84 sizeof(struct adapter) 85 }; 86 87 88 /* T4 port (cxgbe) interface */ 89 static int cxgbe_probe(device_t); 90 static int cxgbe_attach(device_t); 91 static int cxgbe_detach(device_t); 92 static device_method_t cxgbe_methods[] = { 93 DEVMETHOD(device_probe, cxgbe_probe), 94 DEVMETHOD(device_attach, cxgbe_attach), 95 DEVMETHOD(device_detach, cxgbe_detach), 96 { 0, 0 } 97 }; 98 static driver_t cxgbe_driver = { 99 "cxgbe", 100 cxgbe_methods, 101 sizeof(struct port_info) 102 }; 103 104 static d_ioctl_t t4_ioctl; 105 static d_open_t t4_open; 106 static d_close_t t4_close; 107 108 static struct cdevsw t4_cdevsw = { 109 .d_version = D_VERSION, 110 .d_flags = 0, 111 .d_open = t4_open, 112 .d_close = t4_close, 113 .d_ioctl = t4_ioctl, 114 .d_name = "t4nex", 115 }; 116 117 /* T5 bus driver interface */ 118 static int t5_probe(device_t); 119 static device_method_t t5_methods[] = { 120 DEVMETHOD(device_probe, t5_probe), 121 DEVMETHOD(device_attach, t4_attach), 122 DEVMETHOD(device_detach, t4_detach), 123 124 DEVMETHOD_END 125 }; 126 static driver_t t5_driver = { 127 "t5nex", 128 t5_methods, 129 sizeof(struct adapter) 130 }; 131 132 133 /* T5 port (cxl) interface */ 134 static driver_t cxl_driver = { 135 "cxl", 136 cxgbe_methods, 137 sizeof(struct port_info) 138 }; 139 140 static struct cdevsw t5_cdevsw = { 141 .d_version = D_VERSION, 142 .d_flags = 0, 143 .d_open = t4_open, 144 .d_close = t4_close, 145 .d_ioctl = t4_ioctl, 146 .d_name = "t5nex", 147 }; 148 149 /* ifnet + media interface */ 150 static void cxgbe_init(void *); 151 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 152 static int cxgbe_transmit(struct ifnet *, struct mbuf *); 153 static void cxgbe_qflush(struct ifnet *); 154 static int cxgbe_media_change(struct ifnet *); 155 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 156 157 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 158 159 /* 160 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 161 * then ADAPTER_LOCK, then t4_uld_list_lock. 162 */ 163 static struct sx t4_list_lock; 164 SLIST_HEAD(, adapter) t4_list; 165 #ifdef TCP_OFFLOAD 166 static struct sx t4_uld_list_lock; 167 SLIST_HEAD(, uld_info) t4_uld_list; 168 #endif 169 170 /* 171 * Tunables. See tweak_tunables() too. 172 * 173 * Each tunable is set to a default value here if it's known at compile-time. 174 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 175 * provide a reasonable default when the driver is loaded. 176 * 177 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 178 * T5 are under hw.cxl. 179 */ 180 181 /* 182 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 183 */ 184 #define NTXQ_10G 16 185 static int t4_ntxq10g = -1; 186 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 187 188 #define NRXQ_10G 8 189 static int t4_nrxq10g = -1; 190 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 191 192 #define NTXQ_1G 4 193 static int t4_ntxq1g = -1; 194 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 195 196 #define NRXQ_1G 2 197 static int t4_nrxq1g = -1; 198 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 199 200 static int t4_rsrv_noflowq = 0; 201 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq); 202 203 #ifdef TCP_OFFLOAD 204 #define NOFLDTXQ_10G 8 205 static int t4_nofldtxq10g = -1; 206 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 207 208 #define NOFLDRXQ_10G 2 209 static int t4_nofldrxq10g = -1; 210 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 211 212 #define NOFLDTXQ_1G 2 213 static int t4_nofldtxq1g = -1; 214 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 215 216 #define NOFLDRXQ_1G 1 217 static int t4_nofldrxq1g = -1; 218 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 219 #endif 220 221 #ifdef DEV_NETMAP 222 #define NNMTXQ_10G 2 223 static int t4_nnmtxq10g = -1; 224 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g); 225 226 #define NNMRXQ_10G 2 227 static int t4_nnmrxq10g = -1; 228 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g); 229 230 #define NNMTXQ_1G 1 231 static int t4_nnmtxq1g = -1; 232 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g); 233 234 #define NNMRXQ_1G 1 235 static int t4_nnmrxq1g = -1; 236 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g); 237 #endif 238 239 /* 240 * Holdoff parameters for 10G and 1G ports. 241 */ 242 #define TMR_IDX_10G 1 243 static int t4_tmr_idx_10g = TMR_IDX_10G; 244 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 245 246 #define PKTC_IDX_10G (-1) 247 static int t4_pktc_idx_10g = PKTC_IDX_10G; 248 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 249 250 #define TMR_IDX_1G 1 251 static int t4_tmr_idx_1g = TMR_IDX_1G; 252 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 253 254 #define PKTC_IDX_1G (-1) 255 static int t4_pktc_idx_1g = PKTC_IDX_1G; 256 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 257 258 /* 259 * Size (# of entries) of each tx and rx queue. 260 */ 261 static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 262 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 263 264 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 265 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 266 267 /* 268 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 269 */ 270 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 271 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 272 273 /* 274 * Configuration file. 275 */ 276 #define DEFAULT_CF "default" 277 #define FLASH_CF "flash" 278 #define UWIRE_CF "uwire" 279 #define FPGA_CF "fpga" 280 static char t4_cfg_file[32] = DEFAULT_CF; 281 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 282 283 /* 284 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 285 * encouraged respectively). 286 */ 287 static unsigned int t4_fw_install = 1; 288 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 289 290 /* 291 * ASIC features that will be used. Disable the ones you don't want so that the 292 * chip resources aren't wasted on features that will not be used. 293 */ 294 static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 295 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 296 297 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 298 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 299 300 static int t4_toecaps_allowed = -1; 301 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 302 303 static int t4_rdmacaps_allowed = 0; 304 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 305 306 static int t4_iscsicaps_allowed = 0; 307 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 308 309 static int t4_fcoecaps_allowed = 0; 310 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 311 312 static int t5_write_combine = 0; 313 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 314 315 struct intrs_and_queues { 316 uint16_t intr_type; /* INTx, MSI, or MSI-X */ 317 uint16_t nirq; /* Total # of vectors */ 318 uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */ 319 uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */ 320 uint16_t ntxq10g; /* # of NIC txq's for each 10G port */ 321 uint16_t nrxq10g; /* # of NIC rxq's for each 10G port */ 322 uint16_t ntxq1g; /* # of NIC txq's for each 1G port */ 323 uint16_t nrxq1g; /* # of NIC rxq's for each 1G port */ 324 uint16_t rsrv_noflowq; /* Flag whether to reserve queue 0 */ 325 #ifdef TCP_OFFLOAD 326 uint16_t nofldtxq10g; /* # of TOE txq's for each 10G port */ 327 uint16_t nofldrxq10g; /* # of TOE rxq's for each 10G port */ 328 uint16_t nofldtxq1g; /* # of TOE txq's for each 1G port */ 329 uint16_t nofldrxq1g; /* # of TOE rxq's for each 1G port */ 330 #endif 331 #ifdef DEV_NETMAP 332 uint16_t nnmtxq10g; /* # of netmap txq's for each 10G port */ 333 uint16_t nnmrxq10g; /* # of netmap rxq's for each 10G port */ 334 uint16_t nnmtxq1g; /* # of netmap txq's for each 1G port */ 335 uint16_t nnmrxq1g; /* # of netmap rxq's for each 1G port */ 336 #endif 337 }; 338 339 struct filter_entry { 340 uint32_t valid:1; /* filter allocated and valid */ 341 uint32_t locked:1; /* filter is administratively locked */ 342 uint32_t pending:1; /* filter action is pending firmware reply */ 343 uint32_t smtidx:8; /* Source MAC Table index for smac */ 344 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 345 346 struct t4_filter_specification fs; 347 }; 348 349 static int map_bars_0_and_4(struct adapter *); 350 static int map_bar_2(struct adapter *); 351 static void setup_memwin(struct adapter *); 352 static int validate_mem_range(struct adapter *, uint32_t, int); 353 static int fwmtype_to_hwmtype(int); 354 static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 355 uint32_t *); 356 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 357 static uint32_t position_memwin(struct adapter *, int, uint32_t); 358 static int cfg_itype_and_nqueues(struct adapter *, int, int, 359 struct intrs_and_queues *); 360 static int prep_firmware(struct adapter *); 361 static int partition_resources(struct adapter *, const struct firmware *, 362 const char *); 363 static int get_params__pre_init(struct adapter *); 364 static int get_params__post_init(struct adapter *); 365 static int set_params__post_init(struct adapter *); 366 static void t4_set_desc(struct adapter *); 367 static void build_medialist(struct port_info *, struct ifmedia *); 368 static int cxgbe_init_synchronized(struct port_info *); 369 static int cxgbe_uninit_synchronized(struct port_info *); 370 static int setup_intr_handlers(struct adapter *); 371 static void quiesce_eq(struct adapter *, struct sge_eq *); 372 static void quiesce_iq(struct adapter *, struct sge_iq *); 373 static void quiesce_fl(struct adapter *, struct sge_fl *); 374 static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 375 driver_intr_t *, void *, char *); 376 static int t4_free_irq(struct adapter *, struct irq *); 377 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 378 unsigned int); 379 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 380 static void cxgbe_tick(void *); 381 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 382 static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 383 struct mbuf *); 384 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 385 static int fw_msg_not_handled(struct adapter *, const __be64 *); 386 static int t4_sysctls(struct adapter *); 387 static int cxgbe_sysctls(struct port_info *); 388 static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 389 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 390 static int sysctl_btphy(SYSCTL_HANDLER_ARGS); 391 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); 392 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 393 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 394 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 395 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 396 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 397 static int sysctl_temperature(SYSCTL_HANDLER_ARGS); 398 #ifdef SBUF_DRAIN 399 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 400 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 401 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 402 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); 403 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); 404 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 405 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 406 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 407 static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 408 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 409 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 410 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 411 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); 412 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 413 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); 414 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 415 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 416 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 417 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 418 static int sysctl_tids(SYSCTL_HANDLER_ARGS); 419 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 420 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 421 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 422 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 423 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 424 #endif 425 static inline void txq_start(struct ifnet *, struct sge_txq *); 426 static uint32_t fconf_to_mode(uint32_t); 427 static uint32_t mode_to_fconf(uint32_t); 428 static uint32_t fspec_to_fconf(struct t4_filter_specification *); 429 static int get_filter_mode(struct adapter *, uint32_t *); 430 static int set_filter_mode(struct adapter *, uint32_t); 431 static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 432 static int get_filter(struct adapter *, struct t4_filter *); 433 static int set_filter(struct adapter *, struct t4_filter *); 434 static int del_filter(struct adapter *, struct t4_filter *); 435 static void clear_filter(struct filter_entry *); 436 static int set_filter_wr(struct adapter *, int); 437 static int del_filter_wr(struct adapter *, int); 438 static int get_sge_context(struct adapter *, struct t4_sge_context *); 439 static int load_fw(struct adapter *, struct t4_data *); 440 static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 441 static int read_i2c(struct adapter *, struct t4_i2c_data *); 442 static int set_sched_class(struct adapter *, struct t4_sched_params *); 443 static int set_sched_queue(struct adapter *, struct t4_sched_queue *); 444 #ifdef TCP_OFFLOAD 445 static int toe_capability(struct port_info *, int); 446 #endif 447 static int mod_event(module_t, int, void *); 448 449 struct { 450 uint16_t device; 451 char *desc; 452 } t4_pciids[] = { 453 {0xa000, "Chelsio Terminator 4 FPGA"}, 454 {0x4400, "Chelsio T440-dbg"}, 455 {0x4401, "Chelsio T420-CR"}, 456 {0x4402, "Chelsio T422-CR"}, 457 {0x4403, "Chelsio T440-CR"}, 458 {0x4404, "Chelsio T420-BCH"}, 459 {0x4405, "Chelsio T440-BCH"}, 460 {0x4406, "Chelsio T440-CH"}, 461 {0x4407, "Chelsio T420-SO"}, 462 {0x4408, "Chelsio T420-CX"}, 463 {0x4409, "Chelsio T420-BT"}, 464 {0x440a, "Chelsio T404-BT"}, 465 {0x440e, "Chelsio T440-LP-CR"}, 466 }, t5_pciids[] = { 467 {0xb000, "Chelsio Terminator 5 FPGA"}, 468 {0x5400, "Chelsio T580-dbg"}, 469 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ 470 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ 471 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ 472 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ 473 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ 474 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ 475 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ 476 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ 477 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 478 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ 479 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ 480 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ 481 #ifdef notyet 482 {0x5404, "Chelsio T520-BCH"}, 483 {0x5405, "Chelsio T540-BCH"}, 484 {0x5406, "Chelsio T540-CH"}, 485 {0x5408, "Chelsio T520-CX"}, 486 {0x540b, "Chelsio B520-SR"}, 487 {0x540c, "Chelsio B504-BT"}, 488 {0x540f, "Chelsio Amsterdam"}, 489 {0x5413, "Chelsio T580-CHR"}, 490 #endif 491 }; 492 493 #ifdef TCP_OFFLOAD 494 /* 495 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 496 * exactly the same for both rxq and ofld_rxq. 497 */ 498 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 499 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 500 #endif 501 502 /* No easy way to include t4_msg.h before adapter.h so we check this way */ 503 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 504 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 505 506 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); 507 508 static int 509 t4_probe(device_t dev) 510 { 511 int i; 512 uint16_t v = pci_get_vendor(dev); 513 uint16_t d = pci_get_device(dev); 514 uint8_t f = pci_get_function(dev); 515 516 if (v != PCI_VENDOR_ID_CHELSIO) 517 return (ENXIO); 518 519 /* Attach only to PF0 of the FPGA */ 520 if (d == 0xa000 && f != 0) 521 return (ENXIO); 522 523 for (i = 0; i < nitems(t4_pciids); i++) { 524 if (d == t4_pciids[i].device) { 525 device_set_desc(dev, t4_pciids[i].desc); 526 return (BUS_PROBE_DEFAULT); 527 } 528 } 529 530 return (ENXIO); 531 } 532 533 static int 534 t5_probe(device_t dev) 535 { 536 int i; 537 uint16_t v = pci_get_vendor(dev); 538 uint16_t d = pci_get_device(dev); 539 uint8_t f = pci_get_function(dev); 540 541 if (v != PCI_VENDOR_ID_CHELSIO) 542 return (ENXIO); 543 544 /* Attach only to PF0 of the FPGA */ 545 if (d == 0xb000 && f != 0) 546 return (ENXIO); 547 548 for (i = 0; i < nitems(t5_pciids); i++) { 549 if (d == t5_pciids[i].device) { 550 device_set_desc(dev, t5_pciids[i].desc); 551 return (BUS_PROBE_DEFAULT); 552 } 553 } 554 555 return (ENXIO); 556 } 557 558 static int 559 t4_attach(device_t dev) 560 { 561 struct adapter *sc; 562 int rc = 0, i, n10g, n1g, rqidx, tqidx; 563 struct intrs_and_queues iaq; 564 struct sge *s; 565 #ifdef TCP_OFFLOAD 566 int ofld_rqidx, ofld_tqidx; 567 #endif 568 #ifdef DEV_NETMAP 569 int nm_rqidx, nm_tqidx; 570 #endif 571 const char *pcie_ts; 572 573 sc = device_get_softc(dev); 574 sc->dev = dev; 575 576 pci_enable_busmaster(dev); 577 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 578 uint32_t v; 579 580 pci_set_max_read_req(dev, 4096); 581 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 582 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 583 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 584 } 585 586 sc->traceq = -1; 587 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); 588 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", 589 device_get_nameunit(dev)); 590 591 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 592 device_get_nameunit(dev)); 593 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 594 sx_xlock(&t4_list_lock); 595 SLIST_INSERT_HEAD(&t4_list, sc, link); 596 sx_xunlock(&t4_list_lock); 597 598 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 599 TAILQ_INIT(&sc->sfl); 600 callout_init(&sc->sfl_callout, CALLOUT_MPSAFE); 601 602 rc = map_bars_0_and_4(sc); 603 if (rc != 0) 604 goto done; /* error message displayed already */ 605 606 /* 607 * This is the real PF# to which we're attaching. Works from within PCI 608 * passthrough environments too, where pci_get_function() could return a 609 * different PF# depending on the passthrough configuration. We need to 610 * use the real PF# in all our communication with the firmware. 611 */ 612 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 613 sc->mbox = sc->pf; 614 615 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 616 sc->an_handler = an_not_handled; 617 for (i = 0; i < nitems(sc->cpl_handler); i++) 618 sc->cpl_handler[i] = cpl_not_handled; 619 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 620 sc->fw_msg_handler[i] = fw_msg_not_handled; 621 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 622 t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt); 623 t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt); 624 t4_init_sge_cpl_handlers(sc); 625 626 /* Prepare the adapter for operation */ 627 rc = -t4_prep_adapter(sc); 628 if (rc != 0) { 629 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 630 goto done; 631 } 632 633 /* 634 * Do this really early, with the memory windows set up even before the 635 * character device. The userland tool's register i/o and mem read 636 * will work even in "recovery mode". 637 */ 638 setup_memwin(sc); 639 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 640 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 641 device_get_nameunit(dev)); 642 if (sc->cdev == NULL) 643 device_printf(dev, "failed to create nexus char device.\n"); 644 else 645 sc->cdev->si_drv1 = sc; 646 647 /* Go no further if recovery mode has been requested. */ 648 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 649 device_printf(dev, "recovery mode.\n"); 650 goto done; 651 } 652 653 /* Prepare the firmware for operation */ 654 rc = prep_firmware(sc); 655 if (rc != 0) 656 goto done; /* error message displayed already */ 657 658 rc = get_params__post_init(sc); 659 if (rc != 0) 660 goto done; /* error message displayed already */ 661 662 rc = set_params__post_init(sc); 663 if (rc != 0) 664 goto done; /* error message displayed already */ 665 666 rc = map_bar_2(sc); 667 if (rc != 0) 668 goto done; /* error message displayed already */ 669 670 rc = t4_create_dma_tag(sc); 671 if (rc != 0) 672 goto done; /* error message displayed already */ 673 674 /* 675 * First pass over all the ports - allocate VIs and initialize some 676 * basic parameters like mac address, port type, etc. We also figure 677 * out whether a port is 10G or 1G and use that information when 678 * calculating how many interrupts to attempt to allocate. 679 */ 680 n10g = n1g = 0; 681 for_each_port(sc, i) { 682 struct port_info *pi; 683 684 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 685 sc->port[i] = pi; 686 687 /* These must be set before t4_port_init */ 688 pi->adapter = sc; 689 pi->port_id = i; 690 691 /* Allocate the vi and initialize parameters like mac addr */ 692 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 693 if (rc != 0) { 694 device_printf(dev, "unable to initialize port %d: %d\n", 695 i, rc); 696 free(pi, M_CXGBE); 697 sc->port[i] = NULL; 698 goto done; 699 } 700 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 701 if (rc != 0) { 702 device_printf(dev, "port %d l1cfg failed: %d\n", i, rc); 703 free(pi, M_CXGBE); 704 sc->port[i] = NULL; 705 goto done; 706 } 707 708 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 709 device_get_nameunit(dev), i); 710 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 711 sc->chan_map[pi->tx_chan] = i; 712 713 if (is_10G_port(pi) || is_40G_port(pi)) { 714 n10g++; 715 pi->tmr_idx = t4_tmr_idx_10g; 716 pi->pktc_idx = t4_pktc_idx_10g; 717 } else { 718 n1g++; 719 pi->tmr_idx = t4_tmr_idx_1g; 720 pi->pktc_idx = t4_pktc_idx_1g; 721 } 722 723 pi->xact_addr_filt = -1; 724 pi->linkdnrc = -1; 725 726 pi->qsize_rxq = t4_qsize_rxq; 727 pi->qsize_txq = t4_qsize_txq; 728 729 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 730 if (pi->dev == NULL) { 731 device_printf(dev, 732 "failed to add device for port %d.\n", i); 733 rc = ENXIO; 734 goto done; 735 } 736 device_set_softc(pi->dev, pi); 737 } 738 739 /* 740 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 741 */ 742 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 743 if (rc != 0) 744 goto done; /* error message displayed already */ 745 746 sc->intr_type = iaq.intr_type; 747 sc->intr_count = iaq.nirq; 748 749 s = &sc->sge; 750 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 751 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 752 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 753 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 754 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 755 #ifdef TCP_OFFLOAD 756 if (is_offload(sc)) { 757 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 758 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 759 s->neq += s->nofldtxq + s->nofldrxq; 760 s->niq += s->nofldrxq; 761 762 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 763 M_CXGBE, M_ZERO | M_WAITOK); 764 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 765 M_CXGBE, M_ZERO | M_WAITOK); 766 } 767 #endif 768 #ifdef DEV_NETMAP 769 s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g; 770 s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g; 771 s->neq += s->nnmtxq + s->nnmrxq; 772 s->niq += s->nnmrxq; 773 774 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq), 775 M_CXGBE, M_ZERO | M_WAITOK); 776 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq), 777 M_CXGBE, M_ZERO | M_WAITOK); 778 #endif 779 780 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 781 M_ZERO | M_WAITOK); 782 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 783 M_ZERO | M_WAITOK); 784 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 785 M_ZERO | M_WAITOK); 786 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 787 M_ZERO | M_WAITOK); 788 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 789 M_ZERO | M_WAITOK); 790 791 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 792 M_ZERO | M_WAITOK); 793 794 t4_init_l2t(sc, M_WAITOK); 795 796 /* 797 * Second pass over the ports. This time we know the number of rx and 798 * tx queues that each port should get. 799 */ 800 rqidx = tqidx = 0; 801 #ifdef TCP_OFFLOAD 802 ofld_rqidx = ofld_tqidx = 0; 803 #endif 804 #ifdef DEV_NETMAP 805 nm_rqidx = nm_tqidx = 0; 806 #endif 807 for_each_port(sc, i) { 808 struct port_info *pi = sc->port[i]; 809 810 if (pi == NULL) 811 continue; 812 813 pi->first_rxq = rqidx; 814 pi->first_txq = tqidx; 815 if (is_10G_port(pi) || is_40G_port(pi)) { 816 pi->flags |= iaq.intr_flags_10g; 817 pi->nrxq = iaq.nrxq10g; 818 pi->ntxq = iaq.ntxq10g; 819 } else { 820 pi->flags |= iaq.intr_flags_1g; 821 pi->nrxq = iaq.nrxq1g; 822 pi->ntxq = iaq.ntxq1g; 823 } 824 825 if (pi->ntxq > 1) 826 pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0; 827 else 828 pi->rsrv_noflowq = 0; 829 830 rqidx += pi->nrxq; 831 tqidx += pi->ntxq; 832 #ifdef TCP_OFFLOAD 833 if (is_offload(sc)) { 834 pi->first_ofld_rxq = ofld_rqidx; 835 pi->first_ofld_txq = ofld_tqidx; 836 if (is_10G_port(pi) || is_40G_port(pi)) { 837 pi->nofldrxq = iaq.nofldrxq10g; 838 pi->nofldtxq = iaq.nofldtxq10g; 839 } else { 840 pi->nofldrxq = iaq.nofldrxq1g; 841 pi->nofldtxq = iaq.nofldtxq1g; 842 } 843 ofld_rqidx += pi->nofldrxq; 844 ofld_tqidx += pi->nofldtxq; 845 } 846 #endif 847 #ifdef DEV_NETMAP 848 pi->first_nm_rxq = nm_rqidx; 849 pi->first_nm_txq = nm_tqidx; 850 if (is_10G_port(pi) || is_40G_port(pi)) { 851 pi->nnmrxq = iaq.nnmrxq10g; 852 pi->nnmtxq = iaq.nnmtxq10g; 853 } else { 854 pi->nnmrxq = iaq.nnmrxq1g; 855 pi->nnmtxq = iaq.nnmtxq1g; 856 } 857 nm_rqidx += pi->nnmrxq; 858 nm_tqidx += pi->nnmtxq; 859 #endif 860 } 861 862 rc = setup_intr_handlers(sc); 863 if (rc != 0) { 864 device_printf(dev, 865 "failed to setup interrupt handlers: %d\n", rc); 866 goto done; 867 } 868 869 rc = bus_generic_attach(dev); 870 if (rc != 0) { 871 device_printf(dev, 872 "failed to attach all child ports: %d\n", rc); 873 goto done; 874 } 875 876 switch (sc->params.pci.speed) { 877 case 0x1: 878 pcie_ts = "2.5"; 879 break; 880 case 0x2: 881 pcie_ts = "5.0"; 882 break; 883 case 0x3: 884 pcie_ts = "8.0"; 885 break; 886 default: 887 pcie_ts = "??"; 888 break; 889 } 890 device_printf(dev, 891 "PCIe x%d (%s GTS/s) (%d), %d ports, %d %s interrupt%s, %d eq, %d iq\n", 892 sc->params.pci.width, pcie_ts, sc->params.pci.speed, 893 sc->params.nports, sc->intr_count, 894 sc->intr_type == INTR_MSIX ? "MSI-X" : 895 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 896 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 897 898 t4_set_desc(sc); 899 900 done: 901 if (rc != 0 && sc->cdev) { 902 /* cdev was created and so cxgbetool works; recover that way. */ 903 device_printf(dev, 904 "error during attach, adapter is now in recovery mode.\n"); 905 rc = 0; 906 } 907 908 if (rc != 0) 909 t4_detach(dev); 910 else 911 t4_sysctls(sc); 912 913 return (rc); 914 } 915 916 /* 917 * Idempotent 918 */ 919 static int 920 t4_detach(device_t dev) 921 { 922 struct adapter *sc; 923 struct port_info *pi; 924 int i, rc; 925 926 sc = device_get_softc(dev); 927 928 if (sc->flags & FULL_INIT_DONE) 929 t4_intr_disable(sc); 930 931 if (sc->cdev) { 932 destroy_dev(sc->cdev); 933 sc->cdev = NULL; 934 } 935 936 rc = bus_generic_detach(dev); 937 if (rc) { 938 device_printf(dev, 939 "failed to detach child devices: %d\n", rc); 940 return (rc); 941 } 942 943 for (i = 0; i < sc->intr_count; i++) 944 t4_free_irq(sc, &sc->irq[i]); 945 946 for (i = 0; i < MAX_NPORTS; i++) { 947 pi = sc->port[i]; 948 if (pi) { 949 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid); 950 if (pi->dev) 951 device_delete_child(dev, pi->dev); 952 953 mtx_destroy(&pi->pi_lock); 954 free(pi, M_CXGBE); 955 } 956 } 957 958 if (sc->flags & FULL_INIT_DONE) 959 adapter_full_uninit(sc); 960 961 if (sc->flags & FW_OK) 962 t4_fw_bye(sc, sc->mbox); 963 964 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 965 pci_release_msi(dev); 966 967 if (sc->regs_res) 968 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 969 sc->regs_res); 970 971 if (sc->udbs_res) 972 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 973 sc->udbs_res); 974 975 if (sc->msix_res) 976 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 977 sc->msix_res); 978 979 if (sc->l2t) 980 t4_free_l2t(sc->l2t); 981 982 #ifdef TCP_OFFLOAD 983 free(sc->sge.ofld_rxq, M_CXGBE); 984 free(sc->sge.ofld_txq, M_CXGBE); 985 #endif 986 #ifdef DEV_NETMAP 987 free(sc->sge.nm_rxq, M_CXGBE); 988 free(sc->sge.nm_txq, M_CXGBE); 989 #endif 990 free(sc->irq, M_CXGBE); 991 free(sc->sge.rxq, M_CXGBE); 992 free(sc->sge.txq, M_CXGBE); 993 free(sc->sge.ctrlq, M_CXGBE); 994 free(sc->sge.iqmap, M_CXGBE); 995 free(sc->sge.eqmap, M_CXGBE); 996 free(sc->tids.ftid_tab, M_CXGBE); 997 t4_destroy_dma_tag(sc); 998 if (mtx_initialized(&sc->sc_lock)) { 999 sx_xlock(&t4_list_lock); 1000 SLIST_REMOVE(&t4_list, sc, adapter, link); 1001 sx_xunlock(&t4_list_lock); 1002 mtx_destroy(&sc->sc_lock); 1003 } 1004 1005 if (mtx_initialized(&sc->tids.ftid_lock)) 1006 mtx_destroy(&sc->tids.ftid_lock); 1007 if (mtx_initialized(&sc->sfl_lock)) 1008 mtx_destroy(&sc->sfl_lock); 1009 if (mtx_initialized(&sc->ifp_lock)) 1010 mtx_destroy(&sc->ifp_lock); 1011 1012 bzero(sc, sizeof(*sc)); 1013 1014 return (0); 1015 } 1016 1017 static int 1018 cxgbe_probe(device_t dev) 1019 { 1020 char buf[128]; 1021 struct port_info *pi = device_get_softc(dev); 1022 1023 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 1024 device_set_desc_copy(dev, buf); 1025 1026 return (BUS_PROBE_DEFAULT); 1027 } 1028 1029 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 1030 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 1031 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS) 1032 #define T4_CAP_ENABLE (T4_CAP) 1033 1034 static int 1035 cxgbe_attach(device_t dev) 1036 { 1037 struct port_info *pi = device_get_softc(dev); 1038 struct ifnet *ifp; 1039 char *s; 1040 int n, o; 1041 1042 /* Allocate an ifnet and set it up */ 1043 ifp = if_alloc(IFT_ETHER); 1044 if (ifp == NULL) { 1045 device_printf(dev, "Cannot allocate ifnet\n"); 1046 return (ENOMEM); 1047 } 1048 pi->ifp = ifp; 1049 ifp->if_softc = pi; 1050 1051 callout_init(&pi->tick, CALLOUT_MPSAFE); 1052 1053 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1054 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1055 1056 ifp->if_init = cxgbe_init; 1057 ifp->if_ioctl = cxgbe_ioctl; 1058 ifp->if_transmit = cxgbe_transmit; 1059 ifp->if_qflush = cxgbe_qflush; 1060 1061 ifp->if_capabilities = T4_CAP; 1062 #ifdef TCP_OFFLOAD 1063 if (is_offload(pi->adapter)) 1064 ifp->if_capabilities |= IFCAP_TOE; 1065 #endif 1066 ifp->if_capenable = T4_CAP_ENABLE; 1067 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1068 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1069 1070 /* Initialize ifmedia for this port */ 1071 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 1072 cxgbe_media_status); 1073 build_medialist(pi, &pi->media); 1074 1075 pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 1076 EVENTHANDLER_PRI_ANY); 1077 1078 ether_ifattach(ifp, pi->hw_addr); 1079 1080 n = 128; 1081 s = malloc(n, M_CXGBE, M_WAITOK); 1082 o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq); 1083 MPASS(n > o); 1084 #ifdef TCP_OFFLOAD 1085 if (is_offload(pi->adapter)) { 1086 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)", 1087 pi->nofldtxq, pi->nofldrxq); 1088 MPASS(n > o); 1089 } 1090 #endif 1091 #ifdef DEV_NETMAP 1092 o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq, 1093 pi->nnmrxq); 1094 MPASS(n > o); 1095 #endif 1096 device_printf(dev, "%s\n", s); 1097 free(s, M_CXGBE); 1098 1099 #ifdef DEV_NETMAP 1100 /* nm_media handled here to keep implementation private to this file */ 1101 ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change, 1102 cxgbe_media_status); 1103 build_medialist(pi, &pi->nm_media); 1104 create_netmap_ifnet(pi); /* logs errors it something fails */ 1105 #endif 1106 cxgbe_sysctls(pi); 1107 1108 return (0); 1109 } 1110 1111 static int 1112 cxgbe_detach(device_t dev) 1113 { 1114 struct port_info *pi = device_get_softc(dev); 1115 struct adapter *sc = pi->adapter; 1116 struct ifnet *ifp = pi->ifp; 1117 1118 /* Tell if_ioctl and if_init that the port is going away */ 1119 ADAPTER_LOCK(sc); 1120 SET_DOOMED(pi); 1121 wakeup(&sc->flags); 1122 while (IS_BUSY(sc)) 1123 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 1124 SET_BUSY(sc); 1125 #ifdef INVARIANTS 1126 sc->last_op = "t4detach"; 1127 sc->last_op_thr = curthread; 1128 #endif 1129 ADAPTER_UNLOCK(sc); 1130 1131 if (pi->flags & HAS_TRACEQ) { 1132 sc->traceq = -1; /* cloner should not create ifnet */ 1133 t4_tracer_port_detach(sc); 1134 } 1135 1136 if (pi->vlan_c) 1137 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c); 1138 1139 PORT_LOCK(pi); 1140 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1141 callout_stop(&pi->tick); 1142 PORT_UNLOCK(pi); 1143 callout_drain(&pi->tick); 1144 1145 /* Let detach proceed even if these fail. */ 1146 cxgbe_uninit_synchronized(pi); 1147 port_full_uninit(pi); 1148 1149 ifmedia_removeall(&pi->media); 1150 ether_ifdetach(pi->ifp); 1151 if_free(pi->ifp); 1152 1153 #ifdef DEV_NETMAP 1154 /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */ 1155 destroy_netmap_ifnet(pi); 1156 #endif 1157 1158 ADAPTER_LOCK(sc); 1159 CLR_BUSY(sc); 1160 wakeup(&sc->flags); 1161 ADAPTER_UNLOCK(sc); 1162 1163 return (0); 1164 } 1165 1166 static void 1167 cxgbe_init(void *arg) 1168 { 1169 struct port_info *pi = arg; 1170 struct adapter *sc = pi->adapter; 1171 1172 if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0) 1173 return; 1174 cxgbe_init_synchronized(pi); 1175 end_synchronized_op(sc, 0); 1176 } 1177 1178 static int 1179 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1180 { 1181 int rc = 0, mtu, flags; 1182 struct port_info *pi = ifp->if_softc; 1183 struct adapter *sc = pi->adapter; 1184 struct ifreq *ifr = (struct ifreq *)data; 1185 uint32_t mask; 1186 1187 switch (cmd) { 1188 case SIOCSIFMTU: 1189 mtu = ifr->ifr_mtu; 1190 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1191 return (EINVAL); 1192 1193 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu"); 1194 if (rc) 1195 return (rc); 1196 ifp->if_mtu = mtu; 1197 if (pi->flags & PORT_INIT_DONE) { 1198 t4_update_fl_bufsize(ifp); 1199 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1200 rc = update_mac_settings(ifp, XGMAC_MTU); 1201 } 1202 end_synchronized_op(sc, 0); 1203 break; 1204 1205 case SIOCSIFFLAGS: 1206 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4flg"); 1207 if (rc) 1208 return (rc); 1209 1210 if (ifp->if_flags & IFF_UP) { 1211 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1212 flags = pi->if_flags; 1213 if ((ifp->if_flags ^ flags) & 1214 (IFF_PROMISC | IFF_ALLMULTI)) { 1215 rc = update_mac_settings(ifp, 1216 XGMAC_PROMISC | XGMAC_ALLMULTI); 1217 } 1218 } else 1219 rc = cxgbe_init_synchronized(pi); 1220 pi->if_flags = ifp->if_flags; 1221 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1222 rc = cxgbe_uninit_synchronized(pi); 1223 end_synchronized_op(sc, 0); 1224 break; 1225 1226 case SIOCADDMULTI: 1227 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1228 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi"); 1229 if (rc) 1230 return (rc); 1231 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1232 rc = update_mac_settings(ifp, XGMAC_MCADDRS); 1233 end_synchronized_op(sc, LOCK_HELD); 1234 break; 1235 1236 case SIOCSIFCAP: 1237 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap"); 1238 if (rc) 1239 return (rc); 1240 1241 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1242 if (mask & IFCAP_TXCSUM) { 1243 ifp->if_capenable ^= IFCAP_TXCSUM; 1244 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1245 1246 if (IFCAP_TSO4 & ifp->if_capenable && 1247 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1248 ifp->if_capenable &= ~IFCAP_TSO4; 1249 if_printf(ifp, 1250 "tso4 disabled due to -txcsum.\n"); 1251 } 1252 } 1253 if (mask & IFCAP_TXCSUM_IPV6) { 1254 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1255 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1256 1257 if (IFCAP_TSO6 & ifp->if_capenable && 1258 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1259 ifp->if_capenable &= ~IFCAP_TSO6; 1260 if_printf(ifp, 1261 "tso6 disabled due to -txcsum6.\n"); 1262 } 1263 } 1264 if (mask & IFCAP_RXCSUM) 1265 ifp->if_capenable ^= IFCAP_RXCSUM; 1266 if (mask & IFCAP_RXCSUM_IPV6) 1267 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1268 1269 /* 1270 * Note that we leave CSUM_TSO alone (it is always set). The 1271 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1272 * sending a TSO request our way, so it's sufficient to toggle 1273 * IFCAP_TSOx only. 1274 */ 1275 if (mask & IFCAP_TSO4) { 1276 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1277 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1278 if_printf(ifp, "enable txcsum first.\n"); 1279 rc = EAGAIN; 1280 goto fail; 1281 } 1282 ifp->if_capenable ^= IFCAP_TSO4; 1283 } 1284 if (mask & IFCAP_TSO6) { 1285 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1286 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1287 if_printf(ifp, "enable txcsum6 first.\n"); 1288 rc = EAGAIN; 1289 goto fail; 1290 } 1291 ifp->if_capenable ^= IFCAP_TSO6; 1292 } 1293 if (mask & IFCAP_LRO) { 1294 #if defined(INET) || defined(INET6) 1295 int i; 1296 struct sge_rxq *rxq; 1297 1298 ifp->if_capenable ^= IFCAP_LRO; 1299 for_each_rxq(pi, i, rxq) { 1300 if (ifp->if_capenable & IFCAP_LRO) 1301 rxq->iq.flags |= IQ_LRO_ENABLED; 1302 else 1303 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1304 } 1305 #endif 1306 } 1307 #ifdef TCP_OFFLOAD 1308 if (mask & IFCAP_TOE) { 1309 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1310 1311 rc = toe_capability(pi, enable); 1312 if (rc != 0) 1313 goto fail; 1314 1315 ifp->if_capenable ^= mask; 1316 } 1317 #endif 1318 if (mask & IFCAP_VLAN_HWTAGGING) { 1319 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1320 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1321 rc = update_mac_settings(ifp, XGMAC_VLANEX); 1322 } 1323 if (mask & IFCAP_VLAN_MTU) { 1324 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1325 1326 /* Need to find out how to disable auto-mtu-inflation */ 1327 } 1328 if (mask & IFCAP_VLAN_HWTSO) 1329 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1330 if (mask & IFCAP_VLAN_HWCSUM) 1331 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1332 1333 #ifdef VLAN_CAPABILITIES 1334 VLAN_CAPABILITIES(ifp); 1335 #endif 1336 fail: 1337 end_synchronized_op(sc, 0); 1338 break; 1339 1340 case SIOCSIFMEDIA: 1341 case SIOCGIFMEDIA: 1342 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1343 break; 1344 1345 default: 1346 rc = ether_ioctl(ifp, cmd, data); 1347 } 1348 1349 return (rc); 1350 } 1351 1352 static int 1353 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1354 { 1355 struct port_info *pi = ifp->if_softc; 1356 struct adapter *sc = pi->adapter; 1357 struct sge_txq *txq = &sc->sge.txq[pi->first_txq]; 1358 struct buf_ring *br; 1359 int rc; 1360 1361 M_ASSERTPKTHDR(m); 1362 1363 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1364 m_freem(m); 1365 return (ENETDOWN); 1366 } 1367 1368 if (m->m_flags & M_FLOWID) 1369 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) 1370 + pi->rsrv_noflowq); 1371 br = txq->br; 1372 1373 if (TXQ_TRYLOCK(txq) == 0) { 1374 struct sge_eq *eq = &txq->eq; 1375 1376 /* 1377 * It is possible that t4_eth_tx finishes up and releases the 1378 * lock between the TRYLOCK above and the drbr_enqueue here. We 1379 * need to make sure that this mbuf doesn't just sit there in 1380 * the drbr. 1381 */ 1382 1383 rc = drbr_enqueue(ifp, br, m); 1384 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 && 1385 !(eq->flags & EQ_DOOMED)) 1386 callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq); 1387 return (rc); 1388 } 1389 1390 /* 1391 * txq->m is the mbuf that is held up due to a temporary shortage of 1392 * resources and it should be put on the wire first. Then what's in 1393 * drbr and finally the mbuf that was just passed in to us. 1394 * 1395 * Return code should indicate the fate of the mbuf that was passed in 1396 * this time. 1397 */ 1398 1399 TXQ_LOCK_ASSERT_OWNED(txq); 1400 if (drbr_needs_enqueue(ifp, br) || txq->m) { 1401 1402 /* Queued for transmission. */ 1403 1404 rc = drbr_enqueue(ifp, br, m); 1405 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 1406 (void) t4_eth_tx(ifp, txq, m); 1407 TXQ_UNLOCK(txq); 1408 return (rc); 1409 } 1410 1411 /* Direct transmission. */ 1412 rc = t4_eth_tx(ifp, txq, m); 1413 if (rc != 0 && txq->m) 1414 rc = 0; /* held, will be transmitted soon (hopefully) */ 1415 1416 TXQ_UNLOCK(txq); 1417 return (rc); 1418 } 1419 1420 static void 1421 cxgbe_qflush(struct ifnet *ifp) 1422 { 1423 struct port_info *pi = ifp->if_softc; 1424 struct sge_txq *txq; 1425 int i; 1426 struct mbuf *m; 1427 1428 /* queues do not exist if !PORT_INIT_DONE. */ 1429 if (pi->flags & PORT_INIT_DONE) { 1430 for_each_txq(pi, i, txq) { 1431 TXQ_LOCK(txq); 1432 m_freem(txq->m); 1433 txq->m = NULL; 1434 while ((m = buf_ring_dequeue_sc(txq->br)) != NULL) 1435 m_freem(m); 1436 TXQ_UNLOCK(txq); 1437 } 1438 } 1439 if_qflush(ifp); 1440 } 1441 1442 static int 1443 cxgbe_media_change(struct ifnet *ifp) 1444 { 1445 struct port_info *pi = ifp->if_softc; 1446 1447 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1448 1449 return (EOPNOTSUPP); 1450 } 1451 1452 static void 1453 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1454 { 1455 struct port_info *pi = ifp->if_softc; 1456 struct ifmedia *media = NULL; 1457 struct ifmedia_entry *cur; 1458 int speed = pi->link_cfg.speed; 1459 int data = (pi->port_type << 8) | pi->mod_type; 1460 1461 if (ifp == pi->ifp) 1462 media = &pi->media; 1463 #ifdef DEV_NETMAP 1464 else if (ifp == pi->nm_ifp) 1465 media = &pi->nm_media; 1466 #endif 1467 MPASS(media != NULL); 1468 1469 cur = media->ifm_cur; 1470 if (cur->ifm_data != data) { 1471 build_medialist(pi, media); 1472 cur = media->ifm_cur; 1473 } 1474 1475 ifmr->ifm_status = IFM_AVALID; 1476 if (!pi->link_cfg.link_ok) 1477 return; 1478 1479 ifmr->ifm_status |= IFM_ACTIVE; 1480 1481 /* active and current will differ iff current media is autoselect. */ 1482 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1483 return; 1484 1485 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1486 if (speed == SPEED_10000) 1487 ifmr->ifm_active |= IFM_10G_T; 1488 else if (speed == SPEED_1000) 1489 ifmr->ifm_active |= IFM_1000_T; 1490 else if (speed == SPEED_100) 1491 ifmr->ifm_active |= IFM_100_TX; 1492 else if (speed == SPEED_10) 1493 ifmr->ifm_active |= IFM_10_T; 1494 else 1495 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1496 speed)); 1497 } 1498 1499 void 1500 t4_fatal_err(struct adapter *sc) 1501 { 1502 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1503 t4_intr_disable(sc); 1504 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1505 device_get_nameunit(sc->dev)); 1506 } 1507 1508 static int 1509 map_bars_0_and_4(struct adapter *sc) 1510 { 1511 sc->regs_rid = PCIR_BAR(0); 1512 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1513 &sc->regs_rid, RF_ACTIVE); 1514 if (sc->regs_res == NULL) { 1515 device_printf(sc->dev, "cannot map registers.\n"); 1516 return (ENXIO); 1517 } 1518 sc->bt = rman_get_bustag(sc->regs_res); 1519 sc->bh = rman_get_bushandle(sc->regs_res); 1520 sc->mmio_len = rman_get_size(sc->regs_res); 1521 setbit(&sc->doorbells, DOORBELL_KDB); 1522 1523 sc->msix_rid = PCIR_BAR(4); 1524 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1525 &sc->msix_rid, RF_ACTIVE); 1526 if (sc->msix_res == NULL) { 1527 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1528 return (ENXIO); 1529 } 1530 1531 return (0); 1532 } 1533 1534 static int 1535 map_bar_2(struct adapter *sc) 1536 { 1537 1538 /* 1539 * T4: only iWARP driver uses the userspace doorbells. There is no need 1540 * to map it if RDMA is disabled. 1541 */ 1542 if (is_t4(sc) && sc->rdmacaps == 0) 1543 return (0); 1544 1545 sc->udbs_rid = PCIR_BAR(2); 1546 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1547 &sc->udbs_rid, RF_ACTIVE); 1548 if (sc->udbs_res == NULL) { 1549 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1550 return (ENXIO); 1551 } 1552 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1553 1554 if (is_t5(sc)) { 1555 setbit(&sc->doorbells, DOORBELL_UDB); 1556 #if defined(__i386__) || defined(__amd64__) 1557 if (t5_write_combine) { 1558 int rc; 1559 1560 /* 1561 * Enable write combining on BAR2. This is the 1562 * userspace doorbell BAR and is split into 128B 1563 * (UDBS_SEG_SIZE) doorbell regions, each associated 1564 * with an egress queue. The first 64B has the doorbell 1565 * and the second 64B can be used to submit a tx work 1566 * request with an implicit doorbell. 1567 */ 1568 1569 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1570 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1571 if (rc == 0) { 1572 clrbit(&sc->doorbells, DOORBELL_UDB); 1573 setbit(&sc->doorbells, DOORBELL_WCWR); 1574 setbit(&sc->doorbells, DOORBELL_UDBWC); 1575 } else { 1576 device_printf(sc->dev, 1577 "couldn't enable write combining: %d\n", 1578 rc); 1579 } 1580 1581 t4_write_reg(sc, A_SGE_STAT_CFG, 1582 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1583 } 1584 #endif 1585 } 1586 1587 return (0); 1588 } 1589 1590 static const struct memwin t4_memwin[] = { 1591 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1592 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1593 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1594 }; 1595 1596 static const struct memwin t5_memwin[] = { 1597 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1598 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1599 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1600 }; 1601 1602 static void 1603 setup_memwin(struct adapter *sc) 1604 { 1605 const struct memwin *mw; 1606 int i, n; 1607 uint32_t bar0; 1608 1609 if (is_t4(sc)) { 1610 /* 1611 * Read low 32b of bar0 indirectly via the hardware backdoor 1612 * mechanism. Works from within PCI passthrough environments 1613 * too, where rman_get_start() can return a different value. We 1614 * need to program the T4 memory window decoders with the actual 1615 * addresses that will be coming across the PCIe link. 1616 */ 1617 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1618 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1619 1620 mw = &t4_memwin[0]; 1621 n = nitems(t4_memwin); 1622 } else { 1623 /* T5 uses the relative offset inside the PCIe BAR */ 1624 bar0 = 0; 1625 1626 mw = &t5_memwin[0]; 1627 n = nitems(t5_memwin); 1628 } 1629 1630 for (i = 0; i < n; i++, mw++) { 1631 t4_write_reg(sc, 1632 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 1633 (mw->base + bar0) | V_BIR(0) | 1634 V_WINDOW(ilog2(mw->aperture) - 10)); 1635 } 1636 1637 /* flush */ 1638 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1639 } 1640 1641 /* 1642 * Verify that the memory range specified by the addr/len pair is valid and lies 1643 * entirely within a single region (EDCx or MCx). 1644 */ 1645 static int 1646 validate_mem_range(struct adapter *sc, uint32_t addr, int len) 1647 { 1648 uint32_t em, addr_len, maddr, mlen; 1649 1650 /* Memory can only be accessed in naturally aligned 4 byte units */ 1651 if (addr & 3 || len & 3 || len == 0) 1652 return (EINVAL); 1653 1654 /* Enabled memories */ 1655 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1656 if (em & F_EDRAM0_ENABLE) { 1657 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1658 maddr = G_EDRAM0_BASE(addr_len) << 20; 1659 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1660 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1661 addr + len <= maddr + mlen) 1662 return (0); 1663 } 1664 if (em & F_EDRAM1_ENABLE) { 1665 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1666 maddr = G_EDRAM1_BASE(addr_len) << 20; 1667 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1668 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1669 addr + len <= maddr + mlen) 1670 return (0); 1671 } 1672 if (em & F_EXT_MEM_ENABLE) { 1673 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1674 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1675 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1676 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1677 addr + len <= maddr + mlen) 1678 return (0); 1679 } 1680 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 1681 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1682 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1683 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1684 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1685 addr + len <= maddr + mlen) 1686 return (0); 1687 } 1688 1689 return (EFAULT); 1690 } 1691 1692 static int 1693 fwmtype_to_hwmtype(int mtype) 1694 { 1695 1696 switch (mtype) { 1697 case FW_MEMTYPE_EDC0: 1698 return (MEM_EDC0); 1699 case FW_MEMTYPE_EDC1: 1700 return (MEM_EDC1); 1701 case FW_MEMTYPE_EXTMEM: 1702 return (MEM_MC0); 1703 case FW_MEMTYPE_EXTMEM1: 1704 return (MEM_MC1); 1705 default: 1706 panic("%s: cannot translate fw mtype %d.", __func__, mtype); 1707 } 1708 } 1709 1710 /* 1711 * Verify that the memory range specified by the memtype/offset/len pair is 1712 * valid and lies entirely within the memtype specified. The global address of 1713 * the start of the range is returned in addr. 1714 */ 1715 static int 1716 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 1717 uint32_t *addr) 1718 { 1719 uint32_t em, addr_len, maddr, mlen; 1720 1721 /* Memory can only be accessed in naturally aligned 4 byte units */ 1722 if (off & 3 || len & 3 || len == 0) 1723 return (EINVAL); 1724 1725 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1726 switch (fwmtype_to_hwmtype(mtype)) { 1727 case MEM_EDC0: 1728 if (!(em & F_EDRAM0_ENABLE)) 1729 return (EINVAL); 1730 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1731 maddr = G_EDRAM0_BASE(addr_len) << 20; 1732 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1733 break; 1734 case MEM_EDC1: 1735 if (!(em & F_EDRAM1_ENABLE)) 1736 return (EINVAL); 1737 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1738 maddr = G_EDRAM1_BASE(addr_len) << 20; 1739 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1740 break; 1741 case MEM_MC: 1742 if (!(em & F_EXT_MEM_ENABLE)) 1743 return (EINVAL); 1744 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1745 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1746 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1747 break; 1748 case MEM_MC1: 1749 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 1750 return (EINVAL); 1751 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1752 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1753 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1754 break; 1755 default: 1756 return (EINVAL); 1757 } 1758 1759 if (mlen > 0 && off < mlen && off + len <= mlen) { 1760 *addr = maddr + off; /* global address */ 1761 return (0); 1762 } 1763 1764 return (EFAULT); 1765 } 1766 1767 static void 1768 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 1769 { 1770 const struct memwin *mw; 1771 1772 if (is_t4(sc)) { 1773 KASSERT(win >= 0 && win < nitems(t4_memwin), 1774 ("%s: incorrect memwin# (%d)", __func__, win)); 1775 mw = &t4_memwin[win]; 1776 } else { 1777 KASSERT(win >= 0 && win < nitems(t5_memwin), 1778 ("%s: incorrect memwin# (%d)", __func__, win)); 1779 mw = &t5_memwin[win]; 1780 } 1781 1782 if (base != NULL) 1783 *base = mw->base; 1784 if (aperture != NULL) 1785 *aperture = mw->aperture; 1786 } 1787 1788 /* 1789 * Positions the memory window such that it can be used to access the specified 1790 * address in the chip's address space. The return value is the offset of addr 1791 * from the start of the window. 1792 */ 1793 static uint32_t 1794 position_memwin(struct adapter *sc, int n, uint32_t addr) 1795 { 1796 uint32_t start, pf; 1797 uint32_t reg; 1798 1799 KASSERT(n >= 0 && n <= 3, 1800 ("%s: invalid window %d.", __func__, n)); 1801 KASSERT((addr & 3) == 0, 1802 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 1803 1804 if (is_t4(sc)) { 1805 pf = 0; 1806 start = addr & ~0xf; /* start must be 16B aligned */ 1807 } else { 1808 pf = V_PFNUM(sc->pf); 1809 start = addr & ~0x7f; /* start must be 128B aligned */ 1810 } 1811 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 1812 1813 t4_write_reg(sc, reg, start | pf); 1814 t4_read_reg(sc, reg); 1815 1816 return (addr - start); 1817 } 1818 1819 static int 1820 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1821 struct intrs_and_queues *iaq) 1822 { 1823 int rc, itype, navail, nrxq10g, nrxq1g, n; 1824 int nofldrxq10g = 0, nofldrxq1g = 0; 1825 int nnmrxq10g = 0, nnmrxq1g = 0; 1826 1827 bzero(iaq, sizeof(*iaq)); 1828 1829 iaq->ntxq10g = t4_ntxq10g; 1830 iaq->ntxq1g = t4_ntxq1g; 1831 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1832 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1833 iaq->rsrv_noflowq = t4_rsrv_noflowq; 1834 #ifdef TCP_OFFLOAD 1835 if (is_offload(sc)) { 1836 iaq->nofldtxq10g = t4_nofldtxq10g; 1837 iaq->nofldtxq1g = t4_nofldtxq1g; 1838 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1839 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1840 } 1841 #endif 1842 #ifdef DEV_NETMAP 1843 iaq->nnmtxq10g = t4_nnmtxq10g; 1844 iaq->nnmtxq1g = t4_nnmtxq1g; 1845 iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g; 1846 iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g; 1847 #endif 1848 1849 for (itype = INTR_MSIX; itype; itype >>= 1) { 1850 1851 if ((itype & t4_intr_types) == 0) 1852 continue; /* not allowed */ 1853 1854 if (itype == INTR_MSIX) 1855 navail = pci_msix_count(sc->dev); 1856 else if (itype == INTR_MSI) 1857 navail = pci_msi_count(sc->dev); 1858 else 1859 navail = 1; 1860 restart: 1861 if (navail == 0) 1862 continue; 1863 1864 iaq->intr_type = itype; 1865 iaq->intr_flags_10g = 0; 1866 iaq->intr_flags_1g = 0; 1867 1868 /* 1869 * Best option: an interrupt vector for errors, one for the 1870 * firmware event queue, and one for every rxq (NIC, TOE, and 1871 * netmap). 1872 */ 1873 iaq->nirq = T4_EXTRA_INTR; 1874 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g); 1875 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g); 1876 if (iaq->nirq <= navail && 1877 (itype != INTR_MSI || powerof2(iaq->nirq))) { 1878 iaq->intr_flags_10g = INTR_ALL; 1879 iaq->intr_flags_1g = INTR_ALL; 1880 goto allocate; 1881 } 1882 1883 /* 1884 * Second best option: a vector for errors, one for the firmware 1885 * event queue, and vectors for either all the NIC rx queues or 1886 * all the TOE rx queues. The queues that don't get vectors 1887 * will forward their interrupts to those that do. 1888 * 1889 * Note: netmap rx queues cannot be created early and so they 1890 * can't be setup to receive forwarded interrupts for others. 1891 */ 1892 iaq->nirq = T4_EXTRA_INTR; 1893 if (nrxq10g >= nofldrxq10g) { 1894 iaq->intr_flags_10g = INTR_RXQ; 1895 iaq->nirq += n10g * nrxq10g; 1896 #ifdef DEV_NETMAP 1897 iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g); 1898 #endif 1899 } else { 1900 iaq->intr_flags_10g = INTR_OFLD_RXQ; 1901 iaq->nirq += n10g * nofldrxq10g; 1902 #ifdef DEV_NETMAP 1903 iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g); 1904 #endif 1905 } 1906 if (nrxq1g >= nofldrxq1g) { 1907 iaq->intr_flags_1g = INTR_RXQ; 1908 iaq->nirq += n1g * nrxq1g; 1909 #ifdef DEV_NETMAP 1910 iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g); 1911 #endif 1912 } else { 1913 iaq->intr_flags_1g = INTR_OFLD_RXQ; 1914 iaq->nirq += n1g * nofldrxq1g; 1915 #ifdef DEV_NETMAP 1916 iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g); 1917 #endif 1918 } 1919 if (iaq->nirq <= navail && 1920 (itype != INTR_MSI || powerof2(iaq->nirq))) 1921 goto allocate; 1922 1923 /* 1924 * Next best option: an interrupt vector for errors, one for the 1925 * firmware event queue, and at least one per port. At this 1926 * point we know we'll have to downsize nrxq and/or nofldrxq 1927 * and/or nnmrxq to fit what's available to us. 1928 */ 1929 iaq->nirq = T4_EXTRA_INTR; 1930 iaq->nirq += n10g + n1g; 1931 if (iaq->nirq <= navail) { 1932 int leftover = navail - iaq->nirq; 1933 1934 if (n10g > 0) { 1935 int target = max(nrxq10g, nofldrxq10g); 1936 1937 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ? 1938 INTR_RXQ : INTR_OFLD_RXQ; 1939 1940 n = 1; 1941 while (n < target && leftover >= n10g) { 1942 leftover -= n10g; 1943 iaq->nirq += n10g; 1944 n++; 1945 } 1946 iaq->nrxq10g = min(n, nrxq10g); 1947 #ifdef TCP_OFFLOAD 1948 iaq->nofldrxq10g = min(n, nofldrxq10g); 1949 #endif 1950 #ifdef DEV_NETMAP 1951 iaq->nnmrxq10g = min(n, nnmrxq10g); 1952 #endif 1953 } 1954 1955 if (n1g > 0) { 1956 int target = max(nrxq1g, nofldrxq1g); 1957 1958 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ? 1959 INTR_RXQ : INTR_OFLD_RXQ; 1960 1961 n = 1; 1962 while (n < target && leftover >= n1g) { 1963 leftover -= n1g; 1964 iaq->nirq += n1g; 1965 n++; 1966 } 1967 iaq->nrxq1g = min(n, nrxq1g); 1968 #ifdef TCP_OFFLOAD 1969 iaq->nofldrxq1g = min(n, nofldrxq1g); 1970 #endif 1971 #ifdef DEV_NETMAP 1972 iaq->nnmrxq1g = min(n, nnmrxq1g); 1973 #endif 1974 } 1975 1976 if (itype != INTR_MSI || powerof2(iaq->nirq)) 1977 goto allocate; 1978 } 1979 1980 /* 1981 * Least desirable option: one interrupt vector for everything. 1982 */ 1983 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 1984 iaq->intr_flags_10g = iaq->intr_flags_1g = 0; 1985 #ifdef TCP_OFFLOAD 1986 if (is_offload(sc)) 1987 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 1988 #endif 1989 #ifdef DEV_NETMAP 1990 iaq->nnmrxq10g = iaq->nnmrxq1g = 1; 1991 #endif 1992 1993 allocate: 1994 navail = iaq->nirq; 1995 rc = 0; 1996 if (itype == INTR_MSIX) 1997 rc = pci_alloc_msix(sc->dev, &navail); 1998 else if (itype == INTR_MSI) 1999 rc = pci_alloc_msi(sc->dev, &navail); 2000 2001 if (rc == 0) { 2002 if (navail == iaq->nirq) 2003 return (0); 2004 2005 /* 2006 * Didn't get the number requested. Use whatever number 2007 * the kernel is willing to allocate (it's in navail). 2008 */ 2009 device_printf(sc->dev, "fewer vectors than requested, " 2010 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 2011 itype, iaq->nirq, navail); 2012 pci_release_msi(sc->dev); 2013 goto restart; 2014 } 2015 2016 device_printf(sc->dev, 2017 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 2018 itype, rc, iaq->nirq, navail); 2019 } 2020 2021 device_printf(sc->dev, 2022 "failed to find a usable interrupt type. " 2023 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 2024 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 2025 2026 return (ENXIO); 2027 } 2028 2029 #define FW_VERSION(chip) ( \ 2030 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ 2031 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ 2032 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ 2033 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) 2034 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) 2035 2036 struct fw_info { 2037 uint8_t chip; 2038 char *kld_name; 2039 char *fw_mod_name; 2040 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 2041 } fw_info[] = { 2042 { 2043 .chip = CHELSIO_T4, 2044 .kld_name = "t4fw_cfg", 2045 .fw_mod_name = "t4fw", 2046 .fw_hdr = { 2047 .chip = FW_HDR_CHIP_T4, 2048 .fw_ver = htobe32_const(FW_VERSION(T4)), 2049 .intfver_nic = FW_INTFVER(T4, NIC), 2050 .intfver_vnic = FW_INTFVER(T4, VNIC), 2051 .intfver_ofld = FW_INTFVER(T4, OFLD), 2052 .intfver_ri = FW_INTFVER(T4, RI), 2053 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 2054 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 2055 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 2056 .intfver_fcoe = FW_INTFVER(T4, FCOE), 2057 }, 2058 }, { 2059 .chip = CHELSIO_T5, 2060 .kld_name = "t5fw_cfg", 2061 .fw_mod_name = "t5fw", 2062 .fw_hdr = { 2063 .chip = FW_HDR_CHIP_T5, 2064 .fw_ver = htobe32_const(FW_VERSION(T5)), 2065 .intfver_nic = FW_INTFVER(T5, NIC), 2066 .intfver_vnic = FW_INTFVER(T5, VNIC), 2067 .intfver_ofld = FW_INTFVER(T5, OFLD), 2068 .intfver_ri = FW_INTFVER(T5, RI), 2069 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 2070 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 2071 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 2072 .intfver_fcoe = FW_INTFVER(T5, FCOE), 2073 }, 2074 } 2075 }; 2076 2077 static struct fw_info * 2078 find_fw_info(int chip) 2079 { 2080 int i; 2081 2082 for (i = 0; i < nitems(fw_info); i++) { 2083 if (fw_info[i].chip == chip) 2084 return (&fw_info[i]); 2085 } 2086 return (NULL); 2087 } 2088 2089 /* 2090 * Is the given firmware API compatible with the one the driver was compiled 2091 * with? 2092 */ 2093 static int 2094 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 2095 { 2096 2097 /* short circuit if it's the exact same firmware version */ 2098 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 2099 return (1); 2100 2101 /* 2102 * XXX: Is this too conservative? Perhaps I should limit this to the 2103 * features that are supported in the driver. 2104 */ 2105 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 2106 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 2107 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 2108 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 2109 return (1); 2110 #undef SAME_INTF 2111 2112 return (0); 2113 } 2114 2115 /* 2116 * The firmware in the KLD is usable, but should it be installed? This routine 2117 * explains itself in detail if it indicates the KLD firmware should be 2118 * installed. 2119 */ 2120 static int 2121 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 2122 { 2123 const char *reason; 2124 2125 if (!card_fw_usable) { 2126 reason = "incompatible or unusable"; 2127 goto install; 2128 } 2129 2130 if (k > c) { 2131 reason = "older than the version bundled with this driver"; 2132 goto install; 2133 } 2134 2135 if (t4_fw_install == 2 && k != c) { 2136 reason = "different than the version bundled with this driver"; 2137 goto install; 2138 } 2139 2140 return (0); 2141 2142 install: 2143 if (t4_fw_install == 0) { 2144 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2145 "but the driver is prohibited from installing a different " 2146 "firmware on the card.\n", 2147 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2148 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 2149 2150 return (0); 2151 } 2152 2153 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2154 "installing firmware %u.%u.%u.%u on card.\n", 2155 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2156 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 2157 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2158 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2159 2160 return (1); 2161 } 2162 /* 2163 * Establish contact with the firmware and determine if we are the master driver 2164 * or not, and whether we are responsible for chip initialization. 2165 */ 2166 static int 2167 prep_firmware(struct adapter *sc) 2168 { 2169 const struct firmware *fw = NULL, *default_cfg; 2170 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 2171 enum dev_state state; 2172 struct fw_info *fw_info; 2173 struct fw_hdr *card_fw; /* fw on the card */ 2174 const struct fw_hdr *kld_fw; /* fw in the KLD */ 2175 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 2176 against */ 2177 2178 /* Contact firmware. */ 2179 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 2180 if (rc < 0 || state == DEV_STATE_ERR) { 2181 rc = -rc; 2182 device_printf(sc->dev, 2183 "failed to connect to the firmware: %d, %d.\n", rc, state); 2184 return (rc); 2185 } 2186 pf = rc; 2187 if (pf == sc->mbox) 2188 sc->flags |= MASTER_PF; 2189 else if (state == DEV_STATE_UNINIT) { 2190 /* 2191 * We didn't get to be the master so we definitely won't be 2192 * configuring the chip. It's a bug if someone else hasn't 2193 * configured it already. 2194 */ 2195 device_printf(sc->dev, "couldn't be master(%d), " 2196 "device not already initialized either(%d).\n", rc, state); 2197 return (EDOOFUS); 2198 } 2199 2200 /* This is the firmware whose headers the driver was compiled against */ 2201 fw_info = find_fw_info(chip_id(sc)); 2202 if (fw_info == NULL) { 2203 device_printf(sc->dev, 2204 "unable to look up firmware information for chip %d.\n", 2205 chip_id(sc)); 2206 return (EINVAL); 2207 } 2208 drv_fw = &fw_info->fw_hdr; 2209 2210 /* 2211 * The firmware KLD contains many modules. The KLD name is also the 2212 * name of the module that contains the default config file. 2213 */ 2214 default_cfg = firmware_get(fw_info->kld_name); 2215 2216 /* Read the header of the firmware on the card */ 2217 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 2218 rc = -t4_read_flash(sc, FLASH_FW_START, 2219 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 2220 if (rc == 0) 2221 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 2222 else { 2223 device_printf(sc->dev, 2224 "Unable to read card's firmware header: %d\n", rc); 2225 card_fw_usable = 0; 2226 } 2227 2228 /* This is the firmware in the KLD */ 2229 fw = firmware_get(fw_info->fw_mod_name); 2230 if (fw != NULL) { 2231 kld_fw = (const void *)fw->data; 2232 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 2233 } else { 2234 kld_fw = NULL; 2235 kld_fw_usable = 0; 2236 } 2237 2238 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2239 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) { 2240 /* 2241 * Common case: the firmware on the card is an exact match and 2242 * the KLD is an exact match too, or the KLD is 2243 * absent/incompatible. Note that t4_fw_install = 2 is ignored 2244 * here -- use cxgbetool loadfw if you want to reinstall the 2245 * same firmware as the one on the card. 2246 */ 2247 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2248 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2249 be32toh(card_fw->fw_ver))) { 2250 2251 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 2252 if (rc != 0) { 2253 device_printf(sc->dev, 2254 "failed to install firmware: %d\n", rc); 2255 goto done; 2256 } 2257 2258 /* Installed successfully, update the cached header too. */ 2259 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2260 card_fw_usable = 1; 2261 need_fw_reset = 0; /* already reset as part of load_fw */ 2262 } 2263 2264 if (!card_fw_usable) { 2265 uint32_t d, c, k; 2266 2267 d = ntohl(drv_fw->fw_ver); 2268 c = ntohl(card_fw->fw_ver); 2269 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2270 2271 device_printf(sc->dev, "Cannot find a usable firmware: " 2272 "fw_install %d, chip state %d, " 2273 "driver compiled with %d.%d.%d.%d, " 2274 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2275 t4_fw_install, state, 2276 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2277 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2278 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2279 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2280 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2281 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2282 rc = EINVAL; 2283 goto done; 2284 } 2285 2286 /* We're using whatever's on the card and it's known to be good. */ 2287 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2288 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2289 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2290 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2291 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2292 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2293 t4_get_tp_version(sc, &sc->params.tp_vers); 2294 2295 /* Reset device */ 2296 if (need_fw_reset && 2297 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2298 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2299 if (rc != ETIMEDOUT && rc != EIO) 2300 t4_fw_bye(sc, sc->mbox); 2301 goto done; 2302 } 2303 sc->flags |= FW_OK; 2304 2305 rc = get_params__pre_init(sc); 2306 if (rc != 0) 2307 goto done; /* error message displayed already */ 2308 2309 /* Partition adapter resources as specified in the config file. */ 2310 if (state == DEV_STATE_UNINIT) { 2311 2312 KASSERT(sc->flags & MASTER_PF, 2313 ("%s: trying to change chip settings when not master.", 2314 __func__)); 2315 2316 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2317 if (rc != 0) 2318 goto done; /* error message displayed already */ 2319 2320 t4_tweak_chip_settings(sc); 2321 2322 /* get basic stuff going */ 2323 rc = -t4_fw_initialize(sc, sc->mbox); 2324 if (rc != 0) { 2325 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2326 goto done; 2327 } 2328 } else { 2329 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2330 sc->cfcsum = 0; 2331 } 2332 2333 done: 2334 free(card_fw, M_CXGBE); 2335 if (fw != NULL) 2336 firmware_put(fw, FIRMWARE_UNLOAD); 2337 if (default_cfg != NULL) 2338 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2339 2340 return (rc); 2341 } 2342 2343 #define FW_PARAM_DEV(param) \ 2344 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2345 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2346 #define FW_PARAM_PFVF(param) \ 2347 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2348 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2349 2350 /* 2351 * Partition chip resources for use between various PFs, VFs, etc. 2352 */ 2353 static int 2354 partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2355 const char *name_prefix) 2356 { 2357 const struct firmware *cfg = NULL; 2358 int rc = 0; 2359 struct fw_caps_config_cmd caps; 2360 uint32_t mtype, moff, finicsum, cfcsum; 2361 2362 /* 2363 * Figure out what configuration file to use. Pick the default config 2364 * file for the card if the user hasn't specified one explicitly. 2365 */ 2366 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2367 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2368 /* Card specific overrides go here. */ 2369 if (pci_get_device(sc->dev) == 0x440a) 2370 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2371 if (is_fpga(sc)) 2372 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2373 } 2374 2375 /* 2376 * We need to load another module if the profile is anything except 2377 * "default" or "flash". 2378 */ 2379 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2380 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2381 char s[32]; 2382 2383 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2384 cfg = firmware_get(s); 2385 if (cfg == NULL) { 2386 if (default_cfg != NULL) { 2387 device_printf(sc->dev, 2388 "unable to load module \"%s\" for " 2389 "configuration profile \"%s\", will use " 2390 "the default config file instead.\n", 2391 s, sc->cfg_file); 2392 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2393 "%s", DEFAULT_CF); 2394 } else { 2395 device_printf(sc->dev, 2396 "unable to load module \"%s\" for " 2397 "configuration profile \"%s\", will use " 2398 "the config file on the card's flash " 2399 "instead.\n", s, sc->cfg_file); 2400 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2401 "%s", FLASH_CF); 2402 } 2403 } 2404 } 2405 2406 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2407 default_cfg == NULL) { 2408 device_printf(sc->dev, 2409 "default config file not available, will use the config " 2410 "file on the card's flash instead.\n"); 2411 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2412 } 2413 2414 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2415 u_int cflen, i, n; 2416 const uint32_t *cfdata; 2417 uint32_t param, val, addr, off, mw_base, mw_aperture; 2418 2419 KASSERT(cfg != NULL || default_cfg != NULL, 2420 ("%s: no config to upload", __func__)); 2421 2422 /* 2423 * Ask the firmware where it wants us to upload the config file. 2424 */ 2425 param = FW_PARAM_DEV(CF); 2426 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2427 if (rc != 0) { 2428 /* No support for config file? Shouldn't happen. */ 2429 device_printf(sc->dev, 2430 "failed to query config file location: %d.\n", rc); 2431 goto done; 2432 } 2433 mtype = G_FW_PARAMS_PARAM_Y(val); 2434 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2435 2436 /* 2437 * XXX: sheer laziness. We deliberately added 4 bytes of 2438 * useless stuffing/comments at the end of the config file so 2439 * it's ok to simply throw away the last remaining bytes when 2440 * the config file is not an exact multiple of 4. This also 2441 * helps with the validate_mt_off_len check. 2442 */ 2443 if (cfg != NULL) { 2444 cflen = cfg->datasize & ~3; 2445 cfdata = cfg->data; 2446 } else { 2447 cflen = default_cfg->datasize & ~3; 2448 cfdata = default_cfg->data; 2449 } 2450 2451 if (cflen > FLASH_CFG_MAX_SIZE) { 2452 device_printf(sc->dev, 2453 "config file too long (%d, max allowed is %d). " 2454 "Will try to use the config on the card, if any.\n", 2455 cflen, FLASH_CFG_MAX_SIZE); 2456 goto use_config_on_flash; 2457 } 2458 2459 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2460 if (rc != 0) { 2461 device_printf(sc->dev, 2462 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2463 "Will try to use the config on the card, if any.\n", 2464 __func__, mtype, moff, cflen, rc); 2465 goto use_config_on_flash; 2466 } 2467 2468 memwin_info(sc, 2, &mw_base, &mw_aperture); 2469 while (cflen) { 2470 off = position_memwin(sc, 2, addr); 2471 n = min(cflen, mw_aperture - off); 2472 for (i = 0; i < n; i += 4) 2473 t4_write_reg(sc, mw_base + off + i, *cfdata++); 2474 cflen -= n; 2475 addr += n; 2476 } 2477 } else { 2478 use_config_on_flash: 2479 mtype = FW_MEMTYPE_FLASH; 2480 moff = t4_flash_cfg_addr(sc); 2481 } 2482 2483 bzero(&caps, sizeof(caps)); 2484 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2485 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2486 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2487 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2488 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2489 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2490 if (rc != 0) { 2491 device_printf(sc->dev, 2492 "failed to pre-process config file: %d " 2493 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2494 goto done; 2495 } 2496 2497 finicsum = be32toh(caps.finicsum); 2498 cfcsum = be32toh(caps.cfcsum); 2499 if (finicsum != cfcsum) { 2500 device_printf(sc->dev, 2501 "WARNING: config file checksum mismatch: %08x %08x\n", 2502 finicsum, cfcsum); 2503 } 2504 sc->cfcsum = cfcsum; 2505 2506 #define LIMIT_CAPS(x) do { \ 2507 caps.x &= htobe16(t4_##x##_allowed); \ 2508 } while (0) 2509 2510 /* 2511 * Let the firmware know what features will (not) be used so it can tune 2512 * things accordingly. 2513 */ 2514 LIMIT_CAPS(linkcaps); 2515 LIMIT_CAPS(niccaps); 2516 LIMIT_CAPS(toecaps); 2517 LIMIT_CAPS(rdmacaps); 2518 LIMIT_CAPS(iscsicaps); 2519 LIMIT_CAPS(fcoecaps); 2520 #undef LIMIT_CAPS 2521 2522 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2523 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2524 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2525 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2526 if (rc != 0) { 2527 device_printf(sc->dev, 2528 "failed to process config file: %d.\n", rc); 2529 } 2530 done: 2531 if (cfg != NULL) 2532 firmware_put(cfg, FIRMWARE_UNLOAD); 2533 return (rc); 2534 } 2535 2536 /* 2537 * Retrieve parameters that are needed (or nice to have) very early. 2538 */ 2539 static int 2540 get_params__pre_init(struct adapter *sc) 2541 { 2542 int rc; 2543 uint32_t param[2], val[2]; 2544 struct fw_devlog_cmd cmd; 2545 struct devlog_params *dlog = &sc->params.devlog; 2546 2547 param[0] = FW_PARAM_DEV(PORTVEC); 2548 param[1] = FW_PARAM_DEV(CCLK); 2549 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2550 if (rc != 0) { 2551 device_printf(sc->dev, 2552 "failed to query parameters (pre_init): %d.\n", rc); 2553 return (rc); 2554 } 2555 2556 sc->params.portvec = val[0]; 2557 sc->params.nports = bitcount32(val[0]); 2558 sc->params.vpd.cclk = val[1]; 2559 2560 /* Read device log parameters. */ 2561 bzero(&cmd, sizeof(cmd)); 2562 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2563 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2564 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2565 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2566 if (rc != 0) { 2567 device_printf(sc->dev, 2568 "failed to get devlog parameters: %d.\n", rc); 2569 bzero(dlog, sizeof (*dlog)); 2570 rc = 0; /* devlog isn't critical for device operation */ 2571 } else { 2572 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2573 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2574 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2575 dlog->size = be32toh(cmd.memsize_devlog); 2576 } 2577 2578 return (rc); 2579 } 2580 2581 /* 2582 * Retrieve various parameters that are of interest to the driver. The device 2583 * has been initialized by the firmware at this point. 2584 */ 2585 static int 2586 get_params__post_init(struct adapter *sc) 2587 { 2588 int rc; 2589 uint32_t param[7], val[7]; 2590 struct fw_caps_config_cmd caps; 2591 2592 param[0] = FW_PARAM_PFVF(IQFLINT_START); 2593 param[1] = FW_PARAM_PFVF(EQ_START); 2594 param[2] = FW_PARAM_PFVF(FILTER_START); 2595 param[3] = FW_PARAM_PFVF(FILTER_END); 2596 param[4] = FW_PARAM_PFVF(L2T_START); 2597 param[5] = FW_PARAM_PFVF(L2T_END); 2598 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2599 if (rc != 0) { 2600 device_printf(sc->dev, 2601 "failed to query parameters (post_init): %d.\n", rc); 2602 return (rc); 2603 } 2604 2605 sc->sge.iq_start = val[0]; 2606 sc->sge.eq_start = val[1]; 2607 sc->tids.ftid_base = val[2]; 2608 sc->tids.nftids = val[3] - val[2] + 1; 2609 sc->params.ftid_min = val[2]; 2610 sc->params.ftid_max = val[3]; 2611 sc->vres.l2t.start = val[4]; 2612 sc->vres.l2t.size = val[5] - val[4] + 1; 2613 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2614 ("%s: L2 table size (%u) larger than expected (%u)", 2615 __func__, sc->vres.l2t.size, L2T_SIZE)); 2616 2617 /* get capabilites */ 2618 bzero(&caps, sizeof(caps)); 2619 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2620 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2621 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2622 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2623 if (rc != 0) { 2624 device_printf(sc->dev, 2625 "failed to get card capabilities: %d.\n", rc); 2626 return (rc); 2627 } 2628 2629 #define READ_CAPS(x) do { \ 2630 sc->x = htobe16(caps.x); \ 2631 } while (0) 2632 READ_CAPS(linkcaps); 2633 READ_CAPS(niccaps); 2634 READ_CAPS(toecaps); 2635 READ_CAPS(rdmacaps); 2636 READ_CAPS(iscsicaps); 2637 READ_CAPS(fcoecaps); 2638 2639 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { 2640 param[0] = FW_PARAM_PFVF(ETHOFLD_START); 2641 param[1] = FW_PARAM_PFVF(ETHOFLD_END); 2642 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2643 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); 2644 if (rc != 0) { 2645 device_printf(sc->dev, 2646 "failed to query NIC parameters: %d.\n", rc); 2647 return (rc); 2648 } 2649 sc->tids.etid_base = val[0]; 2650 sc->params.etid_min = val[0]; 2651 sc->tids.netids = val[1] - val[0] + 1; 2652 sc->params.netids = sc->tids.netids; 2653 sc->params.eo_wr_cred = val[2]; 2654 sc->params.ethoffload = 1; 2655 } 2656 2657 if (sc->toecaps) { 2658 /* query offload-related parameters */ 2659 param[0] = FW_PARAM_DEV(NTID); 2660 param[1] = FW_PARAM_PFVF(SERVER_START); 2661 param[2] = FW_PARAM_PFVF(SERVER_END); 2662 param[3] = FW_PARAM_PFVF(TDDP_START); 2663 param[4] = FW_PARAM_PFVF(TDDP_END); 2664 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2665 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2666 if (rc != 0) { 2667 device_printf(sc->dev, 2668 "failed to query TOE parameters: %d.\n", rc); 2669 return (rc); 2670 } 2671 sc->tids.ntids = val[0]; 2672 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 2673 sc->tids.stid_base = val[1]; 2674 sc->tids.nstids = val[2] - val[1] + 1; 2675 sc->vres.ddp.start = val[3]; 2676 sc->vres.ddp.size = val[4] - val[3] + 1; 2677 sc->params.ofldq_wr_cred = val[5]; 2678 sc->params.offload = 1; 2679 } 2680 if (sc->rdmacaps) { 2681 param[0] = FW_PARAM_PFVF(STAG_START); 2682 param[1] = FW_PARAM_PFVF(STAG_END); 2683 param[2] = FW_PARAM_PFVF(RQ_START); 2684 param[3] = FW_PARAM_PFVF(RQ_END); 2685 param[4] = FW_PARAM_PFVF(PBL_START); 2686 param[5] = FW_PARAM_PFVF(PBL_END); 2687 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2688 if (rc != 0) { 2689 device_printf(sc->dev, 2690 "failed to query RDMA parameters(1): %d.\n", rc); 2691 return (rc); 2692 } 2693 sc->vres.stag.start = val[0]; 2694 sc->vres.stag.size = val[1] - val[0] + 1; 2695 sc->vres.rq.start = val[2]; 2696 sc->vres.rq.size = val[3] - val[2] + 1; 2697 sc->vres.pbl.start = val[4]; 2698 sc->vres.pbl.size = val[5] - val[4] + 1; 2699 2700 param[0] = FW_PARAM_PFVF(SQRQ_START); 2701 param[1] = FW_PARAM_PFVF(SQRQ_END); 2702 param[2] = FW_PARAM_PFVF(CQ_START); 2703 param[3] = FW_PARAM_PFVF(CQ_END); 2704 param[4] = FW_PARAM_PFVF(OCQ_START); 2705 param[5] = FW_PARAM_PFVF(OCQ_END); 2706 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2707 if (rc != 0) { 2708 device_printf(sc->dev, 2709 "failed to query RDMA parameters(2): %d.\n", rc); 2710 return (rc); 2711 } 2712 sc->vres.qp.start = val[0]; 2713 sc->vres.qp.size = val[1] - val[0] + 1; 2714 sc->vres.cq.start = val[2]; 2715 sc->vres.cq.size = val[3] - val[2] + 1; 2716 sc->vres.ocq.start = val[4]; 2717 sc->vres.ocq.size = val[5] - val[4] + 1; 2718 } 2719 if (sc->iscsicaps) { 2720 param[0] = FW_PARAM_PFVF(ISCSI_START); 2721 param[1] = FW_PARAM_PFVF(ISCSI_END); 2722 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2723 if (rc != 0) { 2724 device_printf(sc->dev, 2725 "failed to query iSCSI parameters: %d.\n", rc); 2726 return (rc); 2727 } 2728 sc->vres.iscsi.start = val[0]; 2729 sc->vres.iscsi.size = val[1] - val[0] + 1; 2730 } 2731 2732 /* 2733 * We've got the params we wanted to query via the firmware. Now grab 2734 * some others directly from the chip. 2735 */ 2736 rc = t4_read_chip_settings(sc); 2737 2738 return (rc); 2739 } 2740 2741 static int 2742 set_params__post_init(struct adapter *sc) 2743 { 2744 uint32_t param, val; 2745 2746 /* ask for encapsulated CPLs */ 2747 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 2748 val = 1; 2749 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2750 2751 return (0); 2752 } 2753 2754 #undef FW_PARAM_PFVF 2755 #undef FW_PARAM_DEV 2756 2757 static void 2758 t4_set_desc(struct adapter *sc) 2759 { 2760 char buf[128]; 2761 struct adapter_params *p = &sc->params; 2762 2763 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, " 2764 "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "", 2765 chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec); 2766 2767 device_set_desc_copy(sc->dev, buf); 2768 } 2769 2770 static void 2771 build_medialist(struct port_info *pi, struct ifmedia *media) 2772 { 2773 int data, m; 2774 2775 PORT_LOCK(pi); 2776 2777 ifmedia_removeall(media); 2778 2779 m = IFM_ETHER | IFM_FDX; 2780 data = (pi->port_type << 8) | pi->mod_type; 2781 2782 switch(pi->port_type) { 2783 case FW_PORT_TYPE_BT_XFI: 2784 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2785 break; 2786 2787 case FW_PORT_TYPE_BT_XAUI: 2788 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2789 /* fall through */ 2790 2791 case FW_PORT_TYPE_BT_SGMII: 2792 ifmedia_add(media, m | IFM_1000_T, data, NULL); 2793 ifmedia_add(media, m | IFM_100_TX, data, NULL); 2794 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL); 2795 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2796 break; 2797 2798 case FW_PORT_TYPE_CX4: 2799 ifmedia_add(media, m | IFM_10G_CX4, data, NULL); 2800 ifmedia_set(media, m | IFM_10G_CX4); 2801 break; 2802 2803 case FW_PORT_TYPE_QSFP_10G: 2804 case FW_PORT_TYPE_SFP: 2805 case FW_PORT_TYPE_FIBER_XFI: 2806 case FW_PORT_TYPE_FIBER_XAUI: 2807 switch (pi->mod_type) { 2808 2809 case FW_PORT_MOD_TYPE_LR: 2810 ifmedia_add(media, m | IFM_10G_LR, data, NULL); 2811 ifmedia_set(media, m | IFM_10G_LR); 2812 break; 2813 2814 case FW_PORT_MOD_TYPE_SR: 2815 ifmedia_add(media, m | IFM_10G_SR, data, NULL); 2816 ifmedia_set(media, m | IFM_10G_SR); 2817 break; 2818 2819 case FW_PORT_MOD_TYPE_LRM: 2820 ifmedia_add(media, m | IFM_10G_LRM, data, NULL); 2821 ifmedia_set(media, m | IFM_10G_LRM); 2822 break; 2823 2824 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2825 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2826 ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL); 2827 ifmedia_set(media, m | IFM_10G_TWINAX); 2828 break; 2829 2830 case FW_PORT_MOD_TYPE_NONE: 2831 m &= ~IFM_FDX; 2832 ifmedia_add(media, m | IFM_NONE, data, NULL); 2833 ifmedia_set(media, m | IFM_NONE); 2834 break; 2835 2836 case FW_PORT_MOD_TYPE_NA: 2837 case FW_PORT_MOD_TYPE_ER: 2838 default: 2839 device_printf(pi->dev, 2840 "unknown port_type (%d), mod_type (%d)\n", 2841 pi->port_type, pi->mod_type); 2842 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2843 ifmedia_set(media, m | IFM_UNKNOWN); 2844 break; 2845 } 2846 break; 2847 2848 case FW_PORT_TYPE_QSFP: 2849 switch (pi->mod_type) { 2850 2851 case FW_PORT_MOD_TYPE_LR: 2852 ifmedia_add(media, m | IFM_40G_LR4, data, NULL); 2853 ifmedia_set(media, m | IFM_40G_LR4); 2854 break; 2855 2856 case FW_PORT_MOD_TYPE_SR: 2857 ifmedia_add(media, m | IFM_40G_SR4, data, NULL); 2858 ifmedia_set(media, m | IFM_40G_SR4); 2859 break; 2860 2861 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2862 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2863 ifmedia_add(media, m | IFM_40G_CR4, data, NULL); 2864 ifmedia_set(media, m | IFM_40G_CR4); 2865 break; 2866 2867 case FW_PORT_MOD_TYPE_NONE: 2868 m &= ~IFM_FDX; 2869 ifmedia_add(media, m | IFM_NONE, data, NULL); 2870 ifmedia_set(media, m | IFM_NONE); 2871 break; 2872 2873 default: 2874 device_printf(pi->dev, 2875 "unknown port_type (%d), mod_type (%d)\n", 2876 pi->port_type, pi->mod_type); 2877 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2878 ifmedia_set(media, m | IFM_UNKNOWN); 2879 break; 2880 } 2881 break; 2882 2883 default: 2884 device_printf(pi->dev, 2885 "unknown port_type (%d), mod_type (%d)\n", pi->port_type, 2886 pi->mod_type); 2887 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2888 ifmedia_set(media, m | IFM_UNKNOWN); 2889 break; 2890 } 2891 2892 PORT_UNLOCK(pi); 2893 } 2894 2895 #define FW_MAC_EXACT_CHUNK 7 2896 2897 /* 2898 * Program the port's XGMAC based on parameters in ifnet. The caller also 2899 * indicates which parameters should be programmed (the rest are left alone). 2900 */ 2901 int 2902 update_mac_settings(struct ifnet *ifp, int flags) 2903 { 2904 int rc = 0; 2905 struct port_info *pi = ifp->if_softc; 2906 struct adapter *sc = pi->adapter; 2907 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 2908 uint16_t viid = 0xffff; 2909 int16_t *xact_addr_filt = NULL; 2910 2911 ASSERT_SYNCHRONIZED_OP(sc); 2912 KASSERT(flags, ("%s: not told what to update.", __func__)); 2913 2914 if (ifp == pi->ifp) { 2915 viid = pi->viid; 2916 xact_addr_filt = &pi->xact_addr_filt; 2917 } 2918 #ifdef DEV_NETMAP 2919 else if (ifp == pi->nm_ifp) { 2920 viid = pi->nm_viid; 2921 xact_addr_filt = &pi->nm_xact_addr_filt; 2922 } 2923 #endif 2924 if (flags & XGMAC_MTU) 2925 mtu = ifp->if_mtu; 2926 2927 if (flags & XGMAC_PROMISC) 2928 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 2929 2930 if (flags & XGMAC_ALLMULTI) 2931 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 2932 2933 if (flags & XGMAC_VLANEX) 2934 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 2935 2936 if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) { 2937 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti, 2938 1, vlanex, false); 2939 if (rc) { 2940 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, 2941 rc); 2942 return (rc); 2943 } 2944 } 2945 2946 if (flags & XGMAC_UCADDR) { 2947 uint8_t ucaddr[ETHER_ADDR_LEN]; 2948 2949 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 2950 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr, 2951 true, true); 2952 if (rc < 0) { 2953 rc = -rc; 2954 if_printf(ifp, "change_mac failed: %d\n", rc); 2955 return (rc); 2956 } else { 2957 *xact_addr_filt = rc; 2958 rc = 0; 2959 } 2960 } 2961 2962 if (flags & XGMAC_MCADDRS) { 2963 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 2964 int del = 1; 2965 uint64_t hash = 0; 2966 struct ifmultiaddr *ifma; 2967 int i = 0, j; 2968 2969 if_maddr_rlock(ifp); 2970 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 2971 if (ifma->ifma_addr->sa_family != AF_LINK) 2972 continue; 2973 mcaddr[i++] = 2974 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 2975 2976 if (i == FW_MAC_EXACT_CHUNK) { 2977 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, 2978 i, mcaddr, NULL, &hash, 0); 2979 if (rc < 0) { 2980 rc = -rc; 2981 for (j = 0; j < i; j++) { 2982 if_printf(ifp, 2983 "failed to add mc address" 2984 " %02x:%02x:%02x:" 2985 "%02x:%02x:%02x rc=%d\n", 2986 mcaddr[j][0], mcaddr[j][1], 2987 mcaddr[j][2], mcaddr[j][3], 2988 mcaddr[j][4], mcaddr[j][5], 2989 rc); 2990 } 2991 goto mcfail; 2992 } 2993 del = 0; 2994 i = 0; 2995 } 2996 } 2997 if (i > 0) { 2998 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i, 2999 mcaddr, NULL, &hash, 0); 3000 if (rc < 0) { 3001 rc = -rc; 3002 for (j = 0; j < i; j++) { 3003 if_printf(ifp, 3004 "failed to add mc address" 3005 " %02x:%02x:%02x:" 3006 "%02x:%02x:%02x rc=%d\n", 3007 mcaddr[j][0], mcaddr[j][1], 3008 mcaddr[j][2], mcaddr[j][3], 3009 mcaddr[j][4], mcaddr[j][5], 3010 rc); 3011 } 3012 goto mcfail; 3013 } 3014 } 3015 3016 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0); 3017 if (rc != 0) 3018 if_printf(ifp, "failed to set mc address hash: %d", rc); 3019 mcfail: 3020 if_maddr_runlock(ifp); 3021 } 3022 3023 return (rc); 3024 } 3025 3026 int 3027 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags, 3028 char *wmesg) 3029 { 3030 int rc, pri; 3031 3032 #ifdef WITNESS 3033 /* the caller thinks it's ok to sleep, but is it really? */ 3034 if (flags & SLEEP_OK) 3035 pause("t4slptst", 1); 3036 #endif 3037 3038 if (INTR_OK) 3039 pri = PCATCH; 3040 else 3041 pri = 0; 3042 3043 ADAPTER_LOCK(sc); 3044 for (;;) { 3045 3046 if (pi && IS_DOOMED(pi)) { 3047 rc = ENXIO; 3048 goto done; 3049 } 3050 3051 if (!IS_BUSY(sc)) { 3052 rc = 0; 3053 break; 3054 } 3055 3056 if (!(flags & SLEEP_OK)) { 3057 rc = EBUSY; 3058 goto done; 3059 } 3060 3061 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 3062 rc = EINTR; 3063 goto done; 3064 } 3065 } 3066 3067 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 3068 SET_BUSY(sc); 3069 #ifdef INVARIANTS 3070 sc->last_op = wmesg; 3071 sc->last_op_thr = curthread; 3072 #endif 3073 3074 done: 3075 if (!(flags & HOLD_LOCK) || rc) 3076 ADAPTER_UNLOCK(sc); 3077 3078 return (rc); 3079 } 3080 3081 void 3082 end_synchronized_op(struct adapter *sc, int flags) 3083 { 3084 3085 if (flags & LOCK_HELD) 3086 ADAPTER_LOCK_ASSERT_OWNED(sc); 3087 else 3088 ADAPTER_LOCK(sc); 3089 3090 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 3091 CLR_BUSY(sc); 3092 wakeup(&sc->flags); 3093 ADAPTER_UNLOCK(sc); 3094 } 3095 3096 static int 3097 cxgbe_init_synchronized(struct port_info *pi) 3098 { 3099 struct adapter *sc = pi->adapter; 3100 struct ifnet *ifp = pi->ifp; 3101 int rc = 0; 3102 3103 ASSERT_SYNCHRONIZED_OP(sc); 3104 3105 if (isset(&sc->open_device_map, pi->port_id)) { 3106 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 3107 ("mismatch between open_device_map and if_drv_flags")); 3108 return (0); /* already running */ 3109 } 3110 3111 if (!(sc->flags & FULL_INIT_DONE) && 3112 ((rc = adapter_full_init(sc)) != 0)) 3113 return (rc); /* error message displayed already */ 3114 3115 if (!(pi->flags & PORT_INIT_DONE) && 3116 ((rc = port_full_init(pi)) != 0)) 3117 return (rc); /* error message displayed already */ 3118 3119 rc = update_mac_settings(ifp, XGMAC_ALL); 3120 if (rc) 3121 goto done; /* error message displayed already */ 3122 3123 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 3124 if (rc != 0) { 3125 if_printf(ifp, "enable_vi failed: %d\n", rc); 3126 goto done; 3127 } 3128 3129 /* 3130 * The first iq of the first port to come up is used for tracing. 3131 */ 3132 if (sc->traceq < 0) { 3133 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id; 3134 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : 3135 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | 3136 V_QUEUENUMBER(sc->traceq)); 3137 pi->flags |= HAS_TRACEQ; 3138 } 3139 3140 /* all ok */ 3141 setbit(&sc->open_device_map, pi->port_id); 3142 PORT_LOCK(pi); 3143 ifp->if_drv_flags |= IFF_DRV_RUNNING; 3144 PORT_UNLOCK(pi); 3145 3146 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 3147 done: 3148 if (rc != 0) 3149 cxgbe_uninit_synchronized(pi); 3150 3151 return (rc); 3152 } 3153 3154 /* 3155 * Idempotent. 3156 */ 3157 static int 3158 cxgbe_uninit_synchronized(struct port_info *pi) 3159 { 3160 struct adapter *sc = pi->adapter; 3161 struct ifnet *ifp = pi->ifp; 3162 int rc; 3163 3164 ASSERT_SYNCHRONIZED_OP(sc); 3165 3166 /* 3167 * Disable the VI so that all its data in either direction is discarded 3168 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 3169 * tick) intact as the TP can deliver negative advice or data that it's 3170 * holding in its RAM (for an offloaded connection) even after the VI is 3171 * disabled. 3172 */ 3173 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 3174 if (rc) { 3175 if_printf(ifp, "disable_vi failed: %d\n", rc); 3176 return (rc); 3177 } 3178 3179 clrbit(&sc->open_device_map, pi->port_id); 3180 PORT_LOCK(pi); 3181 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3182 PORT_UNLOCK(pi); 3183 3184 pi->link_cfg.link_ok = 0; 3185 pi->link_cfg.speed = 0; 3186 pi->linkdnrc = -1; 3187 t4_os_link_changed(sc, pi->port_id, 0, -1); 3188 3189 return (0); 3190 } 3191 3192 /* 3193 * It is ok for this function to fail midway and return right away. t4_detach 3194 * will walk the entire sc->irq list and clean up whatever is valid. 3195 */ 3196 static int 3197 setup_intr_handlers(struct adapter *sc) 3198 { 3199 int rc, rid, p, q; 3200 char s[8]; 3201 struct irq *irq; 3202 struct port_info *pi; 3203 struct sge_rxq *rxq; 3204 #ifdef TCP_OFFLOAD 3205 struct sge_ofld_rxq *ofld_rxq; 3206 #endif 3207 #ifdef DEV_NETMAP 3208 struct sge_nm_rxq *nm_rxq; 3209 #endif 3210 3211 /* 3212 * Setup interrupts. 3213 */ 3214 irq = &sc->irq[0]; 3215 rid = sc->intr_type == INTR_INTX ? 0 : 1; 3216 if (sc->intr_count == 1) 3217 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all")); 3218 3219 /* Multiple interrupts. */ 3220 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 3221 ("%s: too few intr.", __func__)); 3222 3223 /* The first one is always error intr */ 3224 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 3225 if (rc != 0) 3226 return (rc); 3227 irq++; 3228 rid++; 3229 3230 /* The second one is always the firmware event queue */ 3231 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt"); 3232 if (rc != 0) 3233 return (rc); 3234 irq++; 3235 rid++; 3236 3237 for_each_port(sc, p) { 3238 pi = sc->port[p]; 3239 3240 if (pi->flags & INTR_RXQ) { 3241 for_each_rxq(pi, q, rxq) { 3242 snprintf(s, sizeof(s), "%d.%d", p, q); 3243 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq, 3244 s); 3245 if (rc != 0) 3246 return (rc); 3247 irq++; 3248 rid++; 3249 } 3250 } 3251 #ifdef TCP_OFFLOAD 3252 if (pi->flags & INTR_OFLD_RXQ) { 3253 for_each_ofld_rxq(pi, q, ofld_rxq) { 3254 snprintf(s, sizeof(s), "%d,%d", p, q); 3255 rc = t4_alloc_irq(sc, irq, rid, t4_intr, 3256 ofld_rxq, s); 3257 if (rc != 0) 3258 return (rc); 3259 irq++; 3260 rid++; 3261 } 3262 } 3263 #endif 3264 #ifdef DEV_NETMAP 3265 if (pi->flags & INTR_NM_RXQ) { 3266 for_each_nm_rxq(pi, q, nm_rxq) { 3267 snprintf(s, sizeof(s), "%d-%d", p, q); 3268 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr, 3269 nm_rxq, s); 3270 if (rc != 0) 3271 return (rc); 3272 irq++; 3273 rid++; 3274 } 3275 } 3276 #endif 3277 } 3278 MPASS(irq == &sc->irq[sc->intr_count]); 3279 3280 return (0); 3281 } 3282 3283 int 3284 adapter_full_init(struct adapter *sc) 3285 { 3286 int rc, i; 3287 3288 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3289 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 3290 ("%s: FULL_INIT_DONE already", __func__)); 3291 3292 /* 3293 * queues that belong to the adapter (not any particular port). 3294 */ 3295 rc = t4_setup_adapter_queues(sc); 3296 if (rc != 0) 3297 goto done; 3298 3299 for (i = 0; i < nitems(sc->tq); i++) { 3300 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 3301 taskqueue_thread_enqueue, &sc->tq[i]); 3302 if (sc->tq[i] == NULL) { 3303 device_printf(sc->dev, 3304 "failed to allocate task queue %d\n", i); 3305 rc = ENOMEM; 3306 goto done; 3307 } 3308 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3309 device_get_nameunit(sc->dev), i); 3310 } 3311 3312 t4_intr_enable(sc); 3313 sc->flags |= FULL_INIT_DONE; 3314 done: 3315 if (rc != 0) 3316 adapter_full_uninit(sc); 3317 3318 return (rc); 3319 } 3320 3321 int 3322 adapter_full_uninit(struct adapter *sc) 3323 { 3324 int i; 3325 3326 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3327 3328 t4_teardown_adapter_queues(sc); 3329 3330 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3331 taskqueue_free(sc->tq[i]); 3332 sc->tq[i] = NULL; 3333 } 3334 3335 sc->flags &= ~FULL_INIT_DONE; 3336 3337 return (0); 3338 } 3339 3340 int 3341 port_full_init(struct port_info *pi) 3342 { 3343 struct adapter *sc = pi->adapter; 3344 struct ifnet *ifp = pi->ifp; 3345 uint16_t *rss; 3346 struct sge_rxq *rxq; 3347 int rc, i, j; 3348 3349 ASSERT_SYNCHRONIZED_OP(sc); 3350 KASSERT((pi->flags & PORT_INIT_DONE) == 0, 3351 ("%s: PORT_INIT_DONE already", __func__)); 3352 3353 sysctl_ctx_init(&pi->ctx); 3354 pi->flags |= PORT_SYSCTL_CTX; 3355 3356 /* 3357 * Allocate tx/rx/fl queues for this port. 3358 */ 3359 rc = t4_setup_port_queues(pi); 3360 if (rc != 0) 3361 goto done; /* error message displayed already */ 3362 3363 /* 3364 * Setup RSS for this port. Save a copy of the RSS table for later use. 3365 */ 3366 rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); 3367 for (i = 0; i < pi->rss_size;) { 3368 for_each_rxq(pi, j, rxq) { 3369 rss[i++] = rxq->iq.abs_id; 3370 if (i == pi->rss_size) 3371 break; 3372 } 3373 } 3374 3375 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss, 3376 pi->rss_size); 3377 if (rc != 0) { 3378 if_printf(ifp, "rss_config failed: %d\n", rc); 3379 goto done; 3380 } 3381 3382 pi->rss = rss; 3383 pi->flags |= PORT_INIT_DONE; 3384 done: 3385 if (rc != 0) 3386 port_full_uninit(pi); 3387 3388 return (rc); 3389 } 3390 3391 /* 3392 * Idempotent. 3393 */ 3394 int 3395 port_full_uninit(struct port_info *pi) 3396 { 3397 struct adapter *sc = pi->adapter; 3398 int i; 3399 struct sge_rxq *rxq; 3400 struct sge_txq *txq; 3401 #ifdef TCP_OFFLOAD 3402 struct sge_ofld_rxq *ofld_rxq; 3403 struct sge_wrq *ofld_txq; 3404 #endif 3405 3406 if (pi->flags & PORT_INIT_DONE) { 3407 3408 /* Need to quiesce queues. XXX: ctrl queues? */ 3409 3410 for_each_txq(pi, i, txq) { 3411 quiesce_eq(sc, &txq->eq); 3412 } 3413 3414 #ifdef TCP_OFFLOAD 3415 for_each_ofld_txq(pi, i, ofld_txq) { 3416 quiesce_eq(sc, &ofld_txq->eq); 3417 } 3418 #endif 3419 3420 for_each_rxq(pi, i, rxq) { 3421 quiesce_iq(sc, &rxq->iq); 3422 quiesce_fl(sc, &rxq->fl); 3423 } 3424 3425 #ifdef TCP_OFFLOAD 3426 for_each_ofld_rxq(pi, i, ofld_rxq) { 3427 quiesce_iq(sc, &ofld_rxq->iq); 3428 quiesce_fl(sc, &ofld_rxq->fl); 3429 } 3430 #endif 3431 free(pi->rss, M_CXGBE); 3432 } 3433 3434 t4_teardown_port_queues(pi); 3435 pi->flags &= ~PORT_INIT_DONE; 3436 3437 return (0); 3438 } 3439 3440 static void 3441 quiesce_eq(struct adapter *sc, struct sge_eq *eq) 3442 { 3443 EQ_LOCK(eq); 3444 eq->flags |= EQ_DOOMED; 3445 3446 /* 3447 * Wait for the response to a credit flush if one's 3448 * pending. 3449 */ 3450 while (eq->flags & EQ_CRFLUSHED) 3451 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0); 3452 EQ_UNLOCK(eq); 3453 3454 callout_drain(&eq->tx_callout); /* XXX: iffy */ 3455 pause("callout", 10); /* Still iffy */ 3456 3457 taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task); 3458 } 3459 3460 static void 3461 quiesce_iq(struct adapter *sc, struct sge_iq *iq) 3462 { 3463 (void) sc; /* unused */ 3464 3465 /* Synchronize with the interrupt handler */ 3466 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 3467 pause("iqfree", 1); 3468 } 3469 3470 static void 3471 quiesce_fl(struct adapter *sc, struct sge_fl *fl) 3472 { 3473 mtx_lock(&sc->sfl_lock); 3474 FL_LOCK(fl); 3475 fl->flags |= FL_DOOMED; 3476 FL_UNLOCK(fl); 3477 mtx_unlock(&sc->sfl_lock); 3478 3479 callout_drain(&sc->sfl_callout); 3480 KASSERT((fl->flags & FL_STARVING) == 0, 3481 ("%s: still starving", __func__)); 3482 } 3483 3484 static int 3485 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 3486 driver_intr_t *handler, void *arg, char *name) 3487 { 3488 int rc; 3489 3490 irq->rid = rid; 3491 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 3492 RF_SHAREABLE | RF_ACTIVE); 3493 if (irq->res == NULL) { 3494 device_printf(sc->dev, 3495 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 3496 return (ENOMEM); 3497 } 3498 3499 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 3500 NULL, handler, arg, &irq->tag); 3501 if (rc != 0) { 3502 device_printf(sc->dev, 3503 "failed to setup interrupt for rid %d, name %s: %d\n", 3504 rid, name, rc); 3505 } else if (name) 3506 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 3507 3508 return (rc); 3509 } 3510 3511 static int 3512 t4_free_irq(struct adapter *sc, struct irq *irq) 3513 { 3514 if (irq->tag) 3515 bus_teardown_intr(sc->dev, irq->res, irq->tag); 3516 if (irq->res) 3517 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 3518 3519 bzero(irq, sizeof(*irq)); 3520 3521 return (0); 3522 } 3523 3524 static void 3525 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 3526 unsigned int end) 3527 { 3528 uint32_t *p = (uint32_t *)(buf + start); 3529 3530 for ( ; start <= end; start += sizeof(uint32_t)) 3531 *p++ = t4_read_reg(sc, start); 3532 } 3533 3534 static void 3535 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 3536 { 3537 int i, n; 3538 const unsigned int *reg_ranges; 3539 static const unsigned int t4_reg_ranges[] = { 3540 0x1008, 0x1108, 3541 0x1180, 0x11b4, 3542 0x11fc, 0x123c, 3543 0x1300, 0x173c, 3544 0x1800, 0x18fc, 3545 0x3000, 0x30d8, 3546 0x30e0, 0x5924, 3547 0x5960, 0x59d4, 3548 0x5a00, 0x5af8, 3549 0x6000, 0x6098, 3550 0x6100, 0x6150, 3551 0x6200, 0x6208, 3552 0x6240, 0x6248, 3553 0x6280, 0x6338, 3554 0x6370, 0x638c, 3555 0x6400, 0x643c, 3556 0x6500, 0x6524, 3557 0x6a00, 0x6a38, 3558 0x6a60, 0x6a78, 3559 0x6b00, 0x6b84, 3560 0x6bf0, 0x6c84, 3561 0x6cf0, 0x6d84, 3562 0x6df0, 0x6e84, 3563 0x6ef0, 0x6f84, 3564 0x6ff0, 0x7084, 3565 0x70f0, 0x7184, 3566 0x71f0, 0x7284, 3567 0x72f0, 0x7384, 3568 0x73f0, 0x7450, 3569 0x7500, 0x7530, 3570 0x7600, 0x761c, 3571 0x7680, 0x76cc, 3572 0x7700, 0x7798, 3573 0x77c0, 0x77fc, 3574 0x7900, 0x79fc, 3575 0x7b00, 0x7c38, 3576 0x7d00, 0x7efc, 3577 0x8dc0, 0x8e1c, 3578 0x8e30, 0x8e78, 3579 0x8ea0, 0x8f6c, 3580 0x8fc0, 0x9074, 3581 0x90fc, 0x90fc, 3582 0x9400, 0x9458, 3583 0x9600, 0x96bc, 3584 0x9800, 0x9808, 3585 0x9820, 0x983c, 3586 0x9850, 0x9864, 3587 0x9c00, 0x9c6c, 3588 0x9c80, 0x9cec, 3589 0x9d00, 0x9d6c, 3590 0x9d80, 0x9dec, 3591 0x9e00, 0x9e6c, 3592 0x9e80, 0x9eec, 3593 0x9f00, 0x9f6c, 3594 0x9f80, 0x9fec, 3595 0xd004, 0xd03c, 3596 0xdfc0, 0xdfe0, 3597 0xe000, 0xea7c, 3598 0xf000, 0x11110, 3599 0x11118, 0x11190, 3600 0x19040, 0x1906c, 3601 0x19078, 0x19080, 3602 0x1908c, 0x19124, 3603 0x19150, 0x191b0, 3604 0x191d0, 0x191e8, 3605 0x19238, 0x1924c, 3606 0x193f8, 0x19474, 3607 0x19490, 0x194f8, 3608 0x19800, 0x19f30, 3609 0x1a000, 0x1a06c, 3610 0x1a0b0, 0x1a120, 3611 0x1a128, 0x1a138, 3612 0x1a190, 0x1a1c4, 3613 0x1a1fc, 0x1a1fc, 3614 0x1e040, 0x1e04c, 3615 0x1e284, 0x1e28c, 3616 0x1e2c0, 0x1e2c0, 3617 0x1e2e0, 0x1e2e0, 3618 0x1e300, 0x1e384, 3619 0x1e3c0, 0x1e3c8, 3620 0x1e440, 0x1e44c, 3621 0x1e684, 0x1e68c, 3622 0x1e6c0, 0x1e6c0, 3623 0x1e6e0, 0x1e6e0, 3624 0x1e700, 0x1e784, 3625 0x1e7c0, 0x1e7c8, 3626 0x1e840, 0x1e84c, 3627 0x1ea84, 0x1ea8c, 3628 0x1eac0, 0x1eac0, 3629 0x1eae0, 0x1eae0, 3630 0x1eb00, 0x1eb84, 3631 0x1ebc0, 0x1ebc8, 3632 0x1ec40, 0x1ec4c, 3633 0x1ee84, 0x1ee8c, 3634 0x1eec0, 0x1eec0, 3635 0x1eee0, 0x1eee0, 3636 0x1ef00, 0x1ef84, 3637 0x1efc0, 0x1efc8, 3638 0x1f040, 0x1f04c, 3639 0x1f284, 0x1f28c, 3640 0x1f2c0, 0x1f2c0, 3641 0x1f2e0, 0x1f2e0, 3642 0x1f300, 0x1f384, 3643 0x1f3c0, 0x1f3c8, 3644 0x1f440, 0x1f44c, 3645 0x1f684, 0x1f68c, 3646 0x1f6c0, 0x1f6c0, 3647 0x1f6e0, 0x1f6e0, 3648 0x1f700, 0x1f784, 3649 0x1f7c0, 0x1f7c8, 3650 0x1f840, 0x1f84c, 3651 0x1fa84, 0x1fa8c, 3652 0x1fac0, 0x1fac0, 3653 0x1fae0, 0x1fae0, 3654 0x1fb00, 0x1fb84, 3655 0x1fbc0, 0x1fbc8, 3656 0x1fc40, 0x1fc4c, 3657 0x1fe84, 0x1fe8c, 3658 0x1fec0, 0x1fec0, 3659 0x1fee0, 0x1fee0, 3660 0x1ff00, 0x1ff84, 3661 0x1ffc0, 0x1ffc8, 3662 0x20000, 0x2002c, 3663 0x20100, 0x2013c, 3664 0x20190, 0x201c8, 3665 0x20200, 0x20318, 3666 0x20400, 0x20528, 3667 0x20540, 0x20614, 3668 0x21000, 0x21040, 3669 0x2104c, 0x21060, 3670 0x210c0, 0x210ec, 3671 0x21200, 0x21268, 3672 0x21270, 0x21284, 3673 0x212fc, 0x21388, 3674 0x21400, 0x21404, 3675 0x21500, 0x21518, 3676 0x2152c, 0x2153c, 3677 0x21550, 0x21554, 3678 0x21600, 0x21600, 3679 0x21608, 0x21628, 3680 0x21630, 0x2163c, 3681 0x21700, 0x2171c, 3682 0x21780, 0x2178c, 3683 0x21800, 0x21c38, 3684 0x21c80, 0x21d7c, 3685 0x21e00, 0x21e04, 3686 0x22000, 0x2202c, 3687 0x22100, 0x2213c, 3688 0x22190, 0x221c8, 3689 0x22200, 0x22318, 3690 0x22400, 0x22528, 3691 0x22540, 0x22614, 3692 0x23000, 0x23040, 3693 0x2304c, 0x23060, 3694 0x230c0, 0x230ec, 3695 0x23200, 0x23268, 3696 0x23270, 0x23284, 3697 0x232fc, 0x23388, 3698 0x23400, 0x23404, 3699 0x23500, 0x23518, 3700 0x2352c, 0x2353c, 3701 0x23550, 0x23554, 3702 0x23600, 0x23600, 3703 0x23608, 0x23628, 3704 0x23630, 0x2363c, 3705 0x23700, 0x2371c, 3706 0x23780, 0x2378c, 3707 0x23800, 0x23c38, 3708 0x23c80, 0x23d7c, 3709 0x23e00, 0x23e04, 3710 0x24000, 0x2402c, 3711 0x24100, 0x2413c, 3712 0x24190, 0x241c8, 3713 0x24200, 0x24318, 3714 0x24400, 0x24528, 3715 0x24540, 0x24614, 3716 0x25000, 0x25040, 3717 0x2504c, 0x25060, 3718 0x250c0, 0x250ec, 3719 0x25200, 0x25268, 3720 0x25270, 0x25284, 3721 0x252fc, 0x25388, 3722 0x25400, 0x25404, 3723 0x25500, 0x25518, 3724 0x2552c, 0x2553c, 3725 0x25550, 0x25554, 3726 0x25600, 0x25600, 3727 0x25608, 0x25628, 3728 0x25630, 0x2563c, 3729 0x25700, 0x2571c, 3730 0x25780, 0x2578c, 3731 0x25800, 0x25c38, 3732 0x25c80, 0x25d7c, 3733 0x25e00, 0x25e04, 3734 0x26000, 0x2602c, 3735 0x26100, 0x2613c, 3736 0x26190, 0x261c8, 3737 0x26200, 0x26318, 3738 0x26400, 0x26528, 3739 0x26540, 0x26614, 3740 0x27000, 0x27040, 3741 0x2704c, 0x27060, 3742 0x270c0, 0x270ec, 3743 0x27200, 0x27268, 3744 0x27270, 0x27284, 3745 0x272fc, 0x27388, 3746 0x27400, 0x27404, 3747 0x27500, 0x27518, 3748 0x2752c, 0x2753c, 3749 0x27550, 0x27554, 3750 0x27600, 0x27600, 3751 0x27608, 0x27628, 3752 0x27630, 0x2763c, 3753 0x27700, 0x2771c, 3754 0x27780, 0x2778c, 3755 0x27800, 0x27c38, 3756 0x27c80, 0x27d7c, 3757 0x27e00, 0x27e04 3758 }; 3759 static const unsigned int t5_reg_ranges[] = { 3760 0x1008, 0x1148, 3761 0x1180, 0x11b4, 3762 0x11fc, 0x123c, 3763 0x1280, 0x173c, 3764 0x1800, 0x18fc, 3765 0x3000, 0x3028, 3766 0x3060, 0x30d8, 3767 0x30e0, 0x30fc, 3768 0x3140, 0x357c, 3769 0x35a8, 0x35cc, 3770 0x35ec, 0x35ec, 3771 0x3600, 0x5624, 3772 0x56cc, 0x575c, 3773 0x580c, 0x5814, 3774 0x5890, 0x58bc, 3775 0x5940, 0x59dc, 3776 0x59fc, 0x5a18, 3777 0x5a60, 0x5a9c, 3778 0x5b94, 0x5bfc, 3779 0x6000, 0x6040, 3780 0x6058, 0x614c, 3781 0x7700, 0x7798, 3782 0x77c0, 0x78fc, 3783 0x7b00, 0x7c54, 3784 0x7d00, 0x7efc, 3785 0x8dc0, 0x8de0, 3786 0x8df8, 0x8e84, 3787 0x8ea0, 0x8f84, 3788 0x8fc0, 0x90f8, 3789 0x9400, 0x9470, 3790 0x9600, 0x96f4, 3791 0x9800, 0x9808, 3792 0x9820, 0x983c, 3793 0x9850, 0x9864, 3794 0x9c00, 0x9c6c, 3795 0x9c80, 0x9cec, 3796 0x9d00, 0x9d6c, 3797 0x9d80, 0x9dec, 3798 0x9e00, 0x9e6c, 3799 0x9e80, 0x9eec, 3800 0x9f00, 0x9f6c, 3801 0x9f80, 0xa020, 3802 0xd004, 0xd03c, 3803 0xdfc0, 0xdfe0, 3804 0xe000, 0x11088, 3805 0x1109c, 0x11110, 3806 0x11118, 0x1117c, 3807 0x11190, 0x11204, 3808 0x19040, 0x1906c, 3809 0x19078, 0x19080, 3810 0x1908c, 0x19124, 3811 0x19150, 0x191b0, 3812 0x191d0, 0x191e8, 3813 0x19238, 0x19290, 3814 0x193f8, 0x19474, 3815 0x19490, 0x194cc, 3816 0x194f0, 0x194f8, 3817 0x19c00, 0x19c60, 3818 0x19c94, 0x19e10, 3819 0x19e50, 0x19f34, 3820 0x19f40, 0x19f50, 3821 0x19f90, 0x19fe4, 3822 0x1a000, 0x1a06c, 3823 0x1a0b0, 0x1a120, 3824 0x1a128, 0x1a138, 3825 0x1a190, 0x1a1c4, 3826 0x1a1fc, 0x1a1fc, 3827 0x1e008, 0x1e00c, 3828 0x1e040, 0x1e04c, 3829 0x1e284, 0x1e290, 3830 0x1e2c0, 0x1e2c0, 3831 0x1e2e0, 0x1e2e0, 3832 0x1e300, 0x1e384, 3833 0x1e3c0, 0x1e3c8, 3834 0x1e408, 0x1e40c, 3835 0x1e440, 0x1e44c, 3836 0x1e684, 0x1e690, 3837 0x1e6c0, 0x1e6c0, 3838 0x1e6e0, 0x1e6e0, 3839 0x1e700, 0x1e784, 3840 0x1e7c0, 0x1e7c8, 3841 0x1e808, 0x1e80c, 3842 0x1e840, 0x1e84c, 3843 0x1ea84, 0x1ea90, 3844 0x1eac0, 0x1eac0, 3845 0x1eae0, 0x1eae0, 3846 0x1eb00, 0x1eb84, 3847 0x1ebc0, 0x1ebc8, 3848 0x1ec08, 0x1ec0c, 3849 0x1ec40, 0x1ec4c, 3850 0x1ee84, 0x1ee90, 3851 0x1eec0, 0x1eec0, 3852 0x1eee0, 0x1eee0, 3853 0x1ef00, 0x1ef84, 3854 0x1efc0, 0x1efc8, 3855 0x1f008, 0x1f00c, 3856 0x1f040, 0x1f04c, 3857 0x1f284, 0x1f290, 3858 0x1f2c0, 0x1f2c0, 3859 0x1f2e0, 0x1f2e0, 3860 0x1f300, 0x1f384, 3861 0x1f3c0, 0x1f3c8, 3862 0x1f408, 0x1f40c, 3863 0x1f440, 0x1f44c, 3864 0x1f684, 0x1f690, 3865 0x1f6c0, 0x1f6c0, 3866 0x1f6e0, 0x1f6e0, 3867 0x1f700, 0x1f784, 3868 0x1f7c0, 0x1f7c8, 3869 0x1f808, 0x1f80c, 3870 0x1f840, 0x1f84c, 3871 0x1fa84, 0x1fa90, 3872 0x1fac0, 0x1fac0, 3873 0x1fae0, 0x1fae0, 3874 0x1fb00, 0x1fb84, 3875 0x1fbc0, 0x1fbc8, 3876 0x1fc08, 0x1fc0c, 3877 0x1fc40, 0x1fc4c, 3878 0x1fe84, 0x1fe90, 3879 0x1fec0, 0x1fec0, 3880 0x1fee0, 0x1fee0, 3881 0x1ff00, 0x1ff84, 3882 0x1ffc0, 0x1ffc8, 3883 0x30000, 0x30030, 3884 0x30100, 0x30144, 3885 0x30190, 0x301d0, 3886 0x30200, 0x30318, 3887 0x30400, 0x3052c, 3888 0x30540, 0x3061c, 3889 0x30800, 0x30834, 3890 0x308c0, 0x30908, 3891 0x30910, 0x309ac, 3892 0x30a00, 0x30a2c, 3893 0x30a44, 0x30a50, 3894 0x30a74, 0x30c24, 3895 0x30d00, 0x30d00, 3896 0x30d08, 0x30d14, 3897 0x30d1c, 0x30d20, 3898 0x30d3c, 0x30d50, 3899 0x31200, 0x3120c, 3900 0x31220, 0x31220, 3901 0x31240, 0x31240, 3902 0x31600, 0x3160c, 3903 0x31a00, 0x31a1c, 3904 0x31e00, 0x31e20, 3905 0x31e38, 0x31e3c, 3906 0x31e80, 0x31e80, 3907 0x31e88, 0x31ea8, 3908 0x31eb0, 0x31eb4, 3909 0x31ec8, 0x31ed4, 3910 0x31fb8, 0x32004, 3911 0x32200, 0x32200, 3912 0x32208, 0x32240, 3913 0x32248, 0x32280, 3914 0x32288, 0x322c0, 3915 0x322c8, 0x322fc, 3916 0x32600, 0x32630, 3917 0x32a00, 0x32abc, 3918 0x32b00, 0x32b70, 3919 0x33000, 0x33048, 3920 0x33060, 0x3309c, 3921 0x330f0, 0x33148, 3922 0x33160, 0x3319c, 3923 0x331f0, 0x332e4, 3924 0x332f8, 0x333e4, 3925 0x333f8, 0x33448, 3926 0x33460, 0x3349c, 3927 0x334f0, 0x33548, 3928 0x33560, 0x3359c, 3929 0x335f0, 0x336e4, 3930 0x336f8, 0x337e4, 3931 0x337f8, 0x337fc, 3932 0x33814, 0x33814, 3933 0x3382c, 0x3382c, 3934 0x33880, 0x3388c, 3935 0x338e8, 0x338ec, 3936 0x33900, 0x33948, 3937 0x33960, 0x3399c, 3938 0x339f0, 0x33ae4, 3939 0x33af8, 0x33b10, 3940 0x33b28, 0x33b28, 3941 0x33b3c, 0x33b50, 3942 0x33bf0, 0x33c10, 3943 0x33c28, 0x33c28, 3944 0x33c3c, 0x33c50, 3945 0x33cf0, 0x33cfc, 3946 0x34000, 0x34030, 3947 0x34100, 0x34144, 3948 0x34190, 0x341d0, 3949 0x34200, 0x34318, 3950 0x34400, 0x3452c, 3951 0x34540, 0x3461c, 3952 0x34800, 0x34834, 3953 0x348c0, 0x34908, 3954 0x34910, 0x349ac, 3955 0x34a00, 0x34a2c, 3956 0x34a44, 0x34a50, 3957 0x34a74, 0x34c24, 3958 0x34d00, 0x34d00, 3959 0x34d08, 0x34d14, 3960 0x34d1c, 0x34d20, 3961 0x34d3c, 0x34d50, 3962 0x35200, 0x3520c, 3963 0x35220, 0x35220, 3964 0x35240, 0x35240, 3965 0x35600, 0x3560c, 3966 0x35a00, 0x35a1c, 3967 0x35e00, 0x35e20, 3968 0x35e38, 0x35e3c, 3969 0x35e80, 0x35e80, 3970 0x35e88, 0x35ea8, 3971 0x35eb0, 0x35eb4, 3972 0x35ec8, 0x35ed4, 3973 0x35fb8, 0x36004, 3974 0x36200, 0x36200, 3975 0x36208, 0x36240, 3976 0x36248, 0x36280, 3977 0x36288, 0x362c0, 3978 0x362c8, 0x362fc, 3979 0x36600, 0x36630, 3980 0x36a00, 0x36abc, 3981 0x36b00, 0x36b70, 3982 0x37000, 0x37048, 3983 0x37060, 0x3709c, 3984 0x370f0, 0x37148, 3985 0x37160, 0x3719c, 3986 0x371f0, 0x372e4, 3987 0x372f8, 0x373e4, 3988 0x373f8, 0x37448, 3989 0x37460, 0x3749c, 3990 0x374f0, 0x37548, 3991 0x37560, 0x3759c, 3992 0x375f0, 0x376e4, 3993 0x376f8, 0x377e4, 3994 0x377f8, 0x377fc, 3995 0x37814, 0x37814, 3996 0x3782c, 0x3782c, 3997 0x37880, 0x3788c, 3998 0x378e8, 0x378ec, 3999 0x37900, 0x37948, 4000 0x37960, 0x3799c, 4001 0x379f0, 0x37ae4, 4002 0x37af8, 0x37b10, 4003 0x37b28, 0x37b28, 4004 0x37b3c, 0x37b50, 4005 0x37bf0, 0x37c10, 4006 0x37c28, 0x37c28, 4007 0x37c3c, 0x37c50, 4008 0x37cf0, 0x37cfc, 4009 0x38000, 0x38030, 4010 0x38100, 0x38144, 4011 0x38190, 0x381d0, 4012 0x38200, 0x38318, 4013 0x38400, 0x3852c, 4014 0x38540, 0x3861c, 4015 0x38800, 0x38834, 4016 0x388c0, 0x38908, 4017 0x38910, 0x389ac, 4018 0x38a00, 0x38a2c, 4019 0x38a44, 0x38a50, 4020 0x38a74, 0x38c24, 4021 0x38d00, 0x38d00, 4022 0x38d08, 0x38d14, 4023 0x38d1c, 0x38d20, 4024 0x38d3c, 0x38d50, 4025 0x39200, 0x3920c, 4026 0x39220, 0x39220, 4027 0x39240, 0x39240, 4028 0x39600, 0x3960c, 4029 0x39a00, 0x39a1c, 4030 0x39e00, 0x39e20, 4031 0x39e38, 0x39e3c, 4032 0x39e80, 0x39e80, 4033 0x39e88, 0x39ea8, 4034 0x39eb0, 0x39eb4, 4035 0x39ec8, 0x39ed4, 4036 0x39fb8, 0x3a004, 4037 0x3a200, 0x3a200, 4038 0x3a208, 0x3a240, 4039 0x3a248, 0x3a280, 4040 0x3a288, 0x3a2c0, 4041 0x3a2c8, 0x3a2fc, 4042 0x3a600, 0x3a630, 4043 0x3aa00, 0x3aabc, 4044 0x3ab00, 0x3ab70, 4045 0x3b000, 0x3b048, 4046 0x3b060, 0x3b09c, 4047 0x3b0f0, 0x3b148, 4048 0x3b160, 0x3b19c, 4049 0x3b1f0, 0x3b2e4, 4050 0x3b2f8, 0x3b3e4, 4051 0x3b3f8, 0x3b448, 4052 0x3b460, 0x3b49c, 4053 0x3b4f0, 0x3b548, 4054 0x3b560, 0x3b59c, 4055 0x3b5f0, 0x3b6e4, 4056 0x3b6f8, 0x3b7e4, 4057 0x3b7f8, 0x3b7fc, 4058 0x3b814, 0x3b814, 4059 0x3b82c, 0x3b82c, 4060 0x3b880, 0x3b88c, 4061 0x3b8e8, 0x3b8ec, 4062 0x3b900, 0x3b948, 4063 0x3b960, 0x3b99c, 4064 0x3b9f0, 0x3bae4, 4065 0x3baf8, 0x3bb10, 4066 0x3bb28, 0x3bb28, 4067 0x3bb3c, 0x3bb50, 4068 0x3bbf0, 0x3bc10, 4069 0x3bc28, 0x3bc28, 4070 0x3bc3c, 0x3bc50, 4071 0x3bcf0, 0x3bcfc, 4072 0x3c000, 0x3c030, 4073 0x3c100, 0x3c144, 4074 0x3c190, 0x3c1d0, 4075 0x3c200, 0x3c318, 4076 0x3c400, 0x3c52c, 4077 0x3c540, 0x3c61c, 4078 0x3c800, 0x3c834, 4079 0x3c8c0, 0x3c908, 4080 0x3c910, 0x3c9ac, 4081 0x3ca00, 0x3ca2c, 4082 0x3ca44, 0x3ca50, 4083 0x3ca74, 0x3cc24, 4084 0x3cd00, 0x3cd00, 4085 0x3cd08, 0x3cd14, 4086 0x3cd1c, 0x3cd20, 4087 0x3cd3c, 0x3cd50, 4088 0x3d200, 0x3d20c, 4089 0x3d220, 0x3d220, 4090 0x3d240, 0x3d240, 4091 0x3d600, 0x3d60c, 4092 0x3da00, 0x3da1c, 4093 0x3de00, 0x3de20, 4094 0x3de38, 0x3de3c, 4095 0x3de80, 0x3de80, 4096 0x3de88, 0x3dea8, 4097 0x3deb0, 0x3deb4, 4098 0x3dec8, 0x3ded4, 4099 0x3dfb8, 0x3e004, 4100 0x3e200, 0x3e200, 4101 0x3e208, 0x3e240, 4102 0x3e248, 0x3e280, 4103 0x3e288, 0x3e2c0, 4104 0x3e2c8, 0x3e2fc, 4105 0x3e600, 0x3e630, 4106 0x3ea00, 0x3eabc, 4107 0x3eb00, 0x3eb70, 4108 0x3f000, 0x3f048, 4109 0x3f060, 0x3f09c, 4110 0x3f0f0, 0x3f148, 4111 0x3f160, 0x3f19c, 4112 0x3f1f0, 0x3f2e4, 4113 0x3f2f8, 0x3f3e4, 4114 0x3f3f8, 0x3f448, 4115 0x3f460, 0x3f49c, 4116 0x3f4f0, 0x3f548, 4117 0x3f560, 0x3f59c, 4118 0x3f5f0, 0x3f6e4, 4119 0x3f6f8, 0x3f7e4, 4120 0x3f7f8, 0x3f7fc, 4121 0x3f814, 0x3f814, 4122 0x3f82c, 0x3f82c, 4123 0x3f880, 0x3f88c, 4124 0x3f8e8, 0x3f8ec, 4125 0x3f900, 0x3f948, 4126 0x3f960, 0x3f99c, 4127 0x3f9f0, 0x3fae4, 4128 0x3faf8, 0x3fb10, 4129 0x3fb28, 0x3fb28, 4130 0x3fb3c, 0x3fb50, 4131 0x3fbf0, 0x3fc10, 4132 0x3fc28, 0x3fc28, 4133 0x3fc3c, 0x3fc50, 4134 0x3fcf0, 0x3fcfc, 4135 0x40000, 0x4000c, 4136 0x40040, 0x40068, 4137 0x4007c, 0x40144, 4138 0x40180, 0x4018c, 4139 0x40200, 0x40298, 4140 0x402ac, 0x4033c, 4141 0x403f8, 0x403fc, 4142 0x41304, 0x413c4, 4143 0x41400, 0x4141c, 4144 0x41480, 0x414d0, 4145 0x44000, 0x44078, 4146 0x440c0, 0x44278, 4147 0x442c0, 0x44478, 4148 0x444c0, 0x44678, 4149 0x446c0, 0x44878, 4150 0x448c0, 0x449fc, 4151 0x45000, 0x45068, 4152 0x45080, 0x45084, 4153 0x450a0, 0x450b0, 4154 0x45200, 0x45268, 4155 0x45280, 0x45284, 4156 0x452a0, 0x452b0, 4157 0x460c0, 0x460e4, 4158 0x47000, 0x4708c, 4159 0x47200, 0x47250, 4160 0x47400, 0x47420, 4161 0x47600, 0x47618, 4162 0x47800, 0x47814, 4163 0x48000, 0x4800c, 4164 0x48040, 0x48068, 4165 0x4807c, 0x48144, 4166 0x48180, 0x4818c, 4167 0x48200, 0x48298, 4168 0x482ac, 0x4833c, 4169 0x483f8, 0x483fc, 4170 0x49304, 0x493c4, 4171 0x49400, 0x4941c, 4172 0x49480, 0x494d0, 4173 0x4c000, 0x4c078, 4174 0x4c0c0, 0x4c278, 4175 0x4c2c0, 0x4c478, 4176 0x4c4c0, 0x4c678, 4177 0x4c6c0, 0x4c878, 4178 0x4c8c0, 0x4c9fc, 4179 0x4d000, 0x4d068, 4180 0x4d080, 0x4d084, 4181 0x4d0a0, 0x4d0b0, 4182 0x4d200, 0x4d268, 4183 0x4d280, 0x4d284, 4184 0x4d2a0, 0x4d2b0, 4185 0x4e0c0, 0x4e0e4, 4186 0x4f000, 0x4f08c, 4187 0x4f200, 0x4f250, 4188 0x4f400, 0x4f420, 4189 0x4f600, 0x4f618, 4190 0x4f800, 0x4f814, 4191 0x50000, 0x500cc, 4192 0x50400, 0x50400, 4193 0x50800, 0x508cc, 4194 0x50c00, 0x50c00, 4195 0x51000, 0x5101c, 4196 0x51300, 0x51308, 4197 }; 4198 4199 if (is_t4(sc)) { 4200 reg_ranges = &t4_reg_ranges[0]; 4201 n = nitems(t4_reg_ranges); 4202 } else { 4203 reg_ranges = &t5_reg_ranges[0]; 4204 n = nitems(t5_reg_ranges); 4205 } 4206 4207 regs->version = chip_id(sc) | chip_rev(sc) << 10; 4208 for (i = 0; i < n; i += 2) 4209 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 4210 } 4211 4212 static void 4213 cxgbe_tick(void *arg) 4214 { 4215 struct port_info *pi = arg; 4216 struct adapter *sc = pi->adapter; 4217 struct ifnet *ifp = pi->ifp; 4218 struct sge_txq *txq; 4219 int i, drops; 4220 struct port_stats *s = &pi->stats; 4221 4222 PORT_LOCK(pi); 4223 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 4224 PORT_UNLOCK(pi); 4225 return; /* without scheduling another callout */ 4226 } 4227 4228 t4_get_port_stats(sc, pi->tx_chan, s); 4229 4230 ifp->if_opackets = s->tx_frames - s->tx_pause; 4231 ifp->if_ipackets = s->rx_frames - s->rx_pause; 4232 ifp->if_obytes = s->tx_octets - s->tx_pause * 64; 4233 ifp->if_ibytes = s->rx_octets - s->rx_pause * 64; 4234 ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause; 4235 ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause; 4236 ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 4237 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 4238 s->rx_trunc3; 4239 for (i = 0; i < 4; i++) { 4240 if (pi->rx_chan_map & (1 << i)) { 4241 uint32_t v; 4242 4243 /* 4244 * XXX: indirect reads from the same ADDR/DATA pair can 4245 * race with each other. 4246 */ 4247 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 4248 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 4249 ifp->if_iqdrops += v; 4250 } 4251 } 4252 4253 drops = s->tx_drop; 4254 for_each_txq(pi, i, txq) 4255 drops += txq->br->br_drops; 4256 ifp->if_oqdrops = drops; 4257 4258 ifp->if_oerrors = s->tx_error_frames; 4259 ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long + 4260 s->rx_fcs_err + s->rx_len_err; 4261 4262 callout_schedule(&pi->tick, hz); 4263 PORT_UNLOCK(pi); 4264 } 4265 4266 static void 4267 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 4268 { 4269 struct ifnet *vlan; 4270 4271 if (arg != ifp || ifp->if_type != IFT_ETHER) 4272 return; 4273 4274 vlan = VLAN_DEVAT(ifp, vid); 4275 VLAN_SETCOOKIE(vlan, ifp); 4276 } 4277 4278 static int 4279 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 4280 { 4281 4282 #ifdef INVARIANTS 4283 panic("%s: opcode 0x%02x on iq %p with payload %p", 4284 __func__, rss->opcode, iq, m); 4285 #else 4286 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 4287 __func__, rss->opcode, iq, m); 4288 m_freem(m); 4289 #endif 4290 return (EDOOFUS); 4291 } 4292 4293 int 4294 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 4295 { 4296 uintptr_t *loc, new; 4297 4298 if (opcode >= nitems(sc->cpl_handler)) 4299 return (EINVAL); 4300 4301 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 4302 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 4303 atomic_store_rel_ptr(loc, new); 4304 4305 return (0); 4306 } 4307 4308 static int 4309 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 4310 { 4311 4312 #ifdef INVARIANTS 4313 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 4314 #else 4315 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 4316 __func__, iq, ctrl); 4317 #endif 4318 return (EDOOFUS); 4319 } 4320 4321 int 4322 t4_register_an_handler(struct adapter *sc, an_handler_t h) 4323 { 4324 uintptr_t *loc, new; 4325 4326 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 4327 loc = (uintptr_t *) &sc->an_handler; 4328 atomic_store_rel_ptr(loc, new); 4329 4330 return (0); 4331 } 4332 4333 static int 4334 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 4335 { 4336 const struct cpl_fw6_msg *cpl = 4337 __containerof(rpl, struct cpl_fw6_msg, data[0]); 4338 4339 #ifdef INVARIANTS 4340 panic("%s: fw_msg type %d", __func__, cpl->type); 4341 #else 4342 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 4343 #endif 4344 return (EDOOFUS); 4345 } 4346 4347 int 4348 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 4349 { 4350 uintptr_t *loc, new; 4351 4352 if (type >= nitems(sc->fw_msg_handler)) 4353 return (EINVAL); 4354 4355 /* 4356 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 4357 * handler dispatch table. Reject any attempt to install a handler for 4358 * this subtype. 4359 */ 4360 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 4361 return (EINVAL); 4362 4363 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 4364 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 4365 atomic_store_rel_ptr(loc, new); 4366 4367 return (0); 4368 } 4369 4370 static int 4371 t4_sysctls(struct adapter *sc) 4372 { 4373 struct sysctl_ctx_list *ctx; 4374 struct sysctl_oid *oid; 4375 struct sysctl_oid_list *children, *c0; 4376 static char *caps[] = { 4377 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 4378 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL" /* caps[1] niccaps */ 4379 "\6HASHFILTER\7ETHOFLD", 4380 "\20\1TOE", /* caps[2] toecaps */ 4381 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 4382 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 4383 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 4384 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 4385 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 4386 "\4PO_INITIAOR\5PO_TARGET" 4387 }; 4388 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 4389 4390 ctx = device_get_sysctl_ctx(sc->dev); 4391 4392 /* 4393 * dev.t4nex.X. 4394 */ 4395 oid = device_get_sysctl_tree(sc->dev); 4396 c0 = children = SYSCTL_CHILDREN(oid); 4397 4398 sc->sc_do_rxcopy = 1; 4399 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, 4400 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); 4401 4402 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 4403 sc->params.nports, "# of ports"); 4404 4405 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 4406 NULL, chip_rev(sc), "chip hardware revision"); 4407 4408 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 4409 CTLFLAG_RD, &sc->fw_version, 0, "firmware version"); 4410 4411 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 4412 CTLFLAG_RD, &sc->cfg_file, 0, "configuration file"); 4413 4414 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 4415 sc->cfcsum, "config file checksum"); 4416 4417 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 4418 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 4419 sysctl_bitfield, "A", "available doorbells"); 4420 4421 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 4422 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 4423 sysctl_bitfield, "A", "available link capabilities"); 4424 4425 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 4426 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 4427 sysctl_bitfield, "A", "available NIC capabilities"); 4428 4429 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 4430 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 4431 sysctl_bitfield, "A", "available TCP offload capabilities"); 4432 4433 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 4434 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 4435 sysctl_bitfield, "A", "available RDMA capabilities"); 4436 4437 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 4438 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 4439 sysctl_bitfield, "A", "available iSCSI capabilities"); 4440 4441 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 4442 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 4443 sysctl_bitfield, "A", "available FCoE capabilities"); 4444 4445 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 4446 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 4447 4448 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 4449 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 4450 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 4451 "interrupt holdoff timer values (us)"); 4452 4453 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 4454 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 4455 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 4456 "interrupt holdoff packet counter values"); 4457 4458 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, 4459 NULL, sc->tids.nftids, "number of filters"); 4460 4461 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | 4462 CTLFLAG_RD, sc, 0, sysctl_temperature, "I", 4463 "chip temperature (in Celsius)"); 4464 4465 t4_sge_sysctls(sc, ctx, children); 4466 4467 sc->lro_timeout = 100; 4468 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, 4469 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); 4470 4471 #ifdef SBUF_DRAIN 4472 /* 4473 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 4474 */ 4475 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 4476 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 4477 "logs and miscellaneous information"); 4478 children = SYSCTL_CHILDREN(oid); 4479 4480 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 4481 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4482 sysctl_cctrl, "A", "congestion control"); 4483 4484 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 4485 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4486 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 4487 4488 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 4489 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 4490 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 4491 4492 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 4493 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 4494 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 4495 4496 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 4497 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 4498 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 4499 4500 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 4501 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 4502 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 4503 4504 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 4505 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 4506 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 4507 4508 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 4509 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4510 sysctl_cim_la, "A", "CIM logic analyzer"); 4511 4512 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", 4513 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4514 sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); 4515 4516 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 4517 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 4518 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 4519 4520 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 4521 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 4522 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 4523 4524 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 4525 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 4526 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 4527 4528 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 4529 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 4530 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 4531 4532 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 4533 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 4534 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 4535 4536 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 4537 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 4538 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 4539 4540 if (is_t5(sc)) { 4541 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 4542 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 4543 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 4544 4545 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 4546 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 4547 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 4548 } 4549 4550 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", 4551 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4552 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); 4553 4554 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 4555 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4556 sysctl_cim_qcfg, "A", "CIM queue configuration"); 4557 4558 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 4559 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4560 sysctl_cpl_stats, "A", "CPL statistics"); 4561 4562 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 4563 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4564 sysctl_ddp_stats, "A", "DDP statistics"); 4565 4566 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 4567 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4568 sysctl_devlog, "A", "firmware's device log"); 4569 4570 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 4571 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4572 sysctl_fcoe_stats, "A", "FCoE statistics"); 4573 4574 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 4575 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4576 sysctl_hw_sched, "A", "hardware scheduler "); 4577 4578 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 4579 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4580 sysctl_l2t, "A", "hardware L2 table"); 4581 4582 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 4583 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4584 sysctl_lb_stats, "A", "loopback statistics"); 4585 4586 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 4587 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4588 sysctl_meminfo, "A", "memory regions"); 4589 4590 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", 4591 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4592 sysctl_mps_tcam, "A", "MPS TCAM entries"); 4593 4594 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 4595 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4596 sysctl_path_mtus, "A", "path MTUs"); 4597 4598 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 4599 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4600 sysctl_pm_stats, "A", "PM statistics"); 4601 4602 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 4603 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4604 sysctl_rdma_stats, "A", "RDMA statistics"); 4605 4606 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 4607 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4608 sysctl_tcp_stats, "A", "TCP statistics"); 4609 4610 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 4611 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4612 sysctl_tids, "A", "TID information"); 4613 4614 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 4615 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4616 sysctl_tp_err_stats, "A", "TP error statistics"); 4617 4618 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", 4619 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4620 sysctl_tp_la, "A", "TP logic analyzer"); 4621 4622 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 4623 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4624 sysctl_tx_rate, "A", "Tx rate"); 4625 4626 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", 4627 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4628 sysctl_ulprx_la, "A", "ULPRX logic analyzer"); 4629 4630 if (is_t5(sc)) { 4631 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 4632 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4633 sysctl_wcwr_stats, "A", "write combined work requests"); 4634 } 4635 #endif 4636 4637 #ifdef TCP_OFFLOAD 4638 if (is_offload(sc)) { 4639 /* 4640 * dev.t4nex.X.toe. 4641 */ 4642 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 4643 NULL, "TOE parameters"); 4644 children = SYSCTL_CHILDREN(oid); 4645 4646 sc->tt.sndbuf = 256 * 1024; 4647 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 4648 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 4649 4650 sc->tt.ddp = 0; 4651 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 4652 &sc->tt.ddp, 0, "DDP allowed"); 4653 4654 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 4655 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 4656 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 4657 4658 sc->tt.ddp_thres = 4659 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 4660 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 4661 &sc->tt.ddp_thres, 0, "DDP threshold"); 4662 4663 sc->tt.rx_coalesce = 1; 4664 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", 4665 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); 4666 } 4667 #endif 4668 4669 4670 return (0); 4671 } 4672 4673 static int 4674 cxgbe_sysctls(struct port_info *pi) 4675 { 4676 struct sysctl_ctx_list *ctx; 4677 struct sysctl_oid *oid; 4678 struct sysctl_oid_list *children; 4679 struct adapter *sc = pi->adapter; 4680 4681 ctx = device_get_sysctl_ctx(pi->dev); 4682 4683 /* 4684 * dev.cxgbe.X. 4685 */ 4686 oid = device_get_sysctl_tree(pi->dev); 4687 children = SYSCTL_CHILDREN(oid); 4688 4689 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | 4690 CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); 4691 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { 4692 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 4693 CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I", 4694 "PHY temperature (in Celsius)"); 4695 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", 4696 CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I", 4697 "PHY firmware version"); 4698 } 4699 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 4700 &pi->nrxq, 0, "# of rx queues"); 4701 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 4702 &pi->ntxq, 0, "# of tx queues"); 4703 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 4704 &pi->first_rxq, 0, "index of first rx queue"); 4705 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 4706 &pi->first_txq, 0, "index of first tx queue"); 4707 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT | 4708 CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU", 4709 "Reserve queue 0 for non-flowid packets"); 4710 4711 #ifdef TCP_OFFLOAD 4712 if (is_offload(sc)) { 4713 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 4714 &pi->nofldrxq, 0, 4715 "# of rx queues for offloaded TCP connections"); 4716 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 4717 &pi->nofldtxq, 0, 4718 "# of tx queues for offloaded TCP connections"); 4719 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 4720 CTLFLAG_RD, &pi->first_ofld_rxq, 0, 4721 "index of first TOE rx queue"); 4722 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 4723 CTLFLAG_RD, &pi->first_ofld_txq, 0, 4724 "index of first TOE tx queue"); 4725 } 4726 #endif 4727 #ifdef DEV_NETMAP 4728 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD, 4729 &pi->nnmrxq, 0, "# of rx queues for netmap"); 4730 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD, 4731 &pi->nnmtxq, 0, "# of tx queues for netmap"); 4732 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq", 4733 CTLFLAG_RD, &pi->first_nm_rxq, 0, 4734 "index of first netmap rx queue"); 4735 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq", 4736 CTLFLAG_RD, &pi->first_nm_txq, 0, 4737 "index of first netmap tx queue"); 4738 #endif 4739 4740 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 4741 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 4742 "holdoff timer index"); 4743 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 4744 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 4745 "holdoff packet counter index"); 4746 4747 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 4748 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 4749 "rx queue size"); 4750 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 4751 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 4752 "tx queue size"); 4753 4754 /* 4755 * dev.cxgbe.X.stats. 4756 */ 4757 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 4758 NULL, "port statistics"); 4759 children = SYSCTL_CHILDREN(oid); 4760 4761 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 4762 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 4763 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ 4764 sysctl_handle_t4_reg64, "QU", desc) 4765 4766 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 4767 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 4768 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 4769 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 4770 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 4771 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 4772 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 4773 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 4774 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 4775 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 4776 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 4777 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 4778 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 4779 "# of tx frames in this range", 4780 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 4781 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 4782 "# of tx frames in this range", 4783 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 4784 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 4785 "# of tx frames in this range", 4786 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 4787 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 4788 "# of tx frames in this range", 4789 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 4790 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 4791 "# of tx frames in this range", 4792 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 4793 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 4794 "# of tx frames in this range", 4795 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 4796 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 4797 "# of tx frames in this range", 4798 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 4799 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 4800 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 4801 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 4802 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 4803 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 4804 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 4805 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 4806 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 4807 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 4808 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 4809 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 4810 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 4811 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 4812 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 4813 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 4814 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 4815 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 4816 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 4817 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 4818 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 4819 4820 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 4821 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 4822 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 4823 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 4824 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 4825 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 4826 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 4827 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 4828 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 4829 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 4830 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 4831 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 4832 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 4833 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 4834 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 4835 "# of frames received with bad FCS", 4836 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 4837 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 4838 "# of frames received with length error", 4839 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 4840 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 4841 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 4842 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 4843 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 4844 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 4845 "# of rx frames in this range", 4846 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 4847 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 4848 "# of rx frames in this range", 4849 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 4850 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 4851 "# of rx frames in this range", 4852 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 4853 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 4854 "# of rx frames in this range", 4855 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 4856 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 4857 "# of rx frames in this range", 4858 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 4859 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 4860 "# of rx frames in this range", 4861 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 4862 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 4863 "# of rx frames in this range", 4864 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 4865 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 4866 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 4867 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 4868 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 4869 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 4870 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 4871 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 4872 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 4873 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 4874 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 4875 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 4876 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 4877 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 4878 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 4879 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 4880 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 4881 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 4882 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 4883 4884 #undef SYSCTL_ADD_T4_REG64 4885 4886 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 4887 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 4888 &pi->stats.name, desc) 4889 4890 /* We get these from port_stats and they may be stale by upto 1s */ 4891 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 4892 "# drops due to buffer-group 0 overflows"); 4893 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 4894 "# drops due to buffer-group 1 overflows"); 4895 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 4896 "# drops due to buffer-group 2 overflows"); 4897 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 4898 "# drops due to buffer-group 3 overflows"); 4899 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 4900 "# of buffer-group 0 truncated packets"); 4901 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 4902 "# of buffer-group 1 truncated packets"); 4903 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 4904 "# of buffer-group 2 truncated packets"); 4905 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 4906 "# of buffer-group 3 truncated packets"); 4907 4908 #undef SYSCTL_ADD_T4_PORTSTAT 4909 4910 return (0); 4911 } 4912 4913 static int 4914 sysctl_int_array(SYSCTL_HANDLER_ARGS) 4915 { 4916 int rc, *i; 4917 struct sbuf sb; 4918 4919 sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND); 4920 for (i = arg1; arg2; arg2 -= sizeof(int), i++) 4921 sbuf_printf(&sb, "%d ", *i); 4922 sbuf_trim(&sb); 4923 sbuf_finish(&sb); 4924 rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 4925 sbuf_delete(&sb); 4926 return (rc); 4927 } 4928 4929 static int 4930 sysctl_bitfield(SYSCTL_HANDLER_ARGS) 4931 { 4932 int rc; 4933 struct sbuf *sb; 4934 4935 rc = sysctl_wire_old_buffer(req, 0); 4936 if (rc != 0) 4937 return(rc); 4938 4939 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 4940 if (sb == NULL) 4941 return (ENOMEM); 4942 4943 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 4944 rc = sbuf_finish(sb); 4945 sbuf_delete(sb); 4946 4947 return (rc); 4948 } 4949 4950 static int 4951 sysctl_btphy(SYSCTL_HANDLER_ARGS) 4952 { 4953 struct port_info *pi = arg1; 4954 int op = arg2; 4955 struct adapter *sc = pi->adapter; 4956 u_int v; 4957 int rc; 4958 4959 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt"); 4960 if (rc) 4961 return (rc); 4962 /* XXX: magic numbers */ 4963 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, 4964 &v); 4965 end_synchronized_op(sc, 0); 4966 if (rc) 4967 return (rc); 4968 if (op == 0) 4969 v /= 256; 4970 4971 rc = sysctl_handle_int(oidp, &v, 0, req); 4972 return (rc); 4973 } 4974 4975 static int 4976 sysctl_noflowq(SYSCTL_HANDLER_ARGS) 4977 { 4978 struct port_info *pi = arg1; 4979 int rc, val; 4980 4981 val = pi->rsrv_noflowq; 4982 rc = sysctl_handle_int(oidp, &val, 0, req); 4983 if (rc != 0 || req->newptr == NULL) 4984 return (rc); 4985 4986 if ((val >= 1) && (pi->ntxq > 1)) 4987 pi->rsrv_noflowq = 1; 4988 else 4989 pi->rsrv_noflowq = 0; 4990 4991 return (rc); 4992 } 4993 4994 static int 4995 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 4996 { 4997 struct port_info *pi = arg1; 4998 struct adapter *sc = pi->adapter; 4999 int idx, rc, i; 5000 struct sge_rxq *rxq; 5001 #ifdef TCP_OFFLOAD 5002 struct sge_ofld_rxq *ofld_rxq; 5003 #endif 5004 uint8_t v; 5005 5006 idx = pi->tmr_idx; 5007 5008 rc = sysctl_handle_int(oidp, &idx, 0, req); 5009 if (rc != 0 || req->newptr == NULL) 5010 return (rc); 5011 5012 if (idx < 0 || idx >= SGE_NTIMERS) 5013 return (EINVAL); 5014 5015 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5016 "t4tmr"); 5017 if (rc) 5018 return (rc); 5019 5020 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 5021 for_each_rxq(pi, i, rxq) { 5022 #ifdef atomic_store_rel_8 5023 atomic_store_rel_8(&rxq->iq.intr_params, v); 5024 #else 5025 rxq->iq.intr_params = v; 5026 #endif 5027 } 5028 #ifdef TCP_OFFLOAD 5029 for_each_ofld_rxq(pi, i, ofld_rxq) { 5030 #ifdef atomic_store_rel_8 5031 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); 5032 #else 5033 ofld_rxq->iq.intr_params = v; 5034 #endif 5035 } 5036 #endif 5037 pi->tmr_idx = idx; 5038 5039 end_synchronized_op(sc, LOCK_HELD); 5040 return (0); 5041 } 5042 5043 static int 5044 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 5045 { 5046 struct port_info *pi = arg1; 5047 struct adapter *sc = pi->adapter; 5048 int idx, rc; 5049 5050 idx = pi->pktc_idx; 5051 5052 rc = sysctl_handle_int(oidp, &idx, 0, req); 5053 if (rc != 0 || req->newptr == NULL) 5054 return (rc); 5055 5056 if (idx < -1 || idx >= SGE_NCOUNTERS) 5057 return (EINVAL); 5058 5059 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5060 "t4pktc"); 5061 if (rc) 5062 return (rc); 5063 5064 if (pi->flags & PORT_INIT_DONE) 5065 rc = EBUSY; /* cannot be changed once the queues are created */ 5066 else 5067 pi->pktc_idx = idx; 5068 5069 end_synchronized_op(sc, LOCK_HELD); 5070 return (rc); 5071 } 5072 5073 static int 5074 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 5075 { 5076 struct port_info *pi = arg1; 5077 struct adapter *sc = pi->adapter; 5078 int qsize, rc; 5079 5080 qsize = pi->qsize_rxq; 5081 5082 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5083 if (rc != 0 || req->newptr == NULL) 5084 return (rc); 5085 5086 if (qsize < 128 || (qsize & 7)) 5087 return (EINVAL); 5088 5089 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5090 "t4rxqs"); 5091 if (rc) 5092 return (rc); 5093 5094 if (pi->flags & PORT_INIT_DONE) 5095 rc = EBUSY; /* cannot be changed once the queues are created */ 5096 else 5097 pi->qsize_rxq = qsize; 5098 5099 end_synchronized_op(sc, LOCK_HELD); 5100 return (rc); 5101 } 5102 5103 static int 5104 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 5105 { 5106 struct port_info *pi = arg1; 5107 struct adapter *sc = pi->adapter; 5108 int qsize, rc; 5109 5110 qsize = pi->qsize_txq; 5111 5112 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5113 if (rc != 0 || req->newptr == NULL) 5114 return (rc); 5115 5116 /* bufring size must be powerof2 */ 5117 if (qsize < 128 || !powerof2(qsize)) 5118 return (EINVAL); 5119 5120 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5121 "t4txqs"); 5122 if (rc) 5123 return (rc); 5124 5125 if (pi->flags & PORT_INIT_DONE) 5126 rc = EBUSY; /* cannot be changed once the queues are created */ 5127 else 5128 pi->qsize_txq = qsize; 5129 5130 end_synchronized_op(sc, LOCK_HELD); 5131 return (rc); 5132 } 5133 5134 static int 5135 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 5136 { 5137 struct adapter *sc = arg1; 5138 int reg = arg2; 5139 uint64_t val; 5140 5141 val = t4_read_reg64(sc, reg); 5142 5143 return (sysctl_handle_64(oidp, &val, 0, req)); 5144 } 5145 5146 static int 5147 sysctl_temperature(SYSCTL_HANDLER_ARGS) 5148 { 5149 struct adapter *sc = arg1; 5150 int rc, t; 5151 uint32_t param, val; 5152 5153 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); 5154 if (rc) 5155 return (rc); 5156 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 5157 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 5158 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); 5159 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 5160 end_synchronized_op(sc, 0); 5161 if (rc) 5162 return (rc); 5163 5164 /* unknown is returned as 0 but we display -1 in that case */ 5165 t = val == 0 ? -1 : val; 5166 5167 rc = sysctl_handle_int(oidp, &t, 0, req); 5168 return (rc); 5169 } 5170 5171 #ifdef SBUF_DRAIN 5172 static int 5173 sysctl_cctrl(SYSCTL_HANDLER_ARGS) 5174 { 5175 struct adapter *sc = arg1; 5176 struct sbuf *sb; 5177 int rc, i; 5178 uint16_t incr[NMTUS][NCCTRL_WIN]; 5179 static const char *dec_fac[] = { 5180 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 5181 "0.9375" 5182 }; 5183 5184 rc = sysctl_wire_old_buffer(req, 0); 5185 if (rc != 0) 5186 return (rc); 5187 5188 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5189 if (sb == NULL) 5190 return (ENOMEM); 5191 5192 t4_read_cong_tbl(sc, incr); 5193 5194 for (i = 0; i < NCCTRL_WIN; ++i) { 5195 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 5196 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 5197 incr[5][i], incr[6][i], incr[7][i]); 5198 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 5199 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 5200 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 5201 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 5202 } 5203 5204 rc = sbuf_finish(sb); 5205 sbuf_delete(sb); 5206 5207 return (rc); 5208 } 5209 5210 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 5211 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 5212 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 5213 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 5214 }; 5215 5216 static int 5217 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 5218 { 5219 struct adapter *sc = arg1; 5220 struct sbuf *sb; 5221 int rc, i, n, qid = arg2; 5222 uint32_t *buf, *p; 5223 char *qtype; 5224 u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5; 5225 5226 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 5227 ("%s: bad qid %d\n", __func__, qid)); 5228 5229 if (qid < CIM_NUM_IBQ) { 5230 /* inbound queue */ 5231 qtype = "IBQ"; 5232 n = 4 * CIM_IBQ_SIZE; 5233 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5234 rc = t4_read_cim_ibq(sc, qid, buf, n); 5235 } else { 5236 /* outbound queue */ 5237 qtype = "OBQ"; 5238 qid -= CIM_NUM_IBQ; 5239 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 5240 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5241 rc = t4_read_cim_obq(sc, qid, buf, n); 5242 } 5243 5244 if (rc < 0) { 5245 rc = -rc; 5246 goto done; 5247 } 5248 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 5249 5250 rc = sysctl_wire_old_buffer(req, 0); 5251 if (rc != 0) 5252 goto done; 5253 5254 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5255 if (sb == NULL) { 5256 rc = ENOMEM; 5257 goto done; 5258 } 5259 5260 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 5261 for (i = 0, p = buf; i < n; i += 16, p += 4) 5262 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 5263 p[2], p[3]); 5264 5265 rc = sbuf_finish(sb); 5266 sbuf_delete(sb); 5267 done: 5268 free(buf, M_CXGBE); 5269 return (rc); 5270 } 5271 5272 static int 5273 sysctl_cim_la(SYSCTL_HANDLER_ARGS) 5274 { 5275 struct adapter *sc = arg1; 5276 u_int cfg; 5277 struct sbuf *sb; 5278 uint32_t *buf, *p; 5279 int rc; 5280 5281 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 5282 if (rc != 0) 5283 return (rc); 5284 5285 rc = sysctl_wire_old_buffer(req, 0); 5286 if (rc != 0) 5287 return (rc); 5288 5289 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5290 if (sb == NULL) 5291 return (ENOMEM); 5292 5293 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 5294 M_ZERO | M_WAITOK); 5295 5296 rc = -t4_cim_read_la(sc, buf, NULL); 5297 if (rc != 0) 5298 goto done; 5299 5300 sbuf_printf(sb, "Status Data PC%s", 5301 cfg & F_UPDBGLACAPTPCONLY ? "" : 5302 " LS0Stat LS0Addr LS0Data"); 5303 5304 KASSERT((sc->params.cim_la_size & 7) == 0, 5305 ("%s: p will walk off the end of buf", __func__)); 5306 5307 for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) { 5308 if (cfg & F_UPDBGLACAPTPCONLY) { 5309 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 5310 p[6], p[7]); 5311 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 5312 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 5313 p[4] & 0xff, p[5] >> 8); 5314 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 5315 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5316 p[1] & 0xf, p[2] >> 4); 5317 } else { 5318 sbuf_printf(sb, 5319 "\n %02x %x%07x %x%07x %08x %08x " 5320 "%08x%08x%08x%08x", 5321 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5322 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 5323 p[6], p[7]); 5324 } 5325 } 5326 5327 rc = sbuf_finish(sb); 5328 sbuf_delete(sb); 5329 done: 5330 free(buf, M_CXGBE); 5331 return (rc); 5332 } 5333 5334 static int 5335 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) 5336 { 5337 struct adapter *sc = arg1; 5338 u_int i; 5339 struct sbuf *sb; 5340 uint32_t *buf, *p; 5341 int rc; 5342 5343 rc = sysctl_wire_old_buffer(req, 0); 5344 if (rc != 0) 5345 return (rc); 5346 5347 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5348 if (sb == NULL) 5349 return (ENOMEM); 5350 5351 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, 5352 M_ZERO | M_WAITOK); 5353 5354 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); 5355 p = buf; 5356 5357 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5358 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], 5359 p[1], p[0]); 5360 } 5361 5362 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); 5363 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5364 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", 5365 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, 5366 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, 5367 (p[1] >> 2) | ((p[2] & 3) << 30), 5368 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, 5369 p[0] & 1); 5370 } 5371 5372 rc = sbuf_finish(sb); 5373 sbuf_delete(sb); 5374 free(buf, M_CXGBE); 5375 return (rc); 5376 } 5377 5378 static int 5379 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) 5380 { 5381 struct adapter *sc = arg1; 5382 u_int i; 5383 struct sbuf *sb; 5384 uint32_t *buf, *p; 5385 int rc; 5386 5387 rc = sysctl_wire_old_buffer(req, 0); 5388 if (rc != 0) 5389 return (rc); 5390 5391 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5392 if (sb == NULL) 5393 return (ENOMEM); 5394 5395 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, 5396 M_ZERO | M_WAITOK); 5397 5398 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); 5399 p = buf; 5400 5401 sbuf_printf(sb, "Cntl ID DataBE Addr Data"); 5402 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5403 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", 5404 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, 5405 p[4], p[3], p[2], p[1], p[0]); 5406 } 5407 5408 sbuf_printf(sb, "\n\nCntl ID Data"); 5409 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5410 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", 5411 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); 5412 } 5413 5414 rc = sbuf_finish(sb); 5415 sbuf_delete(sb); 5416 free(buf, M_CXGBE); 5417 return (rc); 5418 } 5419 5420 static int 5421 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 5422 { 5423 struct adapter *sc = arg1; 5424 struct sbuf *sb; 5425 int rc, i; 5426 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5427 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5428 uint16_t thres[CIM_NUM_IBQ]; 5429 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 5430 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 5431 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 5432 5433 if (is_t4(sc)) { 5434 cim_num_obq = CIM_NUM_OBQ; 5435 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 5436 obq_rdaddr = A_UP_OBQ_0_REALADDR; 5437 } else { 5438 cim_num_obq = CIM_NUM_OBQ_T5; 5439 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 5440 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 5441 } 5442 nq = CIM_NUM_IBQ + cim_num_obq; 5443 5444 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 5445 if (rc == 0) 5446 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 5447 if (rc != 0) 5448 return (rc); 5449 5450 t4_read_cimq_cfg(sc, base, size, thres); 5451 5452 rc = sysctl_wire_old_buffer(req, 0); 5453 if (rc != 0) 5454 return (rc); 5455 5456 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5457 if (sb == NULL) 5458 return (ENOMEM); 5459 5460 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 5461 5462 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 5463 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 5464 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 5465 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5466 G_QUEREMFLITS(p[2]) * 16); 5467 for ( ; i < nq; i++, p += 4, wr += 2) 5468 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 5469 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 5470 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5471 G_QUEREMFLITS(p[2]) * 16); 5472 5473 rc = sbuf_finish(sb); 5474 sbuf_delete(sb); 5475 5476 return (rc); 5477 } 5478 5479 static int 5480 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 5481 { 5482 struct adapter *sc = arg1; 5483 struct sbuf *sb; 5484 int rc; 5485 struct tp_cpl_stats stats; 5486 5487 rc = sysctl_wire_old_buffer(req, 0); 5488 if (rc != 0) 5489 return (rc); 5490 5491 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5492 if (sb == NULL) 5493 return (ENOMEM); 5494 5495 t4_tp_get_cpl_stats(sc, &stats); 5496 5497 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 5498 "channel 3\n"); 5499 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 5500 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 5501 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 5502 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 5503 5504 rc = sbuf_finish(sb); 5505 sbuf_delete(sb); 5506 5507 return (rc); 5508 } 5509 5510 static int 5511 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 5512 { 5513 struct adapter *sc = arg1; 5514 struct sbuf *sb; 5515 int rc; 5516 struct tp_usm_stats stats; 5517 5518 rc = sysctl_wire_old_buffer(req, 0); 5519 if (rc != 0) 5520 return(rc); 5521 5522 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5523 if (sb == NULL) 5524 return (ENOMEM); 5525 5526 t4_get_usm_stats(sc, &stats); 5527 5528 sbuf_printf(sb, "Frames: %u\n", stats.frames); 5529 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 5530 sbuf_printf(sb, "Drops: %u", stats.drops); 5531 5532 rc = sbuf_finish(sb); 5533 sbuf_delete(sb); 5534 5535 return (rc); 5536 } 5537 5538 const char *devlog_level_strings[] = { 5539 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 5540 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 5541 [FW_DEVLOG_LEVEL_ERR] = "ERR", 5542 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 5543 [FW_DEVLOG_LEVEL_INFO] = "INFO", 5544 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 5545 }; 5546 5547 const char *devlog_facility_strings[] = { 5548 [FW_DEVLOG_FACILITY_CORE] = "CORE", 5549 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 5550 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 5551 [FW_DEVLOG_FACILITY_RES] = "RES", 5552 [FW_DEVLOG_FACILITY_HW] = "HW", 5553 [FW_DEVLOG_FACILITY_FLR] = "FLR", 5554 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 5555 [FW_DEVLOG_FACILITY_PHY] = "PHY", 5556 [FW_DEVLOG_FACILITY_MAC] = "MAC", 5557 [FW_DEVLOG_FACILITY_PORT] = "PORT", 5558 [FW_DEVLOG_FACILITY_VI] = "VI", 5559 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 5560 [FW_DEVLOG_FACILITY_ACL] = "ACL", 5561 [FW_DEVLOG_FACILITY_TM] = "TM", 5562 [FW_DEVLOG_FACILITY_QFC] = "QFC", 5563 [FW_DEVLOG_FACILITY_DCB] = "DCB", 5564 [FW_DEVLOG_FACILITY_ETH] = "ETH", 5565 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 5566 [FW_DEVLOG_FACILITY_RI] = "RI", 5567 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 5568 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 5569 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 5570 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 5571 }; 5572 5573 static int 5574 sysctl_devlog(SYSCTL_HANDLER_ARGS) 5575 { 5576 struct adapter *sc = arg1; 5577 struct devlog_params *dparams = &sc->params.devlog; 5578 struct fw_devlog_e *buf, *e; 5579 int i, j, rc, nentries, first = 0, m; 5580 struct sbuf *sb; 5581 uint64_t ftstamp = UINT64_MAX; 5582 5583 if (dparams->start == 0) { 5584 dparams->memtype = FW_MEMTYPE_EDC0; 5585 dparams->start = 0x84000; 5586 dparams->size = 32768; 5587 } 5588 5589 nentries = dparams->size / sizeof(struct fw_devlog_e); 5590 5591 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 5592 if (buf == NULL) 5593 return (ENOMEM); 5594 5595 m = fwmtype_to_hwmtype(dparams->memtype); 5596 rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf); 5597 if (rc != 0) 5598 goto done; 5599 5600 for (i = 0; i < nentries; i++) { 5601 e = &buf[i]; 5602 5603 if (e->timestamp == 0) 5604 break; /* end */ 5605 5606 e->timestamp = be64toh(e->timestamp); 5607 e->seqno = be32toh(e->seqno); 5608 for (j = 0; j < 8; j++) 5609 e->params[j] = be32toh(e->params[j]); 5610 5611 if (e->timestamp < ftstamp) { 5612 ftstamp = e->timestamp; 5613 first = i; 5614 } 5615 } 5616 5617 if (buf[first].timestamp == 0) 5618 goto done; /* nothing in the log */ 5619 5620 rc = sysctl_wire_old_buffer(req, 0); 5621 if (rc != 0) 5622 goto done; 5623 5624 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5625 if (sb == NULL) { 5626 rc = ENOMEM; 5627 goto done; 5628 } 5629 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 5630 "Seq#", "Tstamp", "Level", "Facility", "Message"); 5631 5632 i = first; 5633 do { 5634 e = &buf[i]; 5635 if (e->timestamp == 0) 5636 break; /* end */ 5637 5638 sbuf_printf(sb, "%10d %15ju %8s %8s ", 5639 e->seqno, e->timestamp, 5640 (e->level < nitems(devlog_level_strings) ? 5641 devlog_level_strings[e->level] : "UNKNOWN"), 5642 (e->facility < nitems(devlog_facility_strings) ? 5643 devlog_facility_strings[e->facility] : "UNKNOWN")); 5644 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 5645 e->params[2], e->params[3], e->params[4], 5646 e->params[5], e->params[6], e->params[7]); 5647 5648 if (++i == nentries) 5649 i = 0; 5650 } while (i != first); 5651 5652 rc = sbuf_finish(sb); 5653 sbuf_delete(sb); 5654 done: 5655 free(buf, M_CXGBE); 5656 return (rc); 5657 } 5658 5659 static int 5660 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 5661 { 5662 struct adapter *sc = arg1; 5663 struct sbuf *sb; 5664 int rc; 5665 struct tp_fcoe_stats stats[4]; 5666 5667 rc = sysctl_wire_old_buffer(req, 0); 5668 if (rc != 0) 5669 return (rc); 5670 5671 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5672 if (sb == NULL) 5673 return (ENOMEM); 5674 5675 t4_get_fcoe_stats(sc, 0, &stats[0]); 5676 t4_get_fcoe_stats(sc, 1, &stats[1]); 5677 t4_get_fcoe_stats(sc, 2, &stats[2]); 5678 t4_get_fcoe_stats(sc, 3, &stats[3]); 5679 5680 sbuf_printf(sb, " channel 0 channel 1 " 5681 "channel 2 channel 3\n"); 5682 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 5683 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 5684 stats[3].octetsDDP); 5685 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 5686 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 5687 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 5688 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 5689 stats[3].framesDrop); 5690 5691 rc = sbuf_finish(sb); 5692 sbuf_delete(sb); 5693 5694 return (rc); 5695 } 5696 5697 static int 5698 sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 5699 { 5700 struct adapter *sc = arg1; 5701 struct sbuf *sb; 5702 int rc, i; 5703 unsigned int map, kbps, ipg, mode; 5704 unsigned int pace_tab[NTX_SCHED]; 5705 5706 rc = sysctl_wire_old_buffer(req, 0); 5707 if (rc != 0) 5708 return (rc); 5709 5710 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5711 if (sb == NULL) 5712 return (ENOMEM); 5713 5714 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 5715 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 5716 t4_read_pace_tbl(sc, pace_tab); 5717 5718 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 5719 "Class IPG (0.1 ns) Flow IPG (us)"); 5720 5721 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 5722 t4_get_tx_sched(sc, i, &kbps, &ipg); 5723 sbuf_printf(sb, "\n %u %-5s %u ", i, 5724 (mode & (1 << i)) ? "flow" : "class", map & 3); 5725 if (kbps) 5726 sbuf_printf(sb, "%9u ", kbps); 5727 else 5728 sbuf_printf(sb, " disabled "); 5729 5730 if (ipg) 5731 sbuf_printf(sb, "%13u ", ipg); 5732 else 5733 sbuf_printf(sb, " disabled "); 5734 5735 if (pace_tab[i]) 5736 sbuf_printf(sb, "%10u", pace_tab[i]); 5737 else 5738 sbuf_printf(sb, " disabled"); 5739 } 5740 5741 rc = sbuf_finish(sb); 5742 sbuf_delete(sb); 5743 5744 return (rc); 5745 } 5746 5747 static int 5748 sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 5749 { 5750 struct adapter *sc = arg1; 5751 struct sbuf *sb; 5752 int rc, i, j; 5753 uint64_t *p0, *p1; 5754 struct lb_port_stats s[2]; 5755 static const char *stat_name[] = { 5756 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 5757 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 5758 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 5759 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 5760 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 5761 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 5762 "BG2FramesTrunc:", "BG3FramesTrunc:" 5763 }; 5764 5765 rc = sysctl_wire_old_buffer(req, 0); 5766 if (rc != 0) 5767 return (rc); 5768 5769 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5770 if (sb == NULL) 5771 return (ENOMEM); 5772 5773 memset(s, 0, sizeof(s)); 5774 5775 for (i = 0; i < 4; i += 2) { 5776 t4_get_lb_stats(sc, i, &s[0]); 5777 t4_get_lb_stats(sc, i + 1, &s[1]); 5778 5779 p0 = &s[0].octets; 5780 p1 = &s[1].octets; 5781 sbuf_printf(sb, "%s Loopback %u" 5782 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 5783 5784 for (j = 0; j < nitems(stat_name); j++) 5785 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 5786 *p0++, *p1++); 5787 } 5788 5789 rc = sbuf_finish(sb); 5790 sbuf_delete(sb); 5791 5792 return (rc); 5793 } 5794 5795 static int 5796 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) 5797 { 5798 int rc = 0; 5799 struct port_info *pi = arg1; 5800 struct sbuf *sb; 5801 static const char *linkdnreasons[] = { 5802 "non-specific", "remote fault", "autoneg failed", "reserved3", 5803 "PHY overheated", "unknown", "rx los", "reserved7" 5804 }; 5805 5806 rc = sysctl_wire_old_buffer(req, 0); 5807 if (rc != 0) 5808 return(rc); 5809 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); 5810 if (sb == NULL) 5811 return (ENOMEM); 5812 5813 if (pi->linkdnrc < 0) 5814 sbuf_printf(sb, "n/a"); 5815 else if (pi->linkdnrc < nitems(linkdnreasons)) 5816 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]); 5817 else 5818 sbuf_printf(sb, "%d", pi->linkdnrc); 5819 5820 rc = sbuf_finish(sb); 5821 sbuf_delete(sb); 5822 5823 return (rc); 5824 } 5825 5826 struct mem_desc { 5827 unsigned int base; 5828 unsigned int limit; 5829 unsigned int idx; 5830 }; 5831 5832 static int 5833 mem_desc_cmp(const void *a, const void *b) 5834 { 5835 return ((const struct mem_desc *)a)->base - 5836 ((const struct mem_desc *)b)->base; 5837 } 5838 5839 static void 5840 mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 5841 unsigned int to) 5842 { 5843 unsigned int size; 5844 5845 size = to - from + 1; 5846 if (size == 0) 5847 return; 5848 5849 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 5850 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 5851 } 5852 5853 static int 5854 sysctl_meminfo(SYSCTL_HANDLER_ARGS) 5855 { 5856 struct adapter *sc = arg1; 5857 struct sbuf *sb; 5858 int rc, i, n; 5859 uint32_t lo, hi, used, alloc; 5860 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 5861 static const char *region[] = { 5862 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 5863 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 5864 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 5865 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 5866 "RQUDP region:", "PBL region:", "TXPBL region:", 5867 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 5868 "On-chip queues:" 5869 }; 5870 struct mem_desc avail[4]; 5871 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 5872 struct mem_desc *md = mem; 5873 5874 rc = sysctl_wire_old_buffer(req, 0); 5875 if (rc != 0) 5876 return (rc); 5877 5878 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5879 if (sb == NULL) 5880 return (ENOMEM); 5881 5882 for (i = 0; i < nitems(mem); i++) { 5883 mem[i].limit = 0; 5884 mem[i].idx = i; 5885 } 5886 5887 /* Find and sort the populated memory ranges */ 5888 i = 0; 5889 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 5890 if (lo & F_EDRAM0_ENABLE) { 5891 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 5892 avail[i].base = G_EDRAM0_BASE(hi) << 20; 5893 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 5894 avail[i].idx = 0; 5895 i++; 5896 } 5897 if (lo & F_EDRAM1_ENABLE) { 5898 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 5899 avail[i].base = G_EDRAM1_BASE(hi) << 20; 5900 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 5901 avail[i].idx = 1; 5902 i++; 5903 } 5904 if (lo & F_EXT_MEM_ENABLE) { 5905 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 5906 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 5907 avail[i].limit = avail[i].base + 5908 (G_EXT_MEM_SIZE(hi) << 20); 5909 avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */ 5910 i++; 5911 } 5912 if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) { 5913 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 5914 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 5915 avail[i].limit = avail[i].base + 5916 (G_EXT_MEM1_SIZE(hi) << 20); 5917 avail[i].idx = 4; 5918 i++; 5919 } 5920 if (!i) /* no memory available */ 5921 return 0; 5922 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 5923 5924 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 5925 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 5926 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 5927 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 5928 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 5929 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 5930 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 5931 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 5932 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 5933 5934 /* the next few have explicit upper bounds */ 5935 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 5936 md->limit = md->base - 1 + 5937 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 5938 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 5939 md++; 5940 5941 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 5942 md->limit = md->base - 1 + 5943 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 5944 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 5945 md++; 5946 5947 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 5948 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 5949 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 5950 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 5951 } else { 5952 md->base = 0; 5953 md->idx = nitems(region); /* hide it */ 5954 } 5955 md++; 5956 5957 #define ulp_region(reg) \ 5958 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 5959 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 5960 5961 ulp_region(RX_ISCSI); 5962 ulp_region(RX_TDDP); 5963 ulp_region(TX_TPT); 5964 ulp_region(RX_STAG); 5965 ulp_region(RX_RQ); 5966 ulp_region(RX_RQUDP); 5967 ulp_region(RX_PBL); 5968 ulp_region(TX_PBL); 5969 #undef ulp_region 5970 5971 md->base = 0; 5972 md->idx = nitems(region); 5973 if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) { 5974 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); 5975 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc, 5976 A_SGE_DBVFIFO_SIZE))) << 2) - 1; 5977 } 5978 md++; 5979 5980 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 5981 md->limit = md->base + sc->tids.ntids - 1; 5982 md++; 5983 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 5984 md->limit = md->base + sc->tids.ntids - 1; 5985 md++; 5986 5987 md->base = sc->vres.ocq.start; 5988 if (sc->vres.ocq.size) 5989 md->limit = md->base + sc->vres.ocq.size - 1; 5990 else 5991 md->idx = nitems(region); /* hide it */ 5992 md++; 5993 5994 /* add any address-space holes, there can be up to 3 */ 5995 for (n = 0; n < i - 1; n++) 5996 if (avail[n].limit < avail[n + 1].base) 5997 (md++)->base = avail[n].limit; 5998 if (avail[n].limit) 5999 (md++)->base = avail[n].limit; 6000 6001 n = md - mem; 6002 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 6003 6004 for (lo = 0; lo < i; lo++) 6005 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 6006 avail[lo].limit - 1); 6007 6008 sbuf_printf(sb, "\n"); 6009 for (i = 0; i < n; i++) { 6010 if (mem[i].idx >= nitems(region)) 6011 continue; /* skip holes */ 6012 if (!mem[i].limit) 6013 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 6014 mem_region_show(sb, region[mem[i].idx], mem[i].base, 6015 mem[i].limit); 6016 } 6017 6018 sbuf_printf(sb, "\n"); 6019 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 6020 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 6021 mem_region_show(sb, "uP RAM:", lo, hi); 6022 6023 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 6024 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 6025 mem_region_show(sb, "uP Extmem2:", lo, hi); 6026 6027 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 6028 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 6029 G_PMRXMAXPAGE(lo), 6030 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 6031 (lo & F_PMRXNUMCHN) ? 2 : 1); 6032 6033 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 6034 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 6035 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 6036 G_PMTXMAXPAGE(lo), 6037 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 6038 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 6039 sbuf_printf(sb, "%u p-structs\n", 6040 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 6041 6042 for (i = 0; i < 4; i++) { 6043 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 6044 if (is_t4(sc)) { 6045 used = G_USED(lo); 6046 alloc = G_ALLOC(lo); 6047 } else { 6048 used = G_T5_USED(lo); 6049 alloc = G_T5_ALLOC(lo); 6050 } 6051 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 6052 i, used, alloc); 6053 } 6054 for (i = 0; i < 4; i++) { 6055 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 6056 if (is_t4(sc)) { 6057 used = G_USED(lo); 6058 alloc = G_ALLOC(lo); 6059 } else { 6060 used = G_T5_USED(lo); 6061 alloc = G_T5_ALLOC(lo); 6062 } 6063 sbuf_printf(sb, 6064 "\nLoopback %d using %u pages out of %u allocated", 6065 i, used, alloc); 6066 } 6067 6068 rc = sbuf_finish(sb); 6069 sbuf_delete(sb); 6070 6071 return (rc); 6072 } 6073 6074 static inline void 6075 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) 6076 { 6077 *mask = x | y; 6078 y = htobe64(y); 6079 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); 6080 } 6081 6082 static int 6083 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) 6084 { 6085 struct adapter *sc = arg1; 6086 struct sbuf *sb; 6087 int rc, i, n; 6088 6089 rc = sysctl_wire_old_buffer(req, 0); 6090 if (rc != 0) 6091 return (rc); 6092 6093 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6094 if (sb == NULL) 6095 return (ENOMEM); 6096 6097 sbuf_printf(sb, 6098 "Idx Ethernet address Mask Vld Ports PF" 6099 " VF Replication P0 P1 P2 P3 ML"); 6100 n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES : 6101 NUM_MPS_T5_CLS_SRAM_L_INSTANCES; 6102 for (i = 0; i < n; i++) { 6103 uint64_t tcamx, tcamy, mask; 6104 uint32_t cls_lo, cls_hi; 6105 uint8_t addr[ETHER_ADDR_LEN]; 6106 6107 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); 6108 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); 6109 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 6110 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 6111 6112 if (tcamx & tcamy) 6113 continue; 6114 6115 tcamxy2valmask(tcamx, tcamy, addr, &mask); 6116 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" 6117 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], 6118 addr[3], addr[4], addr[5], (uintmax_t)mask, 6119 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', 6120 G_PORTMAP(cls_hi), G_PF(cls_lo), 6121 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); 6122 6123 if (cls_lo & F_REPLICATE) { 6124 struct fw_ldst_cmd ldst_cmd; 6125 6126 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 6127 ldst_cmd.op_to_addrspace = 6128 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 6129 F_FW_CMD_REQUEST | F_FW_CMD_READ | 6130 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 6131 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 6132 ldst_cmd.u.mps.fid_ctl = 6133 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 6134 V_FW_LDST_CMD_CTL(i)); 6135 6136 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 6137 "t4mps"); 6138 if (rc) 6139 break; 6140 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 6141 sizeof(ldst_cmd), &ldst_cmd); 6142 end_synchronized_op(sc, 0); 6143 6144 if (rc != 0) { 6145 sbuf_printf(sb, 6146 " ------------ error %3u ------------", rc); 6147 rc = 0; 6148 } else { 6149 sbuf_printf(sb, " %08x %08x %08x %08x", 6150 be32toh(ldst_cmd.u.mps.rplc127_96), 6151 be32toh(ldst_cmd.u.mps.rplc95_64), 6152 be32toh(ldst_cmd.u.mps.rplc63_32), 6153 be32toh(ldst_cmd.u.mps.rplc31_0)); 6154 } 6155 } else 6156 sbuf_printf(sb, "%36s", ""); 6157 6158 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), 6159 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), 6160 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); 6161 } 6162 6163 if (rc) 6164 (void) sbuf_finish(sb); 6165 else 6166 rc = sbuf_finish(sb); 6167 sbuf_delete(sb); 6168 6169 return (rc); 6170 } 6171 6172 static int 6173 sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 6174 { 6175 struct adapter *sc = arg1; 6176 struct sbuf *sb; 6177 int rc; 6178 uint16_t mtus[NMTUS]; 6179 6180 rc = sysctl_wire_old_buffer(req, 0); 6181 if (rc != 0) 6182 return (rc); 6183 6184 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6185 if (sb == NULL) 6186 return (ENOMEM); 6187 6188 t4_read_mtu_tbl(sc, mtus, NULL); 6189 6190 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 6191 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 6192 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 6193 mtus[14], mtus[15]); 6194 6195 rc = sbuf_finish(sb); 6196 sbuf_delete(sb); 6197 6198 return (rc); 6199 } 6200 6201 static int 6202 sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 6203 { 6204 struct adapter *sc = arg1; 6205 struct sbuf *sb; 6206 int rc, i; 6207 uint32_t cnt[PM_NSTATS]; 6208 uint64_t cyc[PM_NSTATS]; 6209 static const char *rx_stats[] = { 6210 "Read:", "Write bypass:", "Write mem:", "Flush:" 6211 }; 6212 static const char *tx_stats[] = { 6213 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:" 6214 }; 6215 6216 rc = sysctl_wire_old_buffer(req, 0); 6217 if (rc != 0) 6218 return (rc); 6219 6220 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6221 if (sb == NULL) 6222 return (ENOMEM); 6223 6224 t4_pmtx_get_stats(sc, cnt, cyc); 6225 sbuf_printf(sb, " Tx pcmds Tx bytes"); 6226 for (i = 0; i < ARRAY_SIZE(tx_stats); i++) 6227 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i], 6228 cyc[i]); 6229 6230 t4_pmrx_get_stats(sc, cnt, cyc); 6231 sbuf_printf(sb, "\n Rx pcmds Rx bytes"); 6232 for (i = 0; i < ARRAY_SIZE(rx_stats); i++) 6233 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i], 6234 cyc[i]); 6235 6236 rc = sbuf_finish(sb); 6237 sbuf_delete(sb); 6238 6239 return (rc); 6240 } 6241 6242 static int 6243 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 6244 { 6245 struct adapter *sc = arg1; 6246 struct sbuf *sb; 6247 int rc; 6248 struct tp_rdma_stats stats; 6249 6250 rc = sysctl_wire_old_buffer(req, 0); 6251 if (rc != 0) 6252 return (rc); 6253 6254 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6255 if (sb == NULL) 6256 return (ENOMEM); 6257 6258 t4_tp_get_rdma_stats(sc, &stats); 6259 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 6260 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 6261 6262 rc = sbuf_finish(sb); 6263 sbuf_delete(sb); 6264 6265 return (rc); 6266 } 6267 6268 static int 6269 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 6270 { 6271 struct adapter *sc = arg1; 6272 struct sbuf *sb; 6273 int rc; 6274 struct tp_tcp_stats v4, v6; 6275 6276 rc = sysctl_wire_old_buffer(req, 0); 6277 if (rc != 0) 6278 return (rc); 6279 6280 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6281 if (sb == NULL) 6282 return (ENOMEM); 6283 6284 t4_tp_get_tcp_stats(sc, &v4, &v6); 6285 sbuf_printf(sb, 6286 " IP IPv6\n"); 6287 sbuf_printf(sb, "OutRsts: %20u %20u\n", 6288 v4.tcpOutRsts, v6.tcpOutRsts); 6289 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 6290 v4.tcpInSegs, v6.tcpInSegs); 6291 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 6292 v4.tcpOutSegs, v6.tcpOutSegs); 6293 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 6294 v4.tcpRetransSegs, v6.tcpRetransSegs); 6295 6296 rc = sbuf_finish(sb); 6297 sbuf_delete(sb); 6298 6299 return (rc); 6300 } 6301 6302 static int 6303 sysctl_tids(SYSCTL_HANDLER_ARGS) 6304 { 6305 struct adapter *sc = arg1; 6306 struct sbuf *sb; 6307 int rc; 6308 struct tid_info *t = &sc->tids; 6309 6310 rc = sysctl_wire_old_buffer(req, 0); 6311 if (rc != 0) 6312 return (rc); 6313 6314 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6315 if (sb == NULL) 6316 return (ENOMEM); 6317 6318 if (t->natids) { 6319 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 6320 t->atids_in_use); 6321 } 6322 6323 if (t->ntids) { 6324 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6325 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 6326 6327 if (b) { 6328 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 6329 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6330 t->ntids - 1); 6331 } else { 6332 sbuf_printf(sb, "TID range: %u-%u", 6333 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6334 t->ntids - 1); 6335 } 6336 } else 6337 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 6338 sbuf_printf(sb, ", in use: %u\n", 6339 atomic_load_acq_int(&t->tids_in_use)); 6340 } 6341 6342 if (t->nstids) { 6343 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 6344 t->stid_base + t->nstids - 1, t->stids_in_use); 6345 } 6346 6347 if (t->nftids) { 6348 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 6349 t->ftid_base + t->nftids - 1); 6350 } 6351 6352 if (t->netids) { 6353 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base, 6354 t->etid_base + t->netids - 1); 6355 } 6356 6357 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 6358 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 6359 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 6360 6361 rc = sbuf_finish(sb); 6362 sbuf_delete(sb); 6363 6364 return (rc); 6365 } 6366 6367 static int 6368 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 6369 { 6370 struct adapter *sc = arg1; 6371 struct sbuf *sb; 6372 int rc; 6373 struct tp_err_stats stats; 6374 6375 rc = sysctl_wire_old_buffer(req, 0); 6376 if (rc != 0) 6377 return (rc); 6378 6379 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6380 if (sb == NULL) 6381 return (ENOMEM); 6382 6383 t4_tp_get_err_stats(sc, &stats); 6384 6385 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6386 "channel 3\n"); 6387 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 6388 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 6389 stats.macInErrs[3]); 6390 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 6391 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 6392 stats.hdrInErrs[3]); 6393 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 6394 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 6395 stats.tcpInErrs[3]); 6396 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 6397 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 6398 stats.tcp6InErrs[3]); 6399 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 6400 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 6401 stats.tnlCongDrops[3]); 6402 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 6403 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 6404 stats.tnlTxDrops[3]); 6405 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 6406 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 6407 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 6408 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 6409 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 6410 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 6411 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 6412 stats.ofldNoNeigh, stats.ofldCongDefer); 6413 6414 rc = sbuf_finish(sb); 6415 sbuf_delete(sb); 6416 6417 return (rc); 6418 } 6419 6420 struct field_desc { 6421 const char *name; 6422 u_int start; 6423 u_int width; 6424 }; 6425 6426 static void 6427 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) 6428 { 6429 char buf[32]; 6430 int line_size = 0; 6431 6432 while (f->name) { 6433 uint64_t mask = (1ULL << f->width) - 1; 6434 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, 6435 ((uintmax_t)v >> f->start) & mask); 6436 6437 if (line_size + len >= 79) { 6438 line_size = 8; 6439 sbuf_printf(sb, "\n "); 6440 } 6441 sbuf_printf(sb, "%s ", buf); 6442 line_size += len + 1; 6443 f++; 6444 } 6445 sbuf_printf(sb, "\n"); 6446 } 6447 6448 static struct field_desc tp_la0[] = { 6449 { "RcfOpCodeOut", 60, 4 }, 6450 { "State", 56, 4 }, 6451 { "WcfState", 52, 4 }, 6452 { "RcfOpcSrcOut", 50, 2 }, 6453 { "CRxError", 49, 1 }, 6454 { "ERxError", 48, 1 }, 6455 { "SanityFailed", 47, 1 }, 6456 { "SpuriousMsg", 46, 1 }, 6457 { "FlushInputMsg", 45, 1 }, 6458 { "FlushInputCpl", 44, 1 }, 6459 { "RssUpBit", 43, 1 }, 6460 { "RssFilterHit", 42, 1 }, 6461 { "Tid", 32, 10 }, 6462 { "InitTcb", 31, 1 }, 6463 { "LineNumber", 24, 7 }, 6464 { "Emsg", 23, 1 }, 6465 { "EdataOut", 22, 1 }, 6466 { "Cmsg", 21, 1 }, 6467 { "CdataOut", 20, 1 }, 6468 { "EreadPdu", 19, 1 }, 6469 { "CreadPdu", 18, 1 }, 6470 { "TunnelPkt", 17, 1 }, 6471 { "RcfPeerFin", 16, 1 }, 6472 { "RcfReasonOut", 12, 4 }, 6473 { "TxCchannel", 10, 2 }, 6474 { "RcfTxChannel", 8, 2 }, 6475 { "RxEchannel", 6, 2 }, 6476 { "RcfRxChannel", 5, 1 }, 6477 { "RcfDataOutSrdy", 4, 1 }, 6478 { "RxDvld", 3, 1 }, 6479 { "RxOoDvld", 2, 1 }, 6480 { "RxCongestion", 1, 1 }, 6481 { "TxCongestion", 0, 1 }, 6482 { NULL } 6483 }; 6484 6485 static struct field_desc tp_la1[] = { 6486 { "CplCmdIn", 56, 8 }, 6487 { "CplCmdOut", 48, 8 }, 6488 { "ESynOut", 47, 1 }, 6489 { "EAckOut", 46, 1 }, 6490 { "EFinOut", 45, 1 }, 6491 { "ERstOut", 44, 1 }, 6492 { "SynIn", 43, 1 }, 6493 { "AckIn", 42, 1 }, 6494 { "FinIn", 41, 1 }, 6495 { "RstIn", 40, 1 }, 6496 { "DataIn", 39, 1 }, 6497 { "DataInVld", 38, 1 }, 6498 { "PadIn", 37, 1 }, 6499 { "RxBufEmpty", 36, 1 }, 6500 { "RxDdp", 35, 1 }, 6501 { "RxFbCongestion", 34, 1 }, 6502 { "TxFbCongestion", 33, 1 }, 6503 { "TxPktSumSrdy", 32, 1 }, 6504 { "RcfUlpType", 28, 4 }, 6505 { "Eread", 27, 1 }, 6506 { "Ebypass", 26, 1 }, 6507 { "Esave", 25, 1 }, 6508 { "Static0", 24, 1 }, 6509 { "Cread", 23, 1 }, 6510 { "Cbypass", 22, 1 }, 6511 { "Csave", 21, 1 }, 6512 { "CPktOut", 20, 1 }, 6513 { "RxPagePoolFull", 18, 2 }, 6514 { "RxLpbkPkt", 17, 1 }, 6515 { "TxLpbkPkt", 16, 1 }, 6516 { "RxVfValid", 15, 1 }, 6517 { "SynLearned", 14, 1 }, 6518 { "SetDelEntry", 13, 1 }, 6519 { "SetInvEntry", 12, 1 }, 6520 { "CpcmdDvld", 11, 1 }, 6521 { "CpcmdSave", 10, 1 }, 6522 { "RxPstructsFull", 8, 2 }, 6523 { "EpcmdDvld", 7, 1 }, 6524 { "EpcmdFlush", 6, 1 }, 6525 { "EpcmdTrimPrefix", 5, 1 }, 6526 { "EpcmdTrimPostfix", 4, 1 }, 6527 { "ERssIp4Pkt", 3, 1 }, 6528 { "ERssIp6Pkt", 2, 1 }, 6529 { "ERssTcpUdpPkt", 1, 1 }, 6530 { "ERssFceFipPkt", 0, 1 }, 6531 { NULL } 6532 }; 6533 6534 static struct field_desc tp_la2[] = { 6535 { "CplCmdIn", 56, 8 }, 6536 { "MpsVfVld", 55, 1 }, 6537 { "MpsPf", 52, 3 }, 6538 { "MpsVf", 44, 8 }, 6539 { "SynIn", 43, 1 }, 6540 { "AckIn", 42, 1 }, 6541 { "FinIn", 41, 1 }, 6542 { "RstIn", 40, 1 }, 6543 { "DataIn", 39, 1 }, 6544 { "DataInVld", 38, 1 }, 6545 { "PadIn", 37, 1 }, 6546 { "RxBufEmpty", 36, 1 }, 6547 { "RxDdp", 35, 1 }, 6548 { "RxFbCongestion", 34, 1 }, 6549 { "TxFbCongestion", 33, 1 }, 6550 { "TxPktSumSrdy", 32, 1 }, 6551 { "RcfUlpType", 28, 4 }, 6552 { "Eread", 27, 1 }, 6553 { "Ebypass", 26, 1 }, 6554 { "Esave", 25, 1 }, 6555 { "Static0", 24, 1 }, 6556 { "Cread", 23, 1 }, 6557 { "Cbypass", 22, 1 }, 6558 { "Csave", 21, 1 }, 6559 { "CPktOut", 20, 1 }, 6560 { "RxPagePoolFull", 18, 2 }, 6561 { "RxLpbkPkt", 17, 1 }, 6562 { "TxLpbkPkt", 16, 1 }, 6563 { "RxVfValid", 15, 1 }, 6564 { "SynLearned", 14, 1 }, 6565 { "SetDelEntry", 13, 1 }, 6566 { "SetInvEntry", 12, 1 }, 6567 { "CpcmdDvld", 11, 1 }, 6568 { "CpcmdSave", 10, 1 }, 6569 { "RxPstructsFull", 8, 2 }, 6570 { "EpcmdDvld", 7, 1 }, 6571 { "EpcmdFlush", 6, 1 }, 6572 { "EpcmdTrimPrefix", 5, 1 }, 6573 { "EpcmdTrimPostfix", 4, 1 }, 6574 { "ERssIp4Pkt", 3, 1 }, 6575 { "ERssIp6Pkt", 2, 1 }, 6576 { "ERssTcpUdpPkt", 1, 1 }, 6577 { "ERssFceFipPkt", 0, 1 }, 6578 { NULL } 6579 }; 6580 6581 static void 6582 tp_la_show(struct sbuf *sb, uint64_t *p, int idx) 6583 { 6584 6585 field_desc_show(sb, *p, tp_la0); 6586 } 6587 6588 static void 6589 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) 6590 { 6591 6592 if (idx) 6593 sbuf_printf(sb, "\n"); 6594 field_desc_show(sb, p[0], tp_la0); 6595 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6596 field_desc_show(sb, p[1], tp_la0); 6597 } 6598 6599 static void 6600 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) 6601 { 6602 6603 if (idx) 6604 sbuf_printf(sb, "\n"); 6605 field_desc_show(sb, p[0], tp_la0); 6606 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6607 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); 6608 } 6609 6610 static int 6611 sysctl_tp_la(SYSCTL_HANDLER_ARGS) 6612 { 6613 struct adapter *sc = arg1; 6614 struct sbuf *sb; 6615 uint64_t *buf, *p; 6616 int rc; 6617 u_int i, inc; 6618 void (*show_func)(struct sbuf *, uint64_t *, int); 6619 6620 rc = sysctl_wire_old_buffer(req, 0); 6621 if (rc != 0) 6622 return (rc); 6623 6624 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6625 if (sb == NULL) 6626 return (ENOMEM); 6627 6628 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); 6629 6630 t4_tp_read_la(sc, buf, NULL); 6631 p = buf; 6632 6633 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { 6634 case 2: 6635 inc = 2; 6636 show_func = tp_la_show2; 6637 break; 6638 case 3: 6639 inc = 2; 6640 show_func = tp_la_show3; 6641 break; 6642 default: 6643 inc = 1; 6644 show_func = tp_la_show; 6645 } 6646 6647 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) 6648 (*show_func)(sb, p, i); 6649 6650 rc = sbuf_finish(sb); 6651 sbuf_delete(sb); 6652 free(buf, M_CXGBE); 6653 return (rc); 6654 } 6655 6656 static int 6657 sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 6658 { 6659 struct adapter *sc = arg1; 6660 struct sbuf *sb; 6661 int rc; 6662 u64 nrate[NCHAN], orate[NCHAN]; 6663 6664 rc = sysctl_wire_old_buffer(req, 0); 6665 if (rc != 0) 6666 return (rc); 6667 6668 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6669 if (sb == NULL) 6670 return (ENOMEM); 6671 6672 t4_get_chan_txrate(sc, nrate, orate); 6673 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6674 "channel 3\n"); 6675 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 6676 nrate[0], nrate[1], nrate[2], nrate[3]); 6677 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 6678 orate[0], orate[1], orate[2], orate[3]); 6679 6680 rc = sbuf_finish(sb); 6681 sbuf_delete(sb); 6682 6683 return (rc); 6684 } 6685 6686 static int 6687 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) 6688 { 6689 struct adapter *sc = arg1; 6690 struct sbuf *sb; 6691 uint32_t *buf, *p; 6692 int rc, i; 6693 6694 rc = sysctl_wire_old_buffer(req, 0); 6695 if (rc != 0) 6696 return (rc); 6697 6698 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6699 if (sb == NULL) 6700 return (ENOMEM); 6701 6702 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, 6703 M_ZERO | M_WAITOK); 6704 6705 t4_ulprx_read_la(sc, buf); 6706 p = buf; 6707 6708 sbuf_printf(sb, " Pcmd Type Message" 6709 " Data"); 6710 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { 6711 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", 6712 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); 6713 } 6714 6715 rc = sbuf_finish(sb); 6716 sbuf_delete(sb); 6717 free(buf, M_CXGBE); 6718 return (rc); 6719 } 6720 6721 static int 6722 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 6723 { 6724 struct adapter *sc = arg1; 6725 struct sbuf *sb; 6726 int rc, v; 6727 6728 rc = sysctl_wire_old_buffer(req, 0); 6729 if (rc != 0) 6730 return (rc); 6731 6732 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6733 if (sb == NULL) 6734 return (ENOMEM); 6735 6736 v = t4_read_reg(sc, A_SGE_STAT_CFG); 6737 if (G_STATSOURCE_T5(v) == 7) { 6738 if (G_STATMODE(v) == 0) { 6739 sbuf_printf(sb, "total %d, incomplete %d", 6740 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6741 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6742 } else if (G_STATMODE(v) == 1) { 6743 sbuf_printf(sb, "total %d, data overflow %d", 6744 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6745 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6746 } 6747 } 6748 rc = sbuf_finish(sb); 6749 sbuf_delete(sb); 6750 6751 return (rc); 6752 } 6753 #endif 6754 6755 static inline void 6756 txq_start(struct ifnet *ifp, struct sge_txq *txq) 6757 { 6758 struct buf_ring *br; 6759 struct mbuf *m; 6760 6761 TXQ_LOCK_ASSERT_OWNED(txq); 6762 6763 br = txq->br; 6764 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 6765 if (m) 6766 t4_eth_tx(ifp, txq, m); 6767 } 6768 6769 void 6770 t4_tx_callout(void *arg) 6771 { 6772 struct sge_eq *eq = arg; 6773 struct adapter *sc; 6774 6775 if (EQ_TRYLOCK(eq) == 0) 6776 goto reschedule; 6777 6778 if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) { 6779 EQ_UNLOCK(eq); 6780 reschedule: 6781 if (__predict_true(!(eq->flags && EQ_DOOMED))) 6782 callout_schedule(&eq->tx_callout, 1); 6783 return; 6784 } 6785 6786 EQ_LOCK_ASSERT_OWNED(eq); 6787 6788 if (__predict_true((eq->flags & EQ_DOOMED) == 0)) { 6789 6790 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 6791 struct sge_txq *txq = arg; 6792 struct port_info *pi = txq->ifp->if_softc; 6793 6794 sc = pi->adapter; 6795 } else { 6796 struct sge_wrq *wrq = arg; 6797 6798 sc = wrq->adapter; 6799 } 6800 6801 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task); 6802 } 6803 6804 EQ_UNLOCK(eq); 6805 } 6806 6807 void 6808 t4_tx_task(void *arg, int count) 6809 { 6810 struct sge_eq *eq = arg; 6811 6812 EQ_LOCK(eq); 6813 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 6814 struct sge_txq *txq = arg; 6815 txq_start(txq->ifp, txq); 6816 } else { 6817 struct sge_wrq *wrq = arg; 6818 t4_wrq_tx_locked(wrq->adapter, wrq, NULL); 6819 } 6820 EQ_UNLOCK(eq); 6821 } 6822 6823 static uint32_t 6824 fconf_to_mode(uint32_t fconf) 6825 { 6826 uint32_t mode; 6827 6828 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 6829 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 6830 6831 if (fconf & F_FRAGMENTATION) 6832 mode |= T4_FILTER_IP_FRAGMENT; 6833 6834 if (fconf & F_MPSHITTYPE) 6835 mode |= T4_FILTER_MPS_HIT_TYPE; 6836 6837 if (fconf & F_MACMATCH) 6838 mode |= T4_FILTER_MAC_IDX; 6839 6840 if (fconf & F_ETHERTYPE) 6841 mode |= T4_FILTER_ETH_TYPE; 6842 6843 if (fconf & F_PROTOCOL) 6844 mode |= T4_FILTER_IP_PROTO; 6845 6846 if (fconf & F_TOS) 6847 mode |= T4_FILTER_IP_TOS; 6848 6849 if (fconf & F_VLAN) 6850 mode |= T4_FILTER_VLAN; 6851 6852 if (fconf & F_VNIC_ID) 6853 mode |= T4_FILTER_VNIC; 6854 6855 if (fconf & F_PORT) 6856 mode |= T4_FILTER_PORT; 6857 6858 if (fconf & F_FCOE) 6859 mode |= T4_FILTER_FCoE; 6860 6861 return (mode); 6862 } 6863 6864 static uint32_t 6865 mode_to_fconf(uint32_t mode) 6866 { 6867 uint32_t fconf = 0; 6868 6869 if (mode & T4_FILTER_IP_FRAGMENT) 6870 fconf |= F_FRAGMENTATION; 6871 6872 if (mode & T4_FILTER_MPS_HIT_TYPE) 6873 fconf |= F_MPSHITTYPE; 6874 6875 if (mode & T4_FILTER_MAC_IDX) 6876 fconf |= F_MACMATCH; 6877 6878 if (mode & T4_FILTER_ETH_TYPE) 6879 fconf |= F_ETHERTYPE; 6880 6881 if (mode & T4_FILTER_IP_PROTO) 6882 fconf |= F_PROTOCOL; 6883 6884 if (mode & T4_FILTER_IP_TOS) 6885 fconf |= F_TOS; 6886 6887 if (mode & T4_FILTER_VLAN) 6888 fconf |= F_VLAN; 6889 6890 if (mode & T4_FILTER_VNIC) 6891 fconf |= F_VNIC_ID; 6892 6893 if (mode & T4_FILTER_PORT) 6894 fconf |= F_PORT; 6895 6896 if (mode & T4_FILTER_FCoE) 6897 fconf |= F_FCOE; 6898 6899 return (fconf); 6900 } 6901 6902 static uint32_t 6903 fspec_to_fconf(struct t4_filter_specification *fs) 6904 { 6905 uint32_t fconf = 0; 6906 6907 if (fs->val.frag || fs->mask.frag) 6908 fconf |= F_FRAGMENTATION; 6909 6910 if (fs->val.matchtype || fs->mask.matchtype) 6911 fconf |= F_MPSHITTYPE; 6912 6913 if (fs->val.macidx || fs->mask.macidx) 6914 fconf |= F_MACMATCH; 6915 6916 if (fs->val.ethtype || fs->mask.ethtype) 6917 fconf |= F_ETHERTYPE; 6918 6919 if (fs->val.proto || fs->mask.proto) 6920 fconf |= F_PROTOCOL; 6921 6922 if (fs->val.tos || fs->mask.tos) 6923 fconf |= F_TOS; 6924 6925 if (fs->val.vlan_vld || fs->mask.vlan_vld) 6926 fconf |= F_VLAN; 6927 6928 if (fs->val.vnic_vld || fs->mask.vnic_vld) 6929 fconf |= F_VNIC_ID; 6930 6931 if (fs->val.iport || fs->mask.iport) 6932 fconf |= F_PORT; 6933 6934 if (fs->val.fcoe || fs->mask.fcoe) 6935 fconf |= F_FCOE; 6936 6937 return (fconf); 6938 } 6939 6940 static int 6941 get_filter_mode(struct adapter *sc, uint32_t *mode) 6942 { 6943 int rc; 6944 uint32_t fconf; 6945 6946 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 6947 "t4getfm"); 6948 if (rc) 6949 return (rc); 6950 6951 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 6952 A_TP_VLAN_PRI_MAP); 6953 6954 if (sc->params.tp.vlan_pri_map != fconf) { 6955 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 6956 device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map, 6957 fconf); 6958 sc->params.tp.vlan_pri_map = fconf; 6959 } 6960 6961 *mode = fconf_to_mode(sc->params.tp.vlan_pri_map); 6962 6963 end_synchronized_op(sc, LOCK_HELD); 6964 return (0); 6965 } 6966 6967 static int 6968 set_filter_mode(struct adapter *sc, uint32_t mode) 6969 { 6970 uint32_t fconf; 6971 int rc; 6972 6973 fconf = mode_to_fconf(mode); 6974 6975 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 6976 "t4setfm"); 6977 if (rc) 6978 return (rc); 6979 6980 if (sc->tids.ftids_in_use > 0) { 6981 rc = EBUSY; 6982 goto done; 6983 } 6984 6985 #ifdef TCP_OFFLOAD 6986 if (sc->offload_map) { 6987 rc = EBUSY; 6988 goto done; 6989 } 6990 #endif 6991 6992 #ifdef notyet 6993 rc = -t4_set_filter_mode(sc, fconf); 6994 if (rc == 0) 6995 sc->filter_mode = fconf; 6996 #else 6997 rc = ENOTSUP; 6998 #endif 6999 7000 done: 7001 end_synchronized_op(sc, LOCK_HELD); 7002 return (rc); 7003 } 7004 7005 static inline uint64_t 7006 get_filter_hits(struct adapter *sc, uint32_t fid) 7007 { 7008 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 7009 uint64_t hits; 7010 7011 memwin_info(sc, 0, &mw_base, NULL); 7012 off = position_memwin(sc, 0, 7013 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 7014 if (is_t4(sc)) { 7015 hits = t4_read_reg64(sc, mw_base + off + 16); 7016 hits = be64toh(hits); 7017 } else { 7018 hits = t4_read_reg(sc, mw_base + off + 24); 7019 hits = be32toh(hits); 7020 } 7021 7022 return (hits); 7023 } 7024 7025 static int 7026 get_filter(struct adapter *sc, struct t4_filter *t) 7027 { 7028 int i, rc, nfilters = sc->tids.nftids; 7029 struct filter_entry *f; 7030 7031 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7032 "t4getf"); 7033 if (rc) 7034 return (rc); 7035 7036 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 7037 t->idx >= nfilters) { 7038 t->idx = 0xffffffff; 7039 goto done; 7040 } 7041 7042 f = &sc->tids.ftid_tab[t->idx]; 7043 for (i = t->idx; i < nfilters; i++, f++) { 7044 if (f->valid) { 7045 t->idx = i; 7046 t->l2tidx = f->l2t ? f->l2t->idx : 0; 7047 t->smtidx = f->smtidx; 7048 if (f->fs.hitcnts) 7049 t->hits = get_filter_hits(sc, t->idx); 7050 else 7051 t->hits = UINT64_MAX; 7052 t->fs = f->fs; 7053 7054 goto done; 7055 } 7056 } 7057 7058 t->idx = 0xffffffff; 7059 done: 7060 end_synchronized_op(sc, LOCK_HELD); 7061 return (0); 7062 } 7063 7064 static int 7065 set_filter(struct adapter *sc, struct t4_filter *t) 7066 { 7067 unsigned int nfilters, nports; 7068 struct filter_entry *f; 7069 int i, rc; 7070 7071 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 7072 if (rc) 7073 return (rc); 7074 7075 nfilters = sc->tids.nftids; 7076 nports = sc->params.nports; 7077 7078 if (nfilters == 0) { 7079 rc = ENOTSUP; 7080 goto done; 7081 } 7082 7083 if (!(sc->flags & FULL_INIT_DONE)) { 7084 rc = EAGAIN; 7085 goto done; 7086 } 7087 7088 if (t->idx >= nfilters) { 7089 rc = EINVAL; 7090 goto done; 7091 } 7092 7093 /* Validate against the global filter mode */ 7094 if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) != 7095 sc->params.tp.vlan_pri_map) { 7096 rc = E2BIG; 7097 goto done; 7098 } 7099 7100 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 7101 rc = EINVAL; 7102 goto done; 7103 } 7104 7105 if (t->fs.val.iport >= nports) { 7106 rc = EINVAL; 7107 goto done; 7108 } 7109 7110 /* Can't specify an iq if not steering to it */ 7111 if (!t->fs.dirsteer && t->fs.iq) { 7112 rc = EINVAL; 7113 goto done; 7114 } 7115 7116 /* IPv6 filter idx must be 4 aligned */ 7117 if (t->fs.type == 1 && 7118 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 7119 rc = EINVAL; 7120 goto done; 7121 } 7122 7123 if (sc->tids.ftid_tab == NULL) { 7124 KASSERT(sc->tids.ftids_in_use == 0, 7125 ("%s: no memory allocated but filters_in_use > 0", 7126 __func__)); 7127 7128 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 7129 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 7130 if (sc->tids.ftid_tab == NULL) { 7131 rc = ENOMEM; 7132 goto done; 7133 } 7134 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 7135 } 7136 7137 for (i = 0; i < 4; i++) { 7138 f = &sc->tids.ftid_tab[t->idx + i]; 7139 7140 if (f->pending || f->valid) { 7141 rc = EBUSY; 7142 goto done; 7143 } 7144 if (f->locked) { 7145 rc = EPERM; 7146 goto done; 7147 } 7148 7149 if (t->fs.type == 0) 7150 break; 7151 } 7152 7153 f = &sc->tids.ftid_tab[t->idx]; 7154 f->fs = t->fs; 7155 7156 rc = set_filter_wr(sc, t->idx); 7157 done: 7158 end_synchronized_op(sc, 0); 7159 7160 if (rc == 0) { 7161 mtx_lock(&sc->tids.ftid_lock); 7162 for (;;) { 7163 if (f->pending == 0) { 7164 rc = f->valid ? 0 : EIO; 7165 break; 7166 } 7167 7168 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7169 PCATCH, "t4setfw", 0)) { 7170 rc = EINPROGRESS; 7171 break; 7172 } 7173 } 7174 mtx_unlock(&sc->tids.ftid_lock); 7175 } 7176 return (rc); 7177 } 7178 7179 static int 7180 del_filter(struct adapter *sc, struct t4_filter *t) 7181 { 7182 unsigned int nfilters; 7183 struct filter_entry *f; 7184 int rc; 7185 7186 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 7187 if (rc) 7188 return (rc); 7189 7190 nfilters = sc->tids.nftids; 7191 7192 if (nfilters == 0) { 7193 rc = ENOTSUP; 7194 goto done; 7195 } 7196 7197 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 7198 t->idx >= nfilters) { 7199 rc = EINVAL; 7200 goto done; 7201 } 7202 7203 if (!(sc->flags & FULL_INIT_DONE)) { 7204 rc = EAGAIN; 7205 goto done; 7206 } 7207 7208 f = &sc->tids.ftid_tab[t->idx]; 7209 7210 if (f->pending) { 7211 rc = EBUSY; 7212 goto done; 7213 } 7214 if (f->locked) { 7215 rc = EPERM; 7216 goto done; 7217 } 7218 7219 if (f->valid) { 7220 t->fs = f->fs; /* extra info for the caller */ 7221 rc = del_filter_wr(sc, t->idx); 7222 } 7223 7224 done: 7225 end_synchronized_op(sc, 0); 7226 7227 if (rc == 0) { 7228 mtx_lock(&sc->tids.ftid_lock); 7229 for (;;) { 7230 if (f->pending == 0) { 7231 rc = f->valid ? EIO : 0; 7232 break; 7233 } 7234 7235 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7236 PCATCH, "t4delfw", 0)) { 7237 rc = EINPROGRESS; 7238 break; 7239 } 7240 } 7241 mtx_unlock(&sc->tids.ftid_lock); 7242 } 7243 7244 return (rc); 7245 } 7246 7247 static void 7248 clear_filter(struct filter_entry *f) 7249 { 7250 if (f->l2t) 7251 t4_l2t_release(f->l2t); 7252 7253 bzero(f, sizeof (*f)); 7254 } 7255 7256 static int 7257 set_filter_wr(struct adapter *sc, int fidx) 7258 { 7259 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7260 struct wrqe *wr; 7261 struct fw_filter_wr *fwr; 7262 unsigned int ftid; 7263 7264 ASSERT_SYNCHRONIZED_OP(sc); 7265 7266 if (f->fs.newdmac || f->fs.newvlan) { 7267 /* This filter needs an L2T entry; allocate one. */ 7268 f->l2t = t4_l2t_alloc_switching(sc->l2t); 7269 if (f->l2t == NULL) 7270 return (EAGAIN); 7271 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 7272 f->fs.dmac)) { 7273 t4_l2t_release(f->l2t); 7274 f->l2t = NULL; 7275 return (ENOMEM); 7276 } 7277 } 7278 7279 ftid = sc->tids.ftid_base + fidx; 7280 7281 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 7282 if (wr == NULL) 7283 return (ENOMEM); 7284 7285 fwr = wrtod(wr); 7286 bzero(fwr, sizeof (*fwr)); 7287 7288 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 7289 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 7290 fwr->tid_to_iq = 7291 htobe32(V_FW_FILTER_WR_TID(ftid) | 7292 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 7293 V_FW_FILTER_WR_NOREPLY(0) | 7294 V_FW_FILTER_WR_IQ(f->fs.iq)); 7295 fwr->del_filter_to_l2tix = 7296 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 7297 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 7298 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 7299 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 7300 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 7301 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 7302 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 7303 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 7304 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 7305 f->fs.newvlan == VLAN_REWRITE) | 7306 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 7307 f->fs.newvlan == VLAN_REWRITE) | 7308 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 7309 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 7310 V_FW_FILTER_WR_PRIO(f->fs.prio) | 7311 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 7312 fwr->ethtype = htobe16(f->fs.val.ethtype); 7313 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 7314 fwr->frag_to_ovlan_vldm = 7315 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 7316 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 7317 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 7318 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 7319 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 7320 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 7321 fwr->smac_sel = 0; 7322 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 7323 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 7324 fwr->maci_to_matchtypem = 7325 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 7326 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 7327 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 7328 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 7329 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 7330 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 7331 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 7332 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 7333 fwr->ptcl = f->fs.val.proto; 7334 fwr->ptclm = f->fs.mask.proto; 7335 fwr->ttyp = f->fs.val.tos; 7336 fwr->ttypm = f->fs.mask.tos; 7337 fwr->ivlan = htobe16(f->fs.val.vlan); 7338 fwr->ivlanm = htobe16(f->fs.mask.vlan); 7339 fwr->ovlan = htobe16(f->fs.val.vnic); 7340 fwr->ovlanm = htobe16(f->fs.mask.vnic); 7341 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 7342 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 7343 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 7344 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 7345 fwr->lp = htobe16(f->fs.val.dport); 7346 fwr->lpm = htobe16(f->fs.mask.dport); 7347 fwr->fp = htobe16(f->fs.val.sport); 7348 fwr->fpm = htobe16(f->fs.mask.sport); 7349 if (f->fs.newsmac) 7350 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 7351 7352 f->pending = 1; 7353 sc->tids.ftids_in_use++; 7354 7355 t4_wrq_tx(sc, wr); 7356 return (0); 7357 } 7358 7359 static int 7360 del_filter_wr(struct adapter *sc, int fidx) 7361 { 7362 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7363 struct wrqe *wr; 7364 struct fw_filter_wr *fwr; 7365 unsigned int ftid; 7366 7367 ftid = sc->tids.ftid_base + fidx; 7368 7369 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 7370 if (wr == NULL) 7371 return (ENOMEM); 7372 fwr = wrtod(wr); 7373 bzero(fwr, sizeof (*fwr)); 7374 7375 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 7376 7377 f->pending = 1; 7378 t4_wrq_tx(sc, wr); 7379 return (0); 7380 } 7381 7382 int 7383 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 7384 { 7385 struct adapter *sc = iq->adapter; 7386 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 7387 unsigned int idx = GET_TID(rpl); 7388 unsigned int rc; 7389 struct filter_entry *f; 7390 7391 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 7392 rss->opcode)); 7393 7394 if (is_ftid(sc, idx)) { 7395 7396 idx -= sc->tids.ftid_base; 7397 f = &sc->tids.ftid_tab[idx]; 7398 rc = G_COOKIE(rpl->cookie); 7399 7400 mtx_lock(&sc->tids.ftid_lock); 7401 if (rc == FW_FILTER_WR_FLT_ADDED) { 7402 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 7403 __func__, idx)); 7404 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 7405 f->pending = 0; /* asynchronous setup completed */ 7406 f->valid = 1; 7407 } else { 7408 if (rc != FW_FILTER_WR_FLT_DELETED) { 7409 /* Add or delete failed, display an error */ 7410 log(LOG_ERR, 7411 "filter %u setup failed with error %u\n", 7412 idx, rc); 7413 } 7414 7415 clear_filter(f); 7416 sc->tids.ftids_in_use--; 7417 } 7418 wakeup(&sc->tids.ftid_tab); 7419 mtx_unlock(&sc->tids.ftid_lock); 7420 } 7421 7422 return (0); 7423 } 7424 7425 static int 7426 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 7427 { 7428 int rc; 7429 7430 if (cntxt->cid > M_CTXTQID) 7431 return (EINVAL); 7432 7433 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 7434 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 7435 return (EINVAL); 7436 7437 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 7438 if (rc) 7439 return (rc); 7440 7441 if (sc->flags & FW_OK) { 7442 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 7443 &cntxt->data[0]); 7444 if (rc == 0) 7445 goto done; 7446 } 7447 7448 /* 7449 * Read via firmware failed or wasn't even attempted. Read directly via 7450 * the backdoor. 7451 */ 7452 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 7453 done: 7454 end_synchronized_op(sc, 0); 7455 return (rc); 7456 } 7457 7458 static int 7459 load_fw(struct adapter *sc, struct t4_data *fw) 7460 { 7461 int rc; 7462 uint8_t *fw_data; 7463 7464 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 7465 if (rc) 7466 return (rc); 7467 7468 if (sc->flags & FULL_INIT_DONE) { 7469 rc = EBUSY; 7470 goto done; 7471 } 7472 7473 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 7474 if (fw_data == NULL) { 7475 rc = ENOMEM; 7476 goto done; 7477 } 7478 7479 rc = copyin(fw->data, fw_data, fw->len); 7480 if (rc == 0) 7481 rc = -t4_load_fw(sc, fw_data, fw->len); 7482 7483 free(fw_data, M_CXGBE); 7484 done: 7485 end_synchronized_op(sc, 0); 7486 return (rc); 7487 } 7488 7489 static int 7490 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 7491 { 7492 uint32_t addr, off, remaining, i, n; 7493 uint32_t *buf, *b; 7494 uint32_t mw_base, mw_aperture; 7495 int rc; 7496 uint8_t *dst; 7497 7498 rc = validate_mem_range(sc, mr->addr, mr->len); 7499 if (rc != 0) 7500 return (rc); 7501 7502 memwin_info(sc, win, &mw_base, &mw_aperture); 7503 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 7504 addr = mr->addr; 7505 remaining = mr->len; 7506 dst = (void *)mr->data; 7507 7508 while (remaining) { 7509 off = position_memwin(sc, win, addr); 7510 7511 /* number of bytes that we'll copy in the inner loop */ 7512 n = min(remaining, mw_aperture - off); 7513 for (i = 0; i < n; i += 4) 7514 *b++ = t4_read_reg(sc, mw_base + off + i); 7515 7516 rc = copyout(buf, dst, n); 7517 if (rc != 0) 7518 break; 7519 7520 b = buf; 7521 dst += n; 7522 remaining -= n; 7523 addr += n; 7524 } 7525 7526 free(buf, M_CXGBE); 7527 return (rc); 7528 } 7529 7530 static int 7531 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 7532 { 7533 int rc; 7534 7535 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 7536 return (EINVAL); 7537 7538 if (i2cd->len > 1) { 7539 /* XXX: need fw support for longer reads in one go */ 7540 return (ENOTSUP); 7541 } 7542 7543 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 7544 if (rc) 7545 return (rc); 7546 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 7547 i2cd->offset, &i2cd->data[0]); 7548 end_synchronized_op(sc, 0); 7549 7550 return (rc); 7551 } 7552 7553 static int 7554 in_range(int val, int lo, int hi) 7555 { 7556 7557 return (val < 0 || (val <= hi && val >= lo)); 7558 } 7559 7560 static int 7561 set_sched_class(struct adapter *sc, struct t4_sched_params *p) 7562 { 7563 int fw_subcmd, fw_type, rc; 7564 7565 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc"); 7566 if (rc) 7567 return (rc); 7568 7569 if (!(sc->flags & FULL_INIT_DONE)) { 7570 rc = EAGAIN; 7571 goto done; 7572 } 7573 7574 /* 7575 * Translate the cxgbetool parameters into T4 firmware parameters. (The 7576 * sub-command and type are in common locations.) 7577 */ 7578 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG) 7579 fw_subcmd = FW_SCHED_SC_CONFIG; 7580 else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS) 7581 fw_subcmd = FW_SCHED_SC_PARAMS; 7582 else { 7583 rc = EINVAL; 7584 goto done; 7585 } 7586 if (p->type == SCHED_CLASS_TYPE_PACKET) 7587 fw_type = FW_SCHED_TYPE_PKTSCHED; 7588 else { 7589 rc = EINVAL; 7590 goto done; 7591 } 7592 7593 if (fw_subcmd == FW_SCHED_SC_CONFIG) { 7594 /* Vet our parameters ..*/ 7595 if (p->u.config.minmax < 0) { 7596 rc = EINVAL; 7597 goto done; 7598 } 7599 7600 /* And pass the request to the firmware ...*/ 7601 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax); 7602 goto done; 7603 } 7604 7605 if (fw_subcmd == FW_SCHED_SC_PARAMS) { 7606 int fw_level; 7607 int fw_mode; 7608 int fw_rateunit; 7609 int fw_ratemode; 7610 7611 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL) 7612 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL; 7613 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) 7614 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR; 7615 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) 7616 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL; 7617 else { 7618 rc = EINVAL; 7619 goto done; 7620 } 7621 7622 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS) 7623 fw_mode = FW_SCHED_PARAMS_MODE_CLASS; 7624 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW) 7625 fw_mode = FW_SCHED_PARAMS_MODE_FLOW; 7626 else { 7627 rc = EINVAL; 7628 goto done; 7629 } 7630 7631 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS) 7632 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 7633 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS) 7634 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE; 7635 else { 7636 rc = EINVAL; 7637 goto done; 7638 } 7639 7640 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL) 7641 fw_ratemode = FW_SCHED_PARAMS_RATE_REL; 7642 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS) 7643 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS; 7644 else { 7645 rc = EINVAL; 7646 goto done; 7647 } 7648 7649 /* Vet our parameters ... */ 7650 if (!in_range(p->u.params.channel, 0, 3) || 7651 !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) || 7652 !in_range(p->u.params.minrate, 0, 10000000) || 7653 !in_range(p->u.params.maxrate, 0, 10000000) || 7654 !in_range(p->u.params.weight, 0, 100)) { 7655 rc = ERANGE; 7656 goto done; 7657 } 7658 7659 /* 7660 * Translate any unset parameters into the firmware's 7661 * nomenclature and/or fail the call if the parameters 7662 * are required ... 7663 */ 7664 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 || 7665 p->u.params.channel < 0 || p->u.params.cl < 0) { 7666 rc = EINVAL; 7667 goto done; 7668 } 7669 if (p->u.params.minrate < 0) 7670 p->u.params.minrate = 0; 7671 if (p->u.params.maxrate < 0) { 7672 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7673 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7674 rc = EINVAL; 7675 goto done; 7676 } else 7677 p->u.params.maxrate = 0; 7678 } 7679 if (p->u.params.weight < 0) { 7680 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) { 7681 rc = EINVAL; 7682 goto done; 7683 } else 7684 p->u.params.weight = 0; 7685 } 7686 if (p->u.params.pktsize < 0) { 7687 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7688 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7689 rc = EINVAL; 7690 goto done; 7691 } else 7692 p->u.params.pktsize = 0; 7693 } 7694 7695 /* See what the firmware thinks of the request ... */ 7696 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode, 7697 fw_rateunit, fw_ratemode, p->u.params.channel, 7698 p->u.params.cl, p->u.params.minrate, p->u.params.maxrate, 7699 p->u.params.weight, p->u.params.pktsize); 7700 goto done; 7701 } 7702 7703 rc = EINVAL; 7704 done: 7705 end_synchronized_op(sc, 0); 7706 return (rc); 7707 } 7708 7709 static int 7710 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p) 7711 { 7712 struct port_info *pi = NULL; 7713 struct sge_txq *txq; 7714 uint32_t fw_mnem, fw_queue, fw_class; 7715 int i, rc; 7716 7717 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq"); 7718 if (rc) 7719 return (rc); 7720 7721 if (!(sc->flags & FULL_INIT_DONE)) { 7722 rc = EAGAIN; 7723 goto done; 7724 } 7725 7726 if (p->port >= sc->params.nports) { 7727 rc = EINVAL; 7728 goto done; 7729 } 7730 7731 pi = sc->port[p->port]; 7732 if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) { 7733 rc = EINVAL; 7734 goto done; 7735 } 7736 7737 /* 7738 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX 7739 * Scheduling Class in this case). 7740 */ 7741 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 7742 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH)); 7743 fw_class = p->cl < 0 ? 0xffffffff : p->cl; 7744 7745 /* 7746 * If op.queue is non-negative, then we're only changing the scheduling 7747 * on a single specified TX queue. 7748 */ 7749 if (p->queue >= 0) { 7750 txq = &sc->sge.txq[pi->first_txq + p->queue]; 7751 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7752 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7753 &fw_class); 7754 goto done; 7755 } 7756 7757 /* 7758 * Change the scheduling on all the TX queues for the 7759 * interface. 7760 */ 7761 for_each_txq(pi, i, txq) { 7762 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7763 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7764 &fw_class); 7765 if (rc) 7766 goto done; 7767 } 7768 7769 rc = 0; 7770 done: 7771 end_synchronized_op(sc, 0); 7772 return (rc); 7773 } 7774 7775 int 7776 t4_os_find_pci_capability(struct adapter *sc, int cap) 7777 { 7778 int i; 7779 7780 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 7781 } 7782 7783 int 7784 t4_os_pci_save_state(struct adapter *sc) 7785 { 7786 device_t dev; 7787 struct pci_devinfo *dinfo; 7788 7789 dev = sc->dev; 7790 dinfo = device_get_ivars(dev); 7791 7792 pci_cfg_save(dev, dinfo, 0); 7793 return (0); 7794 } 7795 7796 int 7797 t4_os_pci_restore_state(struct adapter *sc) 7798 { 7799 device_t dev; 7800 struct pci_devinfo *dinfo; 7801 7802 dev = sc->dev; 7803 dinfo = device_get_ivars(dev); 7804 7805 pci_cfg_restore(dev, dinfo); 7806 return (0); 7807 } 7808 7809 void 7810 t4_os_portmod_changed(const struct adapter *sc, int idx) 7811 { 7812 struct port_info *pi = sc->port[idx]; 7813 static const char *mod_str[] = { 7814 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 7815 }; 7816 7817 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 7818 if_printf(pi->ifp, "transceiver unplugged.\n"); 7819 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 7820 if_printf(pi->ifp, "unknown transceiver inserted.\n"); 7821 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 7822 if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 7823 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 7824 if_printf(pi->ifp, "%s transceiver inserted.\n", 7825 mod_str[pi->mod_type]); 7826 } else { 7827 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 7828 pi->mod_type); 7829 } 7830 } 7831 7832 void 7833 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason) 7834 { 7835 struct port_info *pi = sc->port[idx]; 7836 struct ifnet *ifp = pi->ifp; 7837 7838 if (link_stat) { 7839 pi->linkdnrc = -1; 7840 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 7841 if_link_state_change(ifp, LINK_STATE_UP); 7842 } else { 7843 if (reason >= 0) 7844 pi->linkdnrc = reason; 7845 if_link_state_change(ifp, LINK_STATE_DOWN); 7846 } 7847 } 7848 7849 void 7850 t4_iterate(void (*func)(struct adapter *, void *), void *arg) 7851 { 7852 struct adapter *sc; 7853 7854 sx_slock(&t4_list_lock); 7855 SLIST_FOREACH(sc, &t4_list, link) { 7856 /* 7857 * func should not make any assumptions about what state sc is 7858 * in - the only guarantee is that sc->sc_lock is a valid lock. 7859 */ 7860 func(sc, arg); 7861 } 7862 sx_sunlock(&t4_list_lock); 7863 } 7864 7865 static int 7866 t4_open(struct cdev *dev, int flags, int type, struct thread *td) 7867 { 7868 return (0); 7869 } 7870 7871 static int 7872 t4_close(struct cdev *dev, int flags, int type, struct thread *td) 7873 { 7874 return (0); 7875 } 7876 7877 static int 7878 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 7879 struct thread *td) 7880 { 7881 int rc; 7882 struct adapter *sc = dev->si_drv1; 7883 7884 rc = priv_check(td, PRIV_DRIVER); 7885 if (rc != 0) 7886 return (rc); 7887 7888 switch (cmd) { 7889 case CHELSIO_T4_GETREG: { 7890 struct t4_reg *edata = (struct t4_reg *)data; 7891 7892 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 7893 return (EFAULT); 7894 7895 if (edata->size == 4) 7896 edata->val = t4_read_reg(sc, edata->addr); 7897 else if (edata->size == 8) 7898 edata->val = t4_read_reg64(sc, edata->addr); 7899 else 7900 return (EINVAL); 7901 7902 break; 7903 } 7904 case CHELSIO_T4_SETREG: { 7905 struct t4_reg *edata = (struct t4_reg *)data; 7906 7907 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 7908 return (EFAULT); 7909 7910 if (edata->size == 4) { 7911 if (edata->val & 0xffffffff00000000) 7912 return (EINVAL); 7913 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 7914 } else if (edata->size == 8) 7915 t4_write_reg64(sc, edata->addr, edata->val); 7916 else 7917 return (EINVAL); 7918 break; 7919 } 7920 case CHELSIO_T4_REGDUMP: { 7921 struct t4_regdump *regs = (struct t4_regdump *)data; 7922 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 7923 uint8_t *buf; 7924 7925 if (regs->len < reglen) { 7926 regs->len = reglen; /* hint to the caller */ 7927 return (ENOBUFS); 7928 } 7929 7930 regs->len = reglen; 7931 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 7932 t4_get_regs(sc, regs, buf); 7933 rc = copyout(buf, regs->data, reglen); 7934 free(buf, M_CXGBE); 7935 break; 7936 } 7937 case CHELSIO_T4_GET_FILTER_MODE: 7938 rc = get_filter_mode(sc, (uint32_t *)data); 7939 break; 7940 case CHELSIO_T4_SET_FILTER_MODE: 7941 rc = set_filter_mode(sc, *(uint32_t *)data); 7942 break; 7943 case CHELSIO_T4_GET_FILTER: 7944 rc = get_filter(sc, (struct t4_filter *)data); 7945 break; 7946 case CHELSIO_T4_SET_FILTER: 7947 rc = set_filter(sc, (struct t4_filter *)data); 7948 break; 7949 case CHELSIO_T4_DEL_FILTER: 7950 rc = del_filter(sc, (struct t4_filter *)data); 7951 break; 7952 case CHELSIO_T4_GET_SGE_CONTEXT: 7953 rc = get_sge_context(sc, (struct t4_sge_context *)data); 7954 break; 7955 case CHELSIO_T4_LOAD_FW: 7956 rc = load_fw(sc, (struct t4_data *)data); 7957 break; 7958 case CHELSIO_T4_GET_MEM: 7959 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 7960 break; 7961 case CHELSIO_T4_GET_I2C: 7962 rc = read_i2c(sc, (struct t4_i2c_data *)data); 7963 break; 7964 case CHELSIO_T4_CLEAR_STATS: { 7965 int i; 7966 u_int port_id = *(uint32_t *)data; 7967 struct port_info *pi; 7968 7969 if (port_id >= sc->params.nports) 7970 return (EINVAL); 7971 pi = sc->port[port_id]; 7972 7973 /* MAC stats */ 7974 t4_clr_port_stats(sc, pi->tx_chan); 7975 7976 if (pi->flags & PORT_INIT_DONE) { 7977 struct sge_rxq *rxq; 7978 struct sge_txq *txq; 7979 struct sge_wrq *wrq; 7980 7981 for_each_rxq(pi, i, rxq) { 7982 #if defined(INET) || defined(INET6) 7983 rxq->lro.lro_queued = 0; 7984 rxq->lro.lro_flushed = 0; 7985 #endif 7986 rxq->rxcsum = 0; 7987 rxq->vlan_extraction = 0; 7988 } 7989 7990 for_each_txq(pi, i, txq) { 7991 txq->txcsum = 0; 7992 txq->tso_wrs = 0; 7993 txq->vlan_insertion = 0; 7994 txq->imm_wrs = 0; 7995 txq->sgl_wrs = 0; 7996 txq->txpkt_wrs = 0; 7997 txq->txpkts_wrs = 0; 7998 txq->txpkts_pkts = 0; 7999 txq->br->br_drops = 0; 8000 txq->no_dmamap = 0; 8001 txq->no_desc = 0; 8002 } 8003 8004 #ifdef TCP_OFFLOAD 8005 /* nothing to clear for each ofld_rxq */ 8006 8007 for_each_ofld_txq(pi, i, wrq) { 8008 wrq->tx_wrs = 0; 8009 wrq->no_desc = 0; 8010 } 8011 #endif 8012 wrq = &sc->sge.ctrlq[pi->port_id]; 8013 wrq->tx_wrs = 0; 8014 wrq->no_desc = 0; 8015 } 8016 break; 8017 } 8018 case CHELSIO_T4_SCHED_CLASS: 8019 rc = set_sched_class(sc, (struct t4_sched_params *)data); 8020 break; 8021 case CHELSIO_T4_SCHED_QUEUE: 8022 rc = set_sched_queue(sc, (struct t4_sched_queue *)data); 8023 break; 8024 case CHELSIO_T4_GET_TRACER: 8025 rc = t4_get_tracer(sc, (struct t4_tracer *)data); 8026 break; 8027 case CHELSIO_T4_SET_TRACER: 8028 rc = t4_set_tracer(sc, (struct t4_tracer *)data); 8029 break; 8030 default: 8031 rc = EINVAL; 8032 } 8033 8034 return (rc); 8035 } 8036 8037 #ifdef TCP_OFFLOAD 8038 static int 8039 toe_capability(struct port_info *pi, int enable) 8040 { 8041 int rc; 8042 struct adapter *sc = pi->adapter; 8043 8044 ASSERT_SYNCHRONIZED_OP(sc); 8045 8046 if (!is_offload(sc)) 8047 return (ENODEV); 8048 8049 if (enable) { 8050 if (!(sc->flags & FULL_INIT_DONE)) { 8051 rc = cxgbe_init_synchronized(pi); 8052 if (rc) 8053 return (rc); 8054 } 8055 8056 if (isset(&sc->offload_map, pi->port_id)) 8057 return (0); 8058 8059 if (!(sc->flags & TOM_INIT_DONE)) { 8060 rc = t4_activate_uld(sc, ULD_TOM); 8061 if (rc == EAGAIN) { 8062 log(LOG_WARNING, 8063 "You must kldload t4_tom.ko before trying " 8064 "to enable TOE on a cxgbe interface.\n"); 8065 } 8066 if (rc != 0) 8067 return (rc); 8068 KASSERT(sc->tom_softc != NULL, 8069 ("%s: TOM activated but softc NULL", __func__)); 8070 KASSERT(sc->flags & TOM_INIT_DONE, 8071 ("%s: TOM activated but flag not set", __func__)); 8072 } 8073 8074 setbit(&sc->offload_map, pi->port_id); 8075 } else { 8076 if (!isset(&sc->offload_map, pi->port_id)) 8077 return (0); 8078 8079 KASSERT(sc->flags & TOM_INIT_DONE, 8080 ("%s: TOM never initialized?", __func__)); 8081 clrbit(&sc->offload_map, pi->port_id); 8082 } 8083 8084 return (0); 8085 } 8086 8087 /* 8088 * Add an upper layer driver to the global list. 8089 */ 8090 int 8091 t4_register_uld(struct uld_info *ui) 8092 { 8093 int rc = 0; 8094 struct uld_info *u; 8095 8096 sx_xlock(&t4_uld_list_lock); 8097 SLIST_FOREACH(u, &t4_uld_list, link) { 8098 if (u->uld_id == ui->uld_id) { 8099 rc = EEXIST; 8100 goto done; 8101 } 8102 } 8103 8104 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 8105 ui->refcount = 0; 8106 done: 8107 sx_xunlock(&t4_uld_list_lock); 8108 return (rc); 8109 } 8110 8111 int 8112 t4_unregister_uld(struct uld_info *ui) 8113 { 8114 int rc = EINVAL; 8115 struct uld_info *u; 8116 8117 sx_xlock(&t4_uld_list_lock); 8118 8119 SLIST_FOREACH(u, &t4_uld_list, link) { 8120 if (u == ui) { 8121 if (ui->refcount > 0) { 8122 rc = EBUSY; 8123 goto done; 8124 } 8125 8126 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 8127 rc = 0; 8128 goto done; 8129 } 8130 } 8131 done: 8132 sx_xunlock(&t4_uld_list_lock); 8133 return (rc); 8134 } 8135 8136 int 8137 t4_activate_uld(struct adapter *sc, int id) 8138 { 8139 int rc = EAGAIN; 8140 struct uld_info *ui; 8141 8142 ASSERT_SYNCHRONIZED_OP(sc); 8143 8144 sx_slock(&t4_uld_list_lock); 8145 8146 SLIST_FOREACH(ui, &t4_uld_list, link) { 8147 if (ui->uld_id == id) { 8148 rc = ui->activate(sc); 8149 if (rc == 0) 8150 ui->refcount++; 8151 goto done; 8152 } 8153 } 8154 done: 8155 sx_sunlock(&t4_uld_list_lock); 8156 8157 return (rc); 8158 } 8159 8160 int 8161 t4_deactivate_uld(struct adapter *sc, int id) 8162 { 8163 int rc = EINVAL; 8164 struct uld_info *ui; 8165 8166 ASSERT_SYNCHRONIZED_OP(sc); 8167 8168 sx_slock(&t4_uld_list_lock); 8169 8170 SLIST_FOREACH(ui, &t4_uld_list, link) { 8171 if (ui->uld_id == id) { 8172 rc = ui->deactivate(sc); 8173 if (rc == 0) 8174 ui->refcount--; 8175 goto done; 8176 } 8177 } 8178 done: 8179 sx_sunlock(&t4_uld_list_lock); 8180 8181 return (rc); 8182 } 8183 #endif 8184 8185 /* 8186 * Come up with reasonable defaults for some of the tunables, provided they're 8187 * not set by the user (in which case we'll use the values as is). 8188 */ 8189 static void 8190 tweak_tunables(void) 8191 { 8192 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 8193 8194 if (t4_ntxq10g < 1) 8195 t4_ntxq10g = min(nc, NTXQ_10G); 8196 8197 if (t4_ntxq1g < 1) 8198 t4_ntxq1g = min(nc, NTXQ_1G); 8199 8200 if (t4_nrxq10g < 1) 8201 t4_nrxq10g = min(nc, NRXQ_10G); 8202 8203 if (t4_nrxq1g < 1) 8204 t4_nrxq1g = min(nc, NRXQ_1G); 8205 8206 #ifdef TCP_OFFLOAD 8207 if (t4_nofldtxq10g < 1) 8208 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 8209 8210 if (t4_nofldtxq1g < 1) 8211 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 8212 8213 if (t4_nofldrxq10g < 1) 8214 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 8215 8216 if (t4_nofldrxq1g < 1) 8217 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 8218 8219 if (t4_toecaps_allowed == -1) 8220 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 8221 #else 8222 if (t4_toecaps_allowed == -1) 8223 t4_toecaps_allowed = 0; 8224 #endif 8225 8226 #ifdef DEV_NETMAP 8227 if (t4_nnmtxq10g < 1) 8228 t4_nnmtxq10g = min(nc, NNMTXQ_10G); 8229 8230 if (t4_nnmtxq1g < 1) 8231 t4_nnmtxq1g = min(nc, NNMTXQ_1G); 8232 8233 if (t4_nnmrxq10g < 1) 8234 t4_nnmrxq10g = min(nc, NNMRXQ_10G); 8235 8236 if (t4_nnmrxq1g < 1) 8237 t4_nnmrxq1g = min(nc, NNMRXQ_1G); 8238 #endif 8239 8240 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 8241 t4_tmr_idx_10g = TMR_IDX_10G; 8242 8243 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 8244 t4_pktc_idx_10g = PKTC_IDX_10G; 8245 8246 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 8247 t4_tmr_idx_1g = TMR_IDX_1G; 8248 8249 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 8250 t4_pktc_idx_1g = PKTC_IDX_1G; 8251 8252 if (t4_qsize_txq < 128) 8253 t4_qsize_txq = 128; 8254 8255 if (t4_qsize_rxq < 128) 8256 t4_qsize_rxq = 128; 8257 while (t4_qsize_rxq & 7) 8258 t4_qsize_rxq++; 8259 8260 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 8261 } 8262 8263 static int 8264 mod_event(module_t mod, int cmd, void *arg) 8265 { 8266 int rc = 0; 8267 static int loaded = 0; 8268 8269 switch (cmd) { 8270 case MOD_LOAD: 8271 if (atomic_fetchadd_int(&loaded, 1)) 8272 break; 8273 t4_sge_modload(); 8274 sx_init(&t4_list_lock, "T4/T5 adapters"); 8275 SLIST_INIT(&t4_list); 8276 #ifdef TCP_OFFLOAD 8277 sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); 8278 SLIST_INIT(&t4_uld_list); 8279 #endif 8280 t4_tracer_modload(); 8281 tweak_tunables(); 8282 break; 8283 8284 case MOD_UNLOAD: 8285 if (atomic_fetchadd_int(&loaded, -1) > 1) 8286 break; 8287 t4_tracer_modunload(); 8288 #ifdef TCP_OFFLOAD 8289 sx_slock(&t4_uld_list_lock); 8290 if (!SLIST_EMPTY(&t4_uld_list)) { 8291 rc = EBUSY; 8292 sx_sunlock(&t4_uld_list_lock); 8293 break; 8294 } 8295 sx_sunlock(&t4_uld_list_lock); 8296 sx_destroy(&t4_uld_list_lock); 8297 #endif 8298 sx_slock(&t4_list_lock); 8299 if (!SLIST_EMPTY(&t4_list)) { 8300 rc = EBUSY; 8301 sx_sunlock(&t4_list_lock); 8302 break; 8303 } 8304 sx_sunlock(&t4_list_lock); 8305 sx_destroy(&t4_list_lock); 8306 break; 8307 } 8308 8309 return (rc); 8310 } 8311 8312 static devclass_t t4_devclass, t5_devclass; 8313 static devclass_t cxgbe_devclass, cxl_devclass; 8314 8315 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 8316 MODULE_VERSION(t4nex, 1); 8317 MODULE_DEPEND(t4nex, firmware, 1, 1, 1); 8318 8319 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 8320 MODULE_VERSION(t5nex, 1); 8321 MODULE_DEPEND(t5nex, firmware, 1, 1, 1); 8322 8323 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 8324 MODULE_VERSION(cxgbe, 1); 8325 8326 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 8327 MODULE_VERSION(cxl, 1); 8328