1 /*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_inet.h" 32 #include "opt_inet6.h" 33 34 #include <sys/param.h> 35 #include <sys/conf.h> 36 #include <sys/priv.h> 37 #include <sys/kernel.h> 38 #include <sys/bus.h> 39 #include <sys/module.h> 40 #include <sys/malloc.h> 41 #include <sys/queue.h> 42 #include <sys/taskqueue.h> 43 #include <sys/pciio.h> 44 #include <dev/pci/pcireg.h> 45 #include <dev/pci/pcivar.h> 46 #include <dev/pci/pci_private.h> 47 #include <sys/firmware.h> 48 #include <sys/sbuf.h> 49 #include <sys/smp.h> 50 #include <sys/socket.h> 51 #include <sys/sockio.h> 52 #include <sys/sysctl.h> 53 #include <net/ethernet.h> 54 #include <net/if.h> 55 #include <net/if_types.h> 56 #include <net/if_dl.h> 57 #include <net/if_vlan_var.h> 58 #if defined(__i386__) || defined(__amd64__) 59 #include <vm/vm.h> 60 #include <vm/pmap.h> 61 #endif 62 63 #include "common/common.h" 64 #include "common/t4_msg.h" 65 #include "common/t4_regs.h" 66 #include "common/t4_regs_values.h" 67 #include "t4_ioctl.h" 68 #include "t4_l2t.h" 69 70 /* T4 bus driver interface */ 71 static int t4_probe(device_t); 72 static int t4_attach(device_t); 73 static int t4_detach(device_t); 74 static device_method_t t4_methods[] = { 75 DEVMETHOD(device_probe, t4_probe), 76 DEVMETHOD(device_attach, t4_attach), 77 DEVMETHOD(device_detach, t4_detach), 78 79 DEVMETHOD_END 80 }; 81 static driver_t t4_driver = { 82 "t4nex", 83 t4_methods, 84 sizeof(struct adapter) 85 }; 86 87 88 /* T4 port (cxgbe) interface */ 89 static int cxgbe_probe(device_t); 90 static int cxgbe_attach(device_t); 91 static int cxgbe_detach(device_t); 92 static device_method_t cxgbe_methods[] = { 93 DEVMETHOD(device_probe, cxgbe_probe), 94 DEVMETHOD(device_attach, cxgbe_attach), 95 DEVMETHOD(device_detach, cxgbe_detach), 96 { 0, 0 } 97 }; 98 static driver_t cxgbe_driver = { 99 "cxgbe", 100 cxgbe_methods, 101 sizeof(struct port_info) 102 }; 103 104 static d_ioctl_t t4_ioctl; 105 static d_open_t t4_open; 106 static d_close_t t4_close; 107 108 static struct cdevsw t4_cdevsw = { 109 .d_version = D_VERSION, 110 .d_flags = 0, 111 .d_open = t4_open, 112 .d_close = t4_close, 113 .d_ioctl = t4_ioctl, 114 .d_name = "t4nex", 115 }; 116 117 /* T5 bus driver interface */ 118 static int t5_probe(device_t); 119 static device_method_t t5_methods[] = { 120 DEVMETHOD(device_probe, t5_probe), 121 DEVMETHOD(device_attach, t4_attach), 122 DEVMETHOD(device_detach, t4_detach), 123 124 DEVMETHOD_END 125 }; 126 static driver_t t5_driver = { 127 "t5nex", 128 t5_methods, 129 sizeof(struct adapter) 130 }; 131 132 133 /* T5 port (cxl) interface */ 134 static driver_t cxl_driver = { 135 "cxl", 136 cxgbe_methods, 137 sizeof(struct port_info) 138 }; 139 140 static struct cdevsw t5_cdevsw = { 141 .d_version = D_VERSION, 142 .d_flags = 0, 143 .d_open = t4_open, 144 .d_close = t4_close, 145 .d_ioctl = t4_ioctl, 146 .d_name = "t5nex", 147 }; 148 149 /* ifnet + media interface */ 150 static void cxgbe_init(void *); 151 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 152 static int cxgbe_transmit(struct ifnet *, struct mbuf *); 153 static void cxgbe_qflush(struct ifnet *); 154 static int cxgbe_media_change(struct ifnet *); 155 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 156 157 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 158 159 /* 160 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 161 * then ADAPTER_LOCK, then t4_uld_list_lock. 162 */ 163 static struct sx t4_list_lock; 164 SLIST_HEAD(, adapter) t4_list; 165 #ifdef TCP_OFFLOAD 166 static struct sx t4_uld_list_lock; 167 SLIST_HEAD(, uld_info) t4_uld_list; 168 #endif 169 170 /* 171 * Tunables. See tweak_tunables() too. 172 * 173 * Each tunable is set to a default value here if it's known at compile-time. 174 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 175 * provide a reasonable default when the driver is loaded. 176 * 177 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 178 * T5 are under hw.cxl. 179 */ 180 181 /* 182 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 183 */ 184 #define NTXQ_10G 16 185 static int t4_ntxq10g = -1; 186 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 187 188 #define NRXQ_10G 8 189 static int t4_nrxq10g = -1; 190 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 191 192 #define NTXQ_1G 4 193 static int t4_ntxq1g = -1; 194 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 195 196 #define NRXQ_1G 2 197 static int t4_nrxq1g = -1; 198 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 199 200 static int t4_rsrv_noflowq = 0; 201 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq); 202 203 #ifdef TCP_OFFLOAD 204 #define NOFLDTXQ_10G 8 205 static int t4_nofldtxq10g = -1; 206 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 207 208 #define NOFLDRXQ_10G 2 209 static int t4_nofldrxq10g = -1; 210 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 211 212 #define NOFLDTXQ_1G 2 213 static int t4_nofldtxq1g = -1; 214 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 215 216 #define NOFLDRXQ_1G 1 217 static int t4_nofldrxq1g = -1; 218 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 219 #endif 220 221 #ifdef DEV_NETMAP 222 #define NNMTXQ_10G 2 223 static int t4_nnmtxq10g = -1; 224 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g); 225 226 #define NNMRXQ_10G 2 227 static int t4_nnmrxq10g = -1; 228 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g); 229 230 #define NNMTXQ_1G 1 231 static int t4_nnmtxq1g = -1; 232 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g); 233 234 #define NNMRXQ_1G 1 235 static int t4_nnmrxq1g = -1; 236 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g); 237 #endif 238 239 /* 240 * Holdoff parameters for 10G and 1G ports. 241 */ 242 #define TMR_IDX_10G 1 243 static int t4_tmr_idx_10g = TMR_IDX_10G; 244 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 245 246 #define PKTC_IDX_10G (-1) 247 static int t4_pktc_idx_10g = PKTC_IDX_10G; 248 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 249 250 #define TMR_IDX_1G 1 251 static int t4_tmr_idx_1g = TMR_IDX_1G; 252 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 253 254 #define PKTC_IDX_1G (-1) 255 static int t4_pktc_idx_1g = PKTC_IDX_1G; 256 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 257 258 /* 259 * Size (# of entries) of each tx and rx queue. 260 */ 261 static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 262 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 263 264 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 265 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 266 267 /* 268 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 269 */ 270 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 271 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 272 273 /* 274 * Configuration file. 275 */ 276 #define DEFAULT_CF "default" 277 #define FLASH_CF "flash" 278 #define UWIRE_CF "uwire" 279 #define FPGA_CF "fpga" 280 static char t4_cfg_file[32] = DEFAULT_CF; 281 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 282 283 /* 284 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 285 * encouraged respectively). 286 */ 287 static unsigned int t4_fw_install = 1; 288 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 289 290 /* 291 * ASIC features that will be used. Disable the ones you don't want so that the 292 * chip resources aren't wasted on features that will not be used. 293 */ 294 static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 295 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 296 297 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 298 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 299 300 static int t4_toecaps_allowed = -1; 301 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 302 303 static int t4_rdmacaps_allowed = 0; 304 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 305 306 static int t4_iscsicaps_allowed = 0; 307 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 308 309 static int t4_fcoecaps_allowed = 0; 310 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 311 312 static int t5_write_combine = 0; 313 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 314 315 struct intrs_and_queues { 316 uint16_t intr_type; /* INTx, MSI, or MSI-X */ 317 uint16_t nirq; /* Total # of vectors */ 318 uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */ 319 uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */ 320 uint16_t ntxq10g; /* # of NIC txq's for each 10G port */ 321 uint16_t nrxq10g; /* # of NIC rxq's for each 10G port */ 322 uint16_t ntxq1g; /* # of NIC txq's for each 1G port */ 323 uint16_t nrxq1g; /* # of NIC rxq's for each 1G port */ 324 uint16_t rsrv_noflowq; /* Flag whether to reserve queue 0 */ 325 #ifdef TCP_OFFLOAD 326 uint16_t nofldtxq10g; /* # of TOE txq's for each 10G port */ 327 uint16_t nofldrxq10g; /* # of TOE rxq's for each 10G port */ 328 uint16_t nofldtxq1g; /* # of TOE txq's for each 1G port */ 329 uint16_t nofldrxq1g; /* # of TOE rxq's for each 1G port */ 330 #endif 331 #ifdef DEV_NETMAP 332 uint16_t nnmtxq10g; /* # of netmap txq's for each 10G port */ 333 uint16_t nnmrxq10g; /* # of netmap rxq's for each 10G port */ 334 uint16_t nnmtxq1g; /* # of netmap txq's for each 1G port */ 335 uint16_t nnmrxq1g; /* # of netmap rxq's for each 1G port */ 336 #endif 337 }; 338 339 struct filter_entry { 340 uint32_t valid:1; /* filter allocated and valid */ 341 uint32_t locked:1; /* filter is administratively locked */ 342 uint32_t pending:1; /* filter action is pending firmware reply */ 343 uint32_t smtidx:8; /* Source MAC Table index for smac */ 344 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 345 346 struct t4_filter_specification fs; 347 }; 348 349 static int map_bars_0_and_4(struct adapter *); 350 static int map_bar_2(struct adapter *); 351 static void setup_memwin(struct adapter *); 352 static int validate_mem_range(struct adapter *, uint32_t, int); 353 static int fwmtype_to_hwmtype(int); 354 static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 355 uint32_t *); 356 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 357 static uint32_t position_memwin(struct adapter *, int, uint32_t); 358 static int cfg_itype_and_nqueues(struct adapter *, int, int, 359 struct intrs_and_queues *); 360 static int prep_firmware(struct adapter *); 361 static int partition_resources(struct adapter *, const struct firmware *, 362 const char *); 363 static int get_params__pre_init(struct adapter *); 364 static int get_params__post_init(struct adapter *); 365 static int set_params__post_init(struct adapter *); 366 static void t4_set_desc(struct adapter *); 367 static void build_medialist(struct port_info *, struct ifmedia *); 368 static int cxgbe_init_synchronized(struct port_info *); 369 static int cxgbe_uninit_synchronized(struct port_info *); 370 static int setup_intr_handlers(struct adapter *); 371 static void quiesce_eq(struct adapter *, struct sge_eq *); 372 static void quiesce_iq(struct adapter *, struct sge_iq *); 373 static void quiesce_fl(struct adapter *, struct sge_fl *); 374 static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 375 driver_intr_t *, void *, char *); 376 static int t4_free_irq(struct adapter *, struct irq *); 377 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 378 unsigned int); 379 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 380 static void cxgbe_tick(void *); 381 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 382 static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 383 struct mbuf *); 384 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 385 static int fw_msg_not_handled(struct adapter *, const __be64 *); 386 static int t4_sysctls(struct adapter *); 387 static int cxgbe_sysctls(struct port_info *); 388 static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 389 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 390 static int sysctl_btphy(SYSCTL_HANDLER_ARGS); 391 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); 392 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 393 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 394 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 395 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 396 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 397 static int sysctl_temperature(SYSCTL_HANDLER_ARGS); 398 #ifdef SBUF_DRAIN 399 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 400 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 401 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 402 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); 403 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); 404 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 405 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 406 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 407 static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 408 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 409 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 410 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 411 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); 412 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 413 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); 414 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 415 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 416 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 417 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 418 static int sysctl_tids(SYSCTL_HANDLER_ARGS); 419 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 420 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 421 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 422 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 423 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 424 #endif 425 static inline void txq_start(struct ifnet *, struct sge_txq *); 426 static uint32_t fconf_to_mode(uint32_t); 427 static uint32_t mode_to_fconf(uint32_t); 428 static uint32_t fspec_to_fconf(struct t4_filter_specification *); 429 static int get_filter_mode(struct adapter *, uint32_t *); 430 static int set_filter_mode(struct adapter *, uint32_t); 431 static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 432 static int get_filter(struct adapter *, struct t4_filter *); 433 static int set_filter(struct adapter *, struct t4_filter *); 434 static int del_filter(struct adapter *, struct t4_filter *); 435 static void clear_filter(struct filter_entry *); 436 static int set_filter_wr(struct adapter *, int); 437 static int del_filter_wr(struct adapter *, int); 438 static int get_sge_context(struct adapter *, struct t4_sge_context *); 439 static int load_fw(struct adapter *, struct t4_data *); 440 static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 441 static int read_i2c(struct adapter *, struct t4_i2c_data *); 442 static int set_sched_class(struct adapter *, struct t4_sched_params *); 443 static int set_sched_queue(struct adapter *, struct t4_sched_queue *); 444 #ifdef TCP_OFFLOAD 445 static int toe_capability(struct port_info *, int); 446 #endif 447 static int mod_event(module_t, int, void *); 448 449 struct { 450 uint16_t device; 451 char *desc; 452 } t4_pciids[] = { 453 {0xa000, "Chelsio Terminator 4 FPGA"}, 454 {0x4400, "Chelsio T440-dbg"}, 455 {0x4401, "Chelsio T420-CR"}, 456 {0x4402, "Chelsio T422-CR"}, 457 {0x4403, "Chelsio T440-CR"}, 458 {0x4404, "Chelsio T420-BCH"}, 459 {0x4405, "Chelsio T440-BCH"}, 460 {0x4406, "Chelsio T440-CH"}, 461 {0x4407, "Chelsio T420-SO"}, 462 {0x4408, "Chelsio T420-CX"}, 463 {0x4409, "Chelsio T420-BT"}, 464 {0x440a, "Chelsio T404-BT"}, 465 {0x440e, "Chelsio T440-LP-CR"}, 466 }, t5_pciids[] = { 467 {0xb000, "Chelsio Terminator 5 FPGA"}, 468 {0x5400, "Chelsio T580-dbg"}, 469 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ 470 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ 471 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ 472 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ 473 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ 474 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ 475 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ 476 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ 477 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 478 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ 479 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ 480 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ 481 #ifdef notyet 482 {0x5404, "Chelsio T520-BCH"}, 483 {0x5405, "Chelsio T540-BCH"}, 484 {0x5406, "Chelsio T540-CH"}, 485 {0x5408, "Chelsio T520-CX"}, 486 {0x540b, "Chelsio B520-SR"}, 487 {0x540c, "Chelsio B504-BT"}, 488 {0x540f, "Chelsio Amsterdam"}, 489 {0x5413, "Chelsio T580-CHR"}, 490 #endif 491 }; 492 493 #ifdef TCP_OFFLOAD 494 /* 495 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 496 * exactly the same for both rxq and ofld_rxq. 497 */ 498 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 499 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 500 #endif 501 502 /* No easy way to include t4_msg.h before adapter.h so we check this way */ 503 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 504 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 505 506 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); 507 508 static int 509 t4_probe(device_t dev) 510 { 511 int i; 512 uint16_t v = pci_get_vendor(dev); 513 uint16_t d = pci_get_device(dev); 514 uint8_t f = pci_get_function(dev); 515 516 if (v != PCI_VENDOR_ID_CHELSIO) 517 return (ENXIO); 518 519 /* Attach only to PF0 of the FPGA */ 520 if (d == 0xa000 && f != 0) 521 return (ENXIO); 522 523 for (i = 0; i < nitems(t4_pciids); i++) { 524 if (d == t4_pciids[i].device) { 525 device_set_desc(dev, t4_pciids[i].desc); 526 return (BUS_PROBE_DEFAULT); 527 } 528 } 529 530 return (ENXIO); 531 } 532 533 static int 534 t5_probe(device_t dev) 535 { 536 int i; 537 uint16_t v = pci_get_vendor(dev); 538 uint16_t d = pci_get_device(dev); 539 uint8_t f = pci_get_function(dev); 540 541 if (v != PCI_VENDOR_ID_CHELSIO) 542 return (ENXIO); 543 544 /* Attach only to PF0 of the FPGA */ 545 if (d == 0xb000 && f != 0) 546 return (ENXIO); 547 548 for (i = 0; i < nitems(t5_pciids); i++) { 549 if (d == t5_pciids[i].device) { 550 device_set_desc(dev, t5_pciids[i].desc); 551 return (BUS_PROBE_DEFAULT); 552 } 553 } 554 555 return (ENXIO); 556 } 557 558 static int 559 t4_attach(device_t dev) 560 { 561 struct adapter *sc; 562 int rc = 0, i, n10g, n1g, rqidx, tqidx; 563 struct intrs_and_queues iaq; 564 struct sge *s; 565 #ifdef TCP_OFFLOAD 566 int ofld_rqidx, ofld_tqidx; 567 #endif 568 #ifdef DEV_NETMAP 569 int nm_rqidx, nm_tqidx; 570 #endif 571 const char *pcie_ts; 572 573 sc = device_get_softc(dev); 574 sc->dev = dev; 575 576 pci_enable_busmaster(dev); 577 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 578 uint32_t v; 579 580 pci_set_max_read_req(dev, 4096); 581 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 582 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 583 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 584 } 585 586 sc->traceq = -1; 587 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); 588 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", 589 device_get_nameunit(dev)); 590 591 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 592 device_get_nameunit(dev)); 593 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 594 sx_xlock(&t4_list_lock); 595 SLIST_INSERT_HEAD(&t4_list, sc, link); 596 sx_xunlock(&t4_list_lock); 597 598 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 599 TAILQ_INIT(&sc->sfl); 600 callout_init(&sc->sfl_callout, CALLOUT_MPSAFE); 601 602 rc = map_bars_0_and_4(sc); 603 if (rc != 0) 604 goto done; /* error message displayed already */ 605 606 /* 607 * This is the real PF# to which we're attaching. Works from within PCI 608 * passthrough environments too, where pci_get_function() could return a 609 * different PF# depending on the passthrough configuration. We need to 610 * use the real PF# in all our communication with the firmware. 611 */ 612 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 613 sc->mbox = sc->pf; 614 615 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 616 sc->an_handler = an_not_handled; 617 for (i = 0; i < nitems(sc->cpl_handler); i++) 618 sc->cpl_handler[i] = cpl_not_handled; 619 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 620 sc->fw_msg_handler[i] = fw_msg_not_handled; 621 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 622 t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt); 623 t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt); 624 t4_init_sge_cpl_handlers(sc); 625 626 /* Prepare the adapter for operation */ 627 rc = -t4_prep_adapter(sc); 628 if (rc != 0) { 629 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 630 goto done; 631 } 632 633 /* 634 * Do this really early, with the memory windows set up even before the 635 * character device. The userland tool's register i/o and mem read 636 * will work even in "recovery mode". 637 */ 638 setup_memwin(sc); 639 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 640 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 641 device_get_nameunit(dev)); 642 if (sc->cdev == NULL) 643 device_printf(dev, "failed to create nexus char device.\n"); 644 else 645 sc->cdev->si_drv1 = sc; 646 647 /* Go no further if recovery mode has been requested. */ 648 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 649 device_printf(dev, "recovery mode.\n"); 650 goto done; 651 } 652 653 /* Prepare the firmware for operation */ 654 rc = prep_firmware(sc); 655 if (rc != 0) 656 goto done; /* error message displayed already */ 657 658 rc = get_params__post_init(sc); 659 if (rc != 0) 660 goto done; /* error message displayed already */ 661 662 rc = set_params__post_init(sc); 663 if (rc != 0) 664 goto done; /* error message displayed already */ 665 666 rc = map_bar_2(sc); 667 if (rc != 0) 668 goto done; /* error message displayed already */ 669 670 rc = t4_create_dma_tag(sc); 671 if (rc != 0) 672 goto done; /* error message displayed already */ 673 674 /* 675 * First pass over all the ports - allocate VIs and initialize some 676 * basic parameters like mac address, port type, etc. We also figure 677 * out whether a port is 10G or 1G and use that information when 678 * calculating how many interrupts to attempt to allocate. 679 */ 680 n10g = n1g = 0; 681 for_each_port(sc, i) { 682 struct port_info *pi; 683 684 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 685 sc->port[i] = pi; 686 687 /* These must be set before t4_port_init */ 688 pi->adapter = sc; 689 pi->port_id = i; 690 691 /* Allocate the vi and initialize parameters like mac addr */ 692 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 693 if (rc != 0) { 694 device_printf(dev, "unable to initialize port %d: %d\n", 695 i, rc); 696 free(pi, M_CXGBE); 697 sc->port[i] = NULL; 698 goto done; 699 } 700 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 701 if (rc != 0) { 702 device_printf(dev, "port %d l1cfg failed: %d\n", i, rc); 703 free(pi, M_CXGBE); 704 sc->port[i] = NULL; 705 goto done; 706 } 707 708 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 709 device_get_nameunit(dev), i); 710 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 711 sc->chan_map[pi->tx_chan] = i; 712 713 if (is_10G_port(pi) || is_40G_port(pi)) { 714 n10g++; 715 pi->tmr_idx = t4_tmr_idx_10g; 716 pi->pktc_idx = t4_pktc_idx_10g; 717 } else { 718 n1g++; 719 pi->tmr_idx = t4_tmr_idx_1g; 720 pi->pktc_idx = t4_pktc_idx_1g; 721 } 722 723 pi->xact_addr_filt = -1; 724 pi->linkdnrc = -1; 725 726 pi->qsize_rxq = t4_qsize_rxq; 727 pi->qsize_txq = t4_qsize_txq; 728 729 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 730 if (pi->dev == NULL) { 731 device_printf(dev, 732 "failed to add device for port %d.\n", i); 733 rc = ENXIO; 734 goto done; 735 } 736 device_set_softc(pi->dev, pi); 737 } 738 739 /* 740 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 741 */ 742 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 743 if (rc != 0) 744 goto done; /* error message displayed already */ 745 746 sc->intr_type = iaq.intr_type; 747 sc->intr_count = iaq.nirq; 748 749 s = &sc->sge; 750 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 751 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 752 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 753 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 754 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 755 #ifdef TCP_OFFLOAD 756 if (is_offload(sc)) { 757 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 758 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 759 s->neq += s->nofldtxq + s->nofldrxq; 760 s->niq += s->nofldrxq; 761 762 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 763 M_CXGBE, M_ZERO | M_WAITOK); 764 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 765 M_CXGBE, M_ZERO | M_WAITOK); 766 } 767 #endif 768 #ifdef DEV_NETMAP 769 s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g; 770 s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g; 771 s->neq += s->nnmtxq + s->nnmrxq; 772 s->niq += s->nnmrxq; 773 774 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq), 775 M_CXGBE, M_ZERO | M_WAITOK); 776 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq), 777 M_CXGBE, M_ZERO | M_WAITOK); 778 #endif 779 780 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 781 M_ZERO | M_WAITOK); 782 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 783 M_ZERO | M_WAITOK); 784 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 785 M_ZERO | M_WAITOK); 786 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 787 M_ZERO | M_WAITOK); 788 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 789 M_ZERO | M_WAITOK); 790 791 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 792 M_ZERO | M_WAITOK); 793 794 t4_init_l2t(sc, M_WAITOK); 795 796 /* 797 * Second pass over the ports. This time we know the number of rx and 798 * tx queues that each port should get. 799 */ 800 rqidx = tqidx = 0; 801 #ifdef TCP_OFFLOAD 802 ofld_rqidx = ofld_tqidx = 0; 803 #endif 804 #ifdef DEV_NETMAP 805 nm_rqidx = nm_tqidx = 0; 806 #endif 807 for_each_port(sc, i) { 808 struct port_info *pi = sc->port[i]; 809 810 if (pi == NULL) 811 continue; 812 813 pi->first_rxq = rqidx; 814 pi->first_txq = tqidx; 815 if (is_10G_port(pi) || is_40G_port(pi)) { 816 pi->flags |= iaq.intr_flags_10g; 817 pi->nrxq = iaq.nrxq10g; 818 pi->ntxq = iaq.ntxq10g; 819 } else { 820 pi->flags |= iaq.intr_flags_1g; 821 pi->nrxq = iaq.nrxq1g; 822 pi->ntxq = iaq.ntxq1g; 823 } 824 825 if (pi->ntxq > 1) 826 pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0; 827 else 828 pi->rsrv_noflowq = 0; 829 830 rqidx += pi->nrxq; 831 tqidx += pi->ntxq; 832 #ifdef TCP_OFFLOAD 833 if (is_offload(sc)) { 834 pi->first_ofld_rxq = ofld_rqidx; 835 pi->first_ofld_txq = ofld_tqidx; 836 if (is_10G_port(pi) || is_40G_port(pi)) { 837 pi->nofldrxq = iaq.nofldrxq10g; 838 pi->nofldtxq = iaq.nofldtxq10g; 839 } else { 840 pi->nofldrxq = iaq.nofldrxq1g; 841 pi->nofldtxq = iaq.nofldtxq1g; 842 } 843 ofld_rqidx += pi->nofldrxq; 844 ofld_tqidx += pi->nofldtxq; 845 } 846 #endif 847 #ifdef DEV_NETMAP 848 pi->first_nm_rxq = nm_rqidx; 849 pi->first_nm_txq = nm_tqidx; 850 if (is_10G_port(pi) || is_40G_port(pi)) { 851 pi->nnmrxq = iaq.nnmrxq10g; 852 pi->nnmtxq = iaq.nnmtxq10g; 853 } else { 854 pi->nnmrxq = iaq.nnmrxq1g; 855 pi->nnmtxq = iaq.nnmtxq1g; 856 } 857 nm_rqidx += pi->nnmrxq; 858 nm_tqidx += pi->nnmtxq; 859 #endif 860 } 861 862 rc = setup_intr_handlers(sc); 863 if (rc != 0) { 864 device_printf(dev, 865 "failed to setup interrupt handlers: %d\n", rc); 866 goto done; 867 } 868 869 rc = bus_generic_attach(dev); 870 if (rc != 0) { 871 device_printf(dev, 872 "failed to attach all child ports: %d\n", rc); 873 goto done; 874 } 875 876 switch (sc->params.pci.speed) { 877 case 0x1: 878 pcie_ts = "2.5"; 879 break; 880 case 0x2: 881 pcie_ts = "5.0"; 882 break; 883 case 0x3: 884 pcie_ts = "8.0"; 885 break; 886 default: 887 pcie_ts = "??"; 888 break; 889 } 890 device_printf(dev, 891 "PCIe x%d (%s GTS/s) (%d), %d ports, %d %s interrupt%s, %d eq, %d iq\n", 892 sc->params.pci.width, pcie_ts, sc->params.pci.speed, 893 sc->params.nports, sc->intr_count, 894 sc->intr_type == INTR_MSIX ? "MSI-X" : 895 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 896 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 897 898 t4_set_desc(sc); 899 900 done: 901 if (rc != 0 && sc->cdev) { 902 /* cdev was created and so cxgbetool works; recover that way. */ 903 device_printf(dev, 904 "error during attach, adapter is now in recovery mode.\n"); 905 rc = 0; 906 } 907 908 if (rc != 0) 909 t4_detach(dev); 910 else 911 t4_sysctls(sc); 912 913 return (rc); 914 } 915 916 /* 917 * Idempotent 918 */ 919 static int 920 t4_detach(device_t dev) 921 { 922 struct adapter *sc; 923 struct port_info *pi; 924 int i, rc; 925 926 sc = device_get_softc(dev); 927 928 if (sc->flags & FULL_INIT_DONE) 929 t4_intr_disable(sc); 930 931 if (sc->cdev) { 932 destroy_dev(sc->cdev); 933 sc->cdev = NULL; 934 } 935 936 rc = bus_generic_detach(dev); 937 if (rc) { 938 device_printf(dev, 939 "failed to detach child devices: %d\n", rc); 940 return (rc); 941 } 942 943 for (i = 0; i < sc->intr_count; i++) 944 t4_free_irq(sc, &sc->irq[i]); 945 946 for (i = 0; i < MAX_NPORTS; i++) { 947 pi = sc->port[i]; 948 if (pi) { 949 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid); 950 if (pi->dev) 951 device_delete_child(dev, pi->dev); 952 953 mtx_destroy(&pi->pi_lock); 954 free(pi, M_CXGBE); 955 } 956 } 957 958 if (sc->flags & FULL_INIT_DONE) 959 adapter_full_uninit(sc); 960 961 if (sc->flags & FW_OK) 962 t4_fw_bye(sc, sc->mbox); 963 964 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 965 pci_release_msi(dev); 966 967 if (sc->regs_res) 968 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 969 sc->regs_res); 970 971 if (sc->udbs_res) 972 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 973 sc->udbs_res); 974 975 if (sc->msix_res) 976 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 977 sc->msix_res); 978 979 if (sc->l2t) 980 t4_free_l2t(sc->l2t); 981 982 #ifdef TCP_OFFLOAD 983 free(sc->sge.ofld_rxq, M_CXGBE); 984 free(sc->sge.ofld_txq, M_CXGBE); 985 #endif 986 #ifdef DEV_NETMAP 987 free(sc->sge.nm_rxq, M_CXGBE); 988 free(sc->sge.nm_txq, M_CXGBE); 989 #endif 990 free(sc->irq, M_CXGBE); 991 free(sc->sge.rxq, M_CXGBE); 992 free(sc->sge.txq, M_CXGBE); 993 free(sc->sge.ctrlq, M_CXGBE); 994 free(sc->sge.iqmap, M_CXGBE); 995 free(sc->sge.eqmap, M_CXGBE); 996 free(sc->tids.ftid_tab, M_CXGBE); 997 t4_destroy_dma_tag(sc); 998 if (mtx_initialized(&sc->sc_lock)) { 999 sx_xlock(&t4_list_lock); 1000 SLIST_REMOVE(&t4_list, sc, adapter, link); 1001 sx_xunlock(&t4_list_lock); 1002 mtx_destroy(&sc->sc_lock); 1003 } 1004 1005 if (mtx_initialized(&sc->tids.ftid_lock)) 1006 mtx_destroy(&sc->tids.ftid_lock); 1007 if (mtx_initialized(&sc->sfl_lock)) 1008 mtx_destroy(&sc->sfl_lock); 1009 if (mtx_initialized(&sc->ifp_lock)) 1010 mtx_destroy(&sc->ifp_lock); 1011 1012 bzero(sc, sizeof(*sc)); 1013 1014 return (0); 1015 } 1016 1017 static int 1018 cxgbe_probe(device_t dev) 1019 { 1020 char buf[128]; 1021 struct port_info *pi = device_get_softc(dev); 1022 1023 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 1024 device_set_desc_copy(dev, buf); 1025 1026 return (BUS_PROBE_DEFAULT); 1027 } 1028 1029 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 1030 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 1031 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS) 1032 #define T4_CAP_ENABLE (T4_CAP) 1033 1034 static int 1035 cxgbe_attach(device_t dev) 1036 { 1037 struct port_info *pi = device_get_softc(dev); 1038 struct ifnet *ifp; 1039 char *s; 1040 int n, o; 1041 1042 /* Allocate an ifnet and set it up */ 1043 ifp = if_alloc(IFT_ETHER); 1044 if (ifp == NULL) { 1045 device_printf(dev, "Cannot allocate ifnet\n"); 1046 return (ENOMEM); 1047 } 1048 pi->ifp = ifp; 1049 ifp->if_softc = pi; 1050 1051 callout_init(&pi->tick, CALLOUT_MPSAFE); 1052 1053 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1054 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1055 1056 ifp->if_init = cxgbe_init; 1057 ifp->if_ioctl = cxgbe_ioctl; 1058 ifp->if_transmit = cxgbe_transmit; 1059 ifp->if_qflush = cxgbe_qflush; 1060 1061 ifp->if_capabilities = T4_CAP; 1062 #ifdef TCP_OFFLOAD 1063 if (is_offload(pi->adapter)) 1064 ifp->if_capabilities |= IFCAP_TOE; 1065 #endif 1066 ifp->if_capenable = T4_CAP_ENABLE; 1067 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1068 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1069 1070 /* Initialize ifmedia for this port */ 1071 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 1072 cxgbe_media_status); 1073 build_medialist(pi, &pi->media); 1074 1075 pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 1076 EVENTHANDLER_PRI_ANY); 1077 1078 ether_ifattach(ifp, pi->hw_addr); 1079 1080 n = 128; 1081 s = malloc(n, M_CXGBE, M_WAITOK); 1082 o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq); 1083 MPASS(n > o); 1084 #ifdef TCP_OFFLOAD 1085 if (is_offload(pi->adapter)) { 1086 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)", 1087 pi->nofldtxq, pi->nofldrxq); 1088 MPASS(n > o); 1089 } 1090 #endif 1091 #ifdef DEV_NETMAP 1092 o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq, 1093 pi->nnmrxq); 1094 MPASS(n > o); 1095 #endif 1096 device_printf(dev, "%s\n", s); 1097 free(s, M_CXGBE); 1098 1099 #ifdef DEV_NETMAP 1100 /* nm_media handled here to keep implementation private to this file */ 1101 ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change, 1102 cxgbe_media_status); 1103 build_medialist(pi, &pi->nm_media); 1104 create_netmap_ifnet(pi); /* logs errors it something fails */ 1105 #endif 1106 cxgbe_sysctls(pi); 1107 1108 return (0); 1109 } 1110 1111 static int 1112 cxgbe_detach(device_t dev) 1113 { 1114 struct port_info *pi = device_get_softc(dev); 1115 struct adapter *sc = pi->adapter; 1116 struct ifnet *ifp = pi->ifp; 1117 1118 /* Tell if_ioctl and if_init that the port is going away */ 1119 ADAPTER_LOCK(sc); 1120 SET_DOOMED(pi); 1121 wakeup(&sc->flags); 1122 while (IS_BUSY(sc)) 1123 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 1124 SET_BUSY(sc); 1125 #ifdef INVARIANTS 1126 sc->last_op = "t4detach"; 1127 sc->last_op_thr = curthread; 1128 #endif 1129 ADAPTER_UNLOCK(sc); 1130 1131 if (pi->flags & HAS_TRACEQ) { 1132 sc->traceq = -1; /* cloner should not create ifnet */ 1133 t4_tracer_port_detach(sc); 1134 } 1135 1136 if (pi->vlan_c) 1137 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c); 1138 1139 PORT_LOCK(pi); 1140 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1141 callout_stop(&pi->tick); 1142 PORT_UNLOCK(pi); 1143 callout_drain(&pi->tick); 1144 1145 /* Let detach proceed even if these fail. */ 1146 cxgbe_uninit_synchronized(pi); 1147 port_full_uninit(pi); 1148 1149 ifmedia_removeall(&pi->media); 1150 ether_ifdetach(pi->ifp); 1151 if_free(pi->ifp); 1152 1153 #ifdef DEV_NETMAP 1154 /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */ 1155 destroy_netmap_ifnet(pi); 1156 #endif 1157 1158 ADAPTER_LOCK(sc); 1159 CLR_BUSY(sc); 1160 wakeup(&sc->flags); 1161 ADAPTER_UNLOCK(sc); 1162 1163 return (0); 1164 } 1165 1166 static void 1167 cxgbe_init(void *arg) 1168 { 1169 struct port_info *pi = arg; 1170 struct adapter *sc = pi->adapter; 1171 1172 if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0) 1173 return; 1174 cxgbe_init_synchronized(pi); 1175 end_synchronized_op(sc, 0); 1176 } 1177 1178 static int 1179 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1180 { 1181 int rc = 0, mtu, flags, can_sleep; 1182 struct port_info *pi = ifp->if_softc; 1183 struct adapter *sc = pi->adapter; 1184 struct ifreq *ifr = (struct ifreq *)data; 1185 uint32_t mask; 1186 1187 switch (cmd) { 1188 case SIOCSIFMTU: 1189 mtu = ifr->ifr_mtu; 1190 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1191 return (EINVAL); 1192 1193 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu"); 1194 if (rc) 1195 return (rc); 1196 ifp->if_mtu = mtu; 1197 if (pi->flags & PORT_INIT_DONE) { 1198 t4_update_fl_bufsize(ifp); 1199 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1200 rc = update_mac_settings(ifp, XGMAC_MTU); 1201 } 1202 end_synchronized_op(sc, 0); 1203 break; 1204 1205 case SIOCSIFFLAGS: 1206 can_sleep = 0; 1207 redo_sifflags: 1208 rc = begin_synchronized_op(sc, pi, 1209 can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg"); 1210 if (rc) 1211 return (rc); 1212 1213 if (ifp->if_flags & IFF_UP) { 1214 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1215 flags = pi->if_flags; 1216 if ((ifp->if_flags ^ flags) & 1217 (IFF_PROMISC | IFF_ALLMULTI)) { 1218 if (can_sleep == 1) { 1219 end_synchronized_op(sc, 0); 1220 can_sleep = 0; 1221 goto redo_sifflags; 1222 } 1223 rc = update_mac_settings(ifp, 1224 XGMAC_PROMISC | XGMAC_ALLMULTI); 1225 } 1226 } else { 1227 if (can_sleep == 0) { 1228 end_synchronized_op(sc, LOCK_HELD); 1229 can_sleep = 1; 1230 goto redo_sifflags; 1231 } 1232 rc = cxgbe_init_synchronized(pi); 1233 } 1234 pi->if_flags = ifp->if_flags; 1235 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1236 if (can_sleep == 0) { 1237 end_synchronized_op(sc, LOCK_HELD); 1238 can_sleep = 1; 1239 goto redo_sifflags; 1240 } 1241 rc = cxgbe_uninit_synchronized(pi); 1242 } 1243 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD); 1244 break; 1245 1246 case SIOCADDMULTI: 1247 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1248 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi"); 1249 if (rc) 1250 return (rc); 1251 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1252 rc = update_mac_settings(ifp, XGMAC_MCADDRS); 1253 end_synchronized_op(sc, LOCK_HELD); 1254 break; 1255 1256 case SIOCSIFCAP: 1257 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap"); 1258 if (rc) 1259 return (rc); 1260 1261 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1262 if (mask & IFCAP_TXCSUM) { 1263 ifp->if_capenable ^= IFCAP_TXCSUM; 1264 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1265 1266 if (IFCAP_TSO4 & ifp->if_capenable && 1267 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1268 ifp->if_capenable &= ~IFCAP_TSO4; 1269 if_printf(ifp, 1270 "tso4 disabled due to -txcsum.\n"); 1271 } 1272 } 1273 if (mask & IFCAP_TXCSUM_IPV6) { 1274 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1275 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1276 1277 if (IFCAP_TSO6 & ifp->if_capenable && 1278 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1279 ifp->if_capenable &= ~IFCAP_TSO6; 1280 if_printf(ifp, 1281 "tso6 disabled due to -txcsum6.\n"); 1282 } 1283 } 1284 if (mask & IFCAP_RXCSUM) 1285 ifp->if_capenable ^= IFCAP_RXCSUM; 1286 if (mask & IFCAP_RXCSUM_IPV6) 1287 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1288 1289 /* 1290 * Note that we leave CSUM_TSO alone (it is always set). The 1291 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1292 * sending a TSO request our way, so it's sufficient to toggle 1293 * IFCAP_TSOx only. 1294 */ 1295 if (mask & IFCAP_TSO4) { 1296 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1297 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1298 if_printf(ifp, "enable txcsum first.\n"); 1299 rc = EAGAIN; 1300 goto fail; 1301 } 1302 ifp->if_capenable ^= IFCAP_TSO4; 1303 } 1304 if (mask & IFCAP_TSO6) { 1305 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1306 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1307 if_printf(ifp, "enable txcsum6 first.\n"); 1308 rc = EAGAIN; 1309 goto fail; 1310 } 1311 ifp->if_capenable ^= IFCAP_TSO6; 1312 } 1313 if (mask & IFCAP_LRO) { 1314 #if defined(INET) || defined(INET6) 1315 int i; 1316 struct sge_rxq *rxq; 1317 1318 ifp->if_capenable ^= IFCAP_LRO; 1319 for_each_rxq(pi, i, rxq) { 1320 if (ifp->if_capenable & IFCAP_LRO) 1321 rxq->iq.flags |= IQ_LRO_ENABLED; 1322 else 1323 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1324 } 1325 #endif 1326 } 1327 #ifdef TCP_OFFLOAD 1328 if (mask & IFCAP_TOE) { 1329 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1330 1331 rc = toe_capability(pi, enable); 1332 if (rc != 0) 1333 goto fail; 1334 1335 ifp->if_capenable ^= mask; 1336 } 1337 #endif 1338 if (mask & IFCAP_VLAN_HWTAGGING) { 1339 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1340 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1341 rc = update_mac_settings(ifp, XGMAC_VLANEX); 1342 } 1343 if (mask & IFCAP_VLAN_MTU) { 1344 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1345 1346 /* Need to find out how to disable auto-mtu-inflation */ 1347 } 1348 if (mask & IFCAP_VLAN_HWTSO) 1349 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1350 if (mask & IFCAP_VLAN_HWCSUM) 1351 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1352 1353 #ifdef VLAN_CAPABILITIES 1354 VLAN_CAPABILITIES(ifp); 1355 #endif 1356 fail: 1357 end_synchronized_op(sc, 0); 1358 break; 1359 1360 case SIOCSIFMEDIA: 1361 case SIOCGIFMEDIA: 1362 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1363 break; 1364 1365 default: 1366 rc = ether_ioctl(ifp, cmd, data); 1367 } 1368 1369 return (rc); 1370 } 1371 1372 static int 1373 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1374 { 1375 struct port_info *pi = ifp->if_softc; 1376 struct adapter *sc = pi->adapter; 1377 struct sge_txq *txq = &sc->sge.txq[pi->first_txq]; 1378 struct buf_ring *br; 1379 int rc; 1380 1381 M_ASSERTPKTHDR(m); 1382 1383 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1384 m_freem(m); 1385 return (ENETDOWN); 1386 } 1387 1388 if (m->m_flags & M_FLOWID) 1389 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) 1390 + pi->rsrv_noflowq); 1391 br = txq->br; 1392 1393 if (TXQ_TRYLOCK(txq) == 0) { 1394 struct sge_eq *eq = &txq->eq; 1395 1396 /* 1397 * It is possible that t4_eth_tx finishes up and releases the 1398 * lock between the TRYLOCK above and the drbr_enqueue here. We 1399 * need to make sure that this mbuf doesn't just sit there in 1400 * the drbr. 1401 */ 1402 1403 rc = drbr_enqueue(ifp, br, m); 1404 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 && 1405 !(eq->flags & EQ_DOOMED)) 1406 callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq); 1407 return (rc); 1408 } 1409 1410 /* 1411 * txq->m is the mbuf that is held up due to a temporary shortage of 1412 * resources and it should be put on the wire first. Then what's in 1413 * drbr and finally the mbuf that was just passed in to us. 1414 * 1415 * Return code should indicate the fate of the mbuf that was passed in 1416 * this time. 1417 */ 1418 1419 TXQ_LOCK_ASSERT_OWNED(txq); 1420 if (drbr_needs_enqueue(ifp, br) || txq->m) { 1421 1422 /* Queued for transmission. */ 1423 1424 rc = drbr_enqueue(ifp, br, m); 1425 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 1426 (void) t4_eth_tx(ifp, txq, m); 1427 TXQ_UNLOCK(txq); 1428 return (rc); 1429 } 1430 1431 /* Direct transmission. */ 1432 rc = t4_eth_tx(ifp, txq, m); 1433 if (rc != 0 && txq->m) 1434 rc = 0; /* held, will be transmitted soon (hopefully) */ 1435 1436 TXQ_UNLOCK(txq); 1437 return (rc); 1438 } 1439 1440 static void 1441 cxgbe_qflush(struct ifnet *ifp) 1442 { 1443 struct port_info *pi = ifp->if_softc; 1444 struct sge_txq *txq; 1445 int i; 1446 struct mbuf *m; 1447 1448 /* queues do not exist if !PORT_INIT_DONE. */ 1449 if (pi->flags & PORT_INIT_DONE) { 1450 for_each_txq(pi, i, txq) { 1451 TXQ_LOCK(txq); 1452 m_freem(txq->m); 1453 txq->m = NULL; 1454 while ((m = buf_ring_dequeue_sc(txq->br)) != NULL) 1455 m_freem(m); 1456 TXQ_UNLOCK(txq); 1457 } 1458 } 1459 if_qflush(ifp); 1460 } 1461 1462 static int 1463 cxgbe_media_change(struct ifnet *ifp) 1464 { 1465 struct port_info *pi = ifp->if_softc; 1466 1467 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1468 1469 return (EOPNOTSUPP); 1470 } 1471 1472 static void 1473 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1474 { 1475 struct port_info *pi = ifp->if_softc; 1476 struct ifmedia *media = NULL; 1477 struct ifmedia_entry *cur; 1478 int speed = pi->link_cfg.speed; 1479 int data = (pi->port_type << 8) | pi->mod_type; 1480 1481 if (ifp == pi->ifp) 1482 media = &pi->media; 1483 #ifdef DEV_NETMAP 1484 else if (ifp == pi->nm_ifp) 1485 media = &pi->nm_media; 1486 #endif 1487 MPASS(media != NULL); 1488 1489 cur = media->ifm_cur; 1490 if (cur->ifm_data != data) { 1491 build_medialist(pi, media); 1492 cur = media->ifm_cur; 1493 } 1494 1495 ifmr->ifm_status = IFM_AVALID; 1496 if (!pi->link_cfg.link_ok) 1497 return; 1498 1499 ifmr->ifm_status |= IFM_ACTIVE; 1500 1501 /* active and current will differ iff current media is autoselect. */ 1502 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1503 return; 1504 1505 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1506 if (speed == SPEED_10000) 1507 ifmr->ifm_active |= IFM_10G_T; 1508 else if (speed == SPEED_1000) 1509 ifmr->ifm_active |= IFM_1000_T; 1510 else if (speed == SPEED_100) 1511 ifmr->ifm_active |= IFM_100_TX; 1512 else if (speed == SPEED_10) 1513 ifmr->ifm_active |= IFM_10_T; 1514 else 1515 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1516 speed)); 1517 } 1518 1519 void 1520 t4_fatal_err(struct adapter *sc) 1521 { 1522 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1523 t4_intr_disable(sc); 1524 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1525 device_get_nameunit(sc->dev)); 1526 } 1527 1528 static int 1529 map_bars_0_and_4(struct adapter *sc) 1530 { 1531 sc->regs_rid = PCIR_BAR(0); 1532 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1533 &sc->regs_rid, RF_ACTIVE); 1534 if (sc->regs_res == NULL) { 1535 device_printf(sc->dev, "cannot map registers.\n"); 1536 return (ENXIO); 1537 } 1538 sc->bt = rman_get_bustag(sc->regs_res); 1539 sc->bh = rman_get_bushandle(sc->regs_res); 1540 sc->mmio_len = rman_get_size(sc->regs_res); 1541 setbit(&sc->doorbells, DOORBELL_KDB); 1542 1543 sc->msix_rid = PCIR_BAR(4); 1544 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1545 &sc->msix_rid, RF_ACTIVE); 1546 if (sc->msix_res == NULL) { 1547 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1548 return (ENXIO); 1549 } 1550 1551 return (0); 1552 } 1553 1554 static int 1555 map_bar_2(struct adapter *sc) 1556 { 1557 1558 /* 1559 * T4: only iWARP driver uses the userspace doorbells. There is no need 1560 * to map it if RDMA is disabled. 1561 */ 1562 if (is_t4(sc) && sc->rdmacaps == 0) 1563 return (0); 1564 1565 sc->udbs_rid = PCIR_BAR(2); 1566 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1567 &sc->udbs_rid, RF_ACTIVE); 1568 if (sc->udbs_res == NULL) { 1569 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1570 return (ENXIO); 1571 } 1572 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1573 1574 if (is_t5(sc)) { 1575 setbit(&sc->doorbells, DOORBELL_UDB); 1576 #if defined(__i386__) || defined(__amd64__) 1577 if (t5_write_combine) { 1578 int rc; 1579 1580 /* 1581 * Enable write combining on BAR2. This is the 1582 * userspace doorbell BAR and is split into 128B 1583 * (UDBS_SEG_SIZE) doorbell regions, each associated 1584 * with an egress queue. The first 64B has the doorbell 1585 * and the second 64B can be used to submit a tx work 1586 * request with an implicit doorbell. 1587 */ 1588 1589 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1590 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1591 if (rc == 0) { 1592 clrbit(&sc->doorbells, DOORBELL_UDB); 1593 setbit(&sc->doorbells, DOORBELL_WCWR); 1594 setbit(&sc->doorbells, DOORBELL_UDBWC); 1595 } else { 1596 device_printf(sc->dev, 1597 "couldn't enable write combining: %d\n", 1598 rc); 1599 } 1600 1601 t4_write_reg(sc, A_SGE_STAT_CFG, 1602 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1603 } 1604 #endif 1605 } 1606 1607 return (0); 1608 } 1609 1610 static const struct memwin t4_memwin[] = { 1611 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1612 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1613 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1614 }; 1615 1616 static const struct memwin t5_memwin[] = { 1617 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1618 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1619 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1620 }; 1621 1622 static void 1623 setup_memwin(struct adapter *sc) 1624 { 1625 const struct memwin *mw; 1626 int i, n; 1627 uint32_t bar0; 1628 1629 if (is_t4(sc)) { 1630 /* 1631 * Read low 32b of bar0 indirectly via the hardware backdoor 1632 * mechanism. Works from within PCI passthrough environments 1633 * too, where rman_get_start() can return a different value. We 1634 * need to program the T4 memory window decoders with the actual 1635 * addresses that will be coming across the PCIe link. 1636 */ 1637 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1638 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1639 1640 mw = &t4_memwin[0]; 1641 n = nitems(t4_memwin); 1642 } else { 1643 /* T5 uses the relative offset inside the PCIe BAR */ 1644 bar0 = 0; 1645 1646 mw = &t5_memwin[0]; 1647 n = nitems(t5_memwin); 1648 } 1649 1650 for (i = 0; i < n; i++, mw++) { 1651 t4_write_reg(sc, 1652 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 1653 (mw->base + bar0) | V_BIR(0) | 1654 V_WINDOW(ilog2(mw->aperture) - 10)); 1655 } 1656 1657 /* flush */ 1658 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1659 } 1660 1661 /* 1662 * Verify that the memory range specified by the addr/len pair is valid and lies 1663 * entirely within a single region (EDCx or MCx). 1664 */ 1665 static int 1666 validate_mem_range(struct adapter *sc, uint32_t addr, int len) 1667 { 1668 uint32_t em, addr_len, maddr, mlen; 1669 1670 /* Memory can only be accessed in naturally aligned 4 byte units */ 1671 if (addr & 3 || len & 3 || len == 0) 1672 return (EINVAL); 1673 1674 /* Enabled memories */ 1675 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1676 if (em & F_EDRAM0_ENABLE) { 1677 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1678 maddr = G_EDRAM0_BASE(addr_len) << 20; 1679 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1680 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1681 addr + len <= maddr + mlen) 1682 return (0); 1683 } 1684 if (em & F_EDRAM1_ENABLE) { 1685 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1686 maddr = G_EDRAM1_BASE(addr_len) << 20; 1687 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1688 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1689 addr + len <= maddr + mlen) 1690 return (0); 1691 } 1692 if (em & F_EXT_MEM_ENABLE) { 1693 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1694 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1695 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1696 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1697 addr + len <= maddr + mlen) 1698 return (0); 1699 } 1700 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 1701 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1702 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1703 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1704 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1705 addr + len <= maddr + mlen) 1706 return (0); 1707 } 1708 1709 return (EFAULT); 1710 } 1711 1712 static int 1713 fwmtype_to_hwmtype(int mtype) 1714 { 1715 1716 switch (mtype) { 1717 case FW_MEMTYPE_EDC0: 1718 return (MEM_EDC0); 1719 case FW_MEMTYPE_EDC1: 1720 return (MEM_EDC1); 1721 case FW_MEMTYPE_EXTMEM: 1722 return (MEM_MC0); 1723 case FW_MEMTYPE_EXTMEM1: 1724 return (MEM_MC1); 1725 default: 1726 panic("%s: cannot translate fw mtype %d.", __func__, mtype); 1727 } 1728 } 1729 1730 /* 1731 * Verify that the memory range specified by the memtype/offset/len pair is 1732 * valid and lies entirely within the memtype specified. The global address of 1733 * the start of the range is returned in addr. 1734 */ 1735 static int 1736 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 1737 uint32_t *addr) 1738 { 1739 uint32_t em, addr_len, maddr, mlen; 1740 1741 /* Memory can only be accessed in naturally aligned 4 byte units */ 1742 if (off & 3 || len & 3 || len == 0) 1743 return (EINVAL); 1744 1745 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1746 switch (fwmtype_to_hwmtype(mtype)) { 1747 case MEM_EDC0: 1748 if (!(em & F_EDRAM0_ENABLE)) 1749 return (EINVAL); 1750 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1751 maddr = G_EDRAM0_BASE(addr_len) << 20; 1752 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1753 break; 1754 case MEM_EDC1: 1755 if (!(em & F_EDRAM1_ENABLE)) 1756 return (EINVAL); 1757 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1758 maddr = G_EDRAM1_BASE(addr_len) << 20; 1759 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1760 break; 1761 case MEM_MC: 1762 if (!(em & F_EXT_MEM_ENABLE)) 1763 return (EINVAL); 1764 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1765 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1766 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1767 break; 1768 case MEM_MC1: 1769 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 1770 return (EINVAL); 1771 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1772 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1773 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1774 break; 1775 default: 1776 return (EINVAL); 1777 } 1778 1779 if (mlen > 0 && off < mlen && off + len <= mlen) { 1780 *addr = maddr + off; /* global address */ 1781 return (0); 1782 } 1783 1784 return (EFAULT); 1785 } 1786 1787 static void 1788 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 1789 { 1790 const struct memwin *mw; 1791 1792 if (is_t4(sc)) { 1793 KASSERT(win >= 0 && win < nitems(t4_memwin), 1794 ("%s: incorrect memwin# (%d)", __func__, win)); 1795 mw = &t4_memwin[win]; 1796 } else { 1797 KASSERT(win >= 0 && win < nitems(t5_memwin), 1798 ("%s: incorrect memwin# (%d)", __func__, win)); 1799 mw = &t5_memwin[win]; 1800 } 1801 1802 if (base != NULL) 1803 *base = mw->base; 1804 if (aperture != NULL) 1805 *aperture = mw->aperture; 1806 } 1807 1808 /* 1809 * Positions the memory window such that it can be used to access the specified 1810 * address in the chip's address space. The return value is the offset of addr 1811 * from the start of the window. 1812 */ 1813 static uint32_t 1814 position_memwin(struct adapter *sc, int n, uint32_t addr) 1815 { 1816 uint32_t start, pf; 1817 uint32_t reg; 1818 1819 KASSERT(n >= 0 && n <= 3, 1820 ("%s: invalid window %d.", __func__, n)); 1821 KASSERT((addr & 3) == 0, 1822 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 1823 1824 if (is_t4(sc)) { 1825 pf = 0; 1826 start = addr & ~0xf; /* start must be 16B aligned */ 1827 } else { 1828 pf = V_PFNUM(sc->pf); 1829 start = addr & ~0x7f; /* start must be 128B aligned */ 1830 } 1831 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 1832 1833 t4_write_reg(sc, reg, start | pf); 1834 t4_read_reg(sc, reg); 1835 1836 return (addr - start); 1837 } 1838 1839 static int 1840 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1841 struct intrs_and_queues *iaq) 1842 { 1843 int rc, itype, navail, nrxq10g, nrxq1g, n; 1844 int nofldrxq10g = 0, nofldrxq1g = 0; 1845 int nnmrxq10g = 0, nnmrxq1g = 0; 1846 1847 bzero(iaq, sizeof(*iaq)); 1848 1849 iaq->ntxq10g = t4_ntxq10g; 1850 iaq->ntxq1g = t4_ntxq1g; 1851 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1852 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1853 iaq->rsrv_noflowq = t4_rsrv_noflowq; 1854 #ifdef TCP_OFFLOAD 1855 if (is_offload(sc)) { 1856 iaq->nofldtxq10g = t4_nofldtxq10g; 1857 iaq->nofldtxq1g = t4_nofldtxq1g; 1858 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1859 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1860 } 1861 #endif 1862 #ifdef DEV_NETMAP 1863 iaq->nnmtxq10g = t4_nnmtxq10g; 1864 iaq->nnmtxq1g = t4_nnmtxq1g; 1865 iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g; 1866 iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g; 1867 #endif 1868 1869 for (itype = INTR_MSIX; itype; itype >>= 1) { 1870 1871 if ((itype & t4_intr_types) == 0) 1872 continue; /* not allowed */ 1873 1874 if (itype == INTR_MSIX) 1875 navail = pci_msix_count(sc->dev); 1876 else if (itype == INTR_MSI) 1877 navail = pci_msi_count(sc->dev); 1878 else 1879 navail = 1; 1880 restart: 1881 if (navail == 0) 1882 continue; 1883 1884 iaq->intr_type = itype; 1885 iaq->intr_flags_10g = 0; 1886 iaq->intr_flags_1g = 0; 1887 1888 /* 1889 * Best option: an interrupt vector for errors, one for the 1890 * firmware event queue, and one for every rxq (NIC, TOE, and 1891 * netmap). 1892 */ 1893 iaq->nirq = T4_EXTRA_INTR; 1894 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g); 1895 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g); 1896 if (iaq->nirq <= navail && 1897 (itype != INTR_MSI || powerof2(iaq->nirq))) { 1898 iaq->intr_flags_10g = INTR_ALL; 1899 iaq->intr_flags_1g = INTR_ALL; 1900 goto allocate; 1901 } 1902 1903 /* 1904 * Second best option: a vector for errors, one for the firmware 1905 * event queue, and vectors for either all the NIC rx queues or 1906 * all the TOE rx queues. The queues that don't get vectors 1907 * will forward their interrupts to those that do. 1908 * 1909 * Note: netmap rx queues cannot be created early and so they 1910 * can't be setup to receive forwarded interrupts for others. 1911 */ 1912 iaq->nirq = T4_EXTRA_INTR; 1913 if (nrxq10g >= nofldrxq10g) { 1914 iaq->intr_flags_10g = INTR_RXQ; 1915 iaq->nirq += n10g * nrxq10g; 1916 #ifdef DEV_NETMAP 1917 iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g); 1918 #endif 1919 } else { 1920 iaq->intr_flags_10g = INTR_OFLD_RXQ; 1921 iaq->nirq += n10g * nofldrxq10g; 1922 #ifdef DEV_NETMAP 1923 iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g); 1924 #endif 1925 } 1926 if (nrxq1g >= nofldrxq1g) { 1927 iaq->intr_flags_1g = INTR_RXQ; 1928 iaq->nirq += n1g * nrxq1g; 1929 #ifdef DEV_NETMAP 1930 iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g); 1931 #endif 1932 } else { 1933 iaq->intr_flags_1g = INTR_OFLD_RXQ; 1934 iaq->nirq += n1g * nofldrxq1g; 1935 #ifdef DEV_NETMAP 1936 iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g); 1937 #endif 1938 } 1939 if (iaq->nirq <= navail && 1940 (itype != INTR_MSI || powerof2(iaq->nirq))) 1941 goto allocate; 1942 1943 /* 1944 * Next best option: an interrupt vector for errors, one for the 1945 * firmware event queue, and at least one per port. At this 1946 * point we know we'll have to downsize nrxq and/or nofldrxq 1947 * and/or nnmrxq to fit what's available to us. 1948 */ 1949 iaq->nirq = T4_EXTRA_INTR; 1950 iaq->nirq += n10g + n1g; 1951 if (iaq->nirq <= navail) { 1952 int leftover = navail - iaq->nirq; 1953 1954 if (n10g > 0) { 1955 int target = max(nrxq10g, nofldrxq10g); 1956 1957 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ? 1958 INTR_RXQ : INTR_OFLD_RXQ; 1959 1960 n = 1; 1961 while (n < target && leftover >= n10g) { 1962 leftover -= n10g; 1963 iaq->nirq += n10g; 1964 n++; 1965 } 1966 iaq->nrxq10g = min(n, nrxq10g); 1967 #ifdef TCP_OFFLOAD 1968 iaq->nofldrxq10g = min(n, nofldrxq10g); 1969 #endif 1970 #ifdef DEV_NETMAP 1971 iaq->nnmrxq10g = min(n, nnmrxq10g); 1972 #endif 1973 } 1974 1975 if (n1g > 0) { 1976 int target = max(nrxq1g, nofldrxq1g); 1977 1978 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ? 1979 INTR_RXQ : INTR_OFLD_RXQ; 1980 1981 n = 1; 1982 while (n < target && leftover >= n1g) { 1983 leftover -= n1g; 1984 iaq->nirq += n1g; 1985 n++; 1986 } 1987 iaq->nrxq1g = min(n, nrxq1g); 1988 #ifdef TCP_OFFLOAD 1989 iaq->nofldrxq1g = min(n, nofldrxq1g); 1990 #endif 1991 #ifdef DEV_NETMAP 1992 iaq->nnmrxq1g = min(n, nnmrxq1g); 1993 #endif 1994 } 1995 1996 if (itype != INTR_MSI || powerof2(iaq->nirq)) 1997 goto allocate; 1998 } 1999 2000 /* 2001 * Least desirable option: one interrupt vector for everything. 2002 */ 2003 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 2004 iaq->intr_flags_10g = iaq->intr_flags_1g = 0; 2005 #ifdef TCP_OFFLOAD 2006 if (is_offload(sc)) 2007 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 2008 #endif 2009 #ifdef DEV_NETMAP 2010 iaq->nnmrxq10g = iaq->nnmrxq1g = 1; 2011 #endif 2012 2013 allocate: 2014 navail = iaq->nirq; 2015 rc = 0; 2016 if (itype == INTR_MSIX) 2017 rc = pci_alloc_msix(sc->dev, &navail); 2018 else if (itype == INTR_MSI) 2019 rc = pci_alloc_msi(sc->dev, &navail); 2020 2021 if (rc == 0) { 2022 if (navail == iaq->nirq) 2023 return (0); 2024 2025 /* 2026 * Didn't get the number requested. Use whatever number 2027 * the kernel is willing to allocate (it's in navail). 2028 */ 2029 device_printf(sc->dev, "fewer vectors than requested, " 2030 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 2031 itype, iaq->nirq, navail); 2032 pci_release_msi(sc->dev); 2033 goto restart; 2034 } 2035 2036 device_printf(sc->dev, 2037 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 2038 itype, rc, iaq->nirq, navail); 2039 } 2040 2041 device_printf(sc->dev, 2042 "failed to find a usable interrupt type. " 2043 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 2044 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 2045 2046 return (ENXIO); 2047 } 2048 2049 #define FW_VERSION(chip) ( \ 2050 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ 2051 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ 2052 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ 2053 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) 2054 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) 2055 2056 struct fw_info { 2057 uint8_t chip; 2058 char *kld_name; 2059 char *fw_mod_name; 2060 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 2061 } fw_info[] = { 2062 { 2063 .chip = CHELSIO_T4, 2064 .kld_name = "t4fw_cfg", 2065 .fw_mod_name = "t4fw", 2066 .fw_hdr = { 2067 .chip = FW_HDR_CHIP_T4, 2068 .fw_ver = htobe32_const(FW_VERSION(T4)), 2069 .intfver_nic = FW_INTFVER(T4, NIC), 2070 .intfver_vnic = FW_INTFVER(T4, VNIC), 2071 .intfver_ofld = FW_INTFVER(T4, OFLD), 2072 .intfver_ri = FW_INTFVER(T4, RI), 2073 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 2074 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 2075 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 2076 .intfver_fcoe = FW_INTFVER(T4, FCOE), 2077 }, 2078 }, { 2079 .chip = CHELSIO_T5, 2080 .kld_name = "t5fw_cfg", 2081 .fw_mod_name = "t5fw", 2082 .fw_hdr = { 2083 .chip = FW_HDR_CHIP_T5, 2084 .fw_ver = htobe32_const(FW_VERSION(T5)), 2085 .intfver_nic = FW_INTFVER(T5, NIC), 2086 .intfver_vnic = FW_INTFVER(T5, VNIC), 2087 .intfver_ofld = FW_INTFVER(T5, OFLD), 2088 .intfver_ri = FW_INTFVER(T5, RI), 2089 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 2090 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 2091 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 2092 .intfver_fcoe = FW_INTFVER(T5, FCOE), 2093 }, 2094 } 2095 }; 2096 2097 static struct fw_info * 2098 find_fw_info(int chip) 2099 { 2100 int i; 2101 2102 for (i = 0; i < nitems(fw_info); i++) { 2103 if (fw_info[i].chip == chip) 2104 return (&fw_info[i]); 2105 } 2106 return (NULL); 2107 } 2108 2109 /* 2110 * Is the given firmware API compatible with the one the driver was compiled 2111 * with? 2112 */ 2113 static int 2114 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 2115 { 2116 2117 /* short circuit if it's the exact same firmware version */ 2118 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 2119 return (1); 2120 2121 /* 2122 * XXX: Is this too conservative? Perhaps I should limit this to the 2123 * features that are supported in the driver. 2124 */ 2125 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 2126 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 2127 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 2128 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 2129 return (1); 2130 #undef SAME_INTF 2131 2132 return (0); 2133 } 2134 2135 /* 2136 * The firmware in the KLD is usable, but should it be installed? This routine 2137 * explains itself in detail if it indicates the KLD firmware should be 2138 * installed. 2139 */ 2140 static int 2141 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 2142 { 2143 const char *reason; 2144 2145 if (!card_fw_usable) { 2146 reason = "incompatible or unusable"; 2147 goto install; 2148 } 2149 2150 if (k > c) { 2151 reason = "older than the version bundled with this driver"; 2152 goto install; 2153 } 2154 2155 if (t4_fw_install == 2 && k != c) { 2156 reason = "different than the version bundled with this driver"; 2157 goto install; 2158 } 2159 2160 return (0); 2161 2162 install: 2163 if (t4_fw_install == 0) { 2164 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2165 "but the driver is prohibited from installing a different " 2166 "firmware on the card.\n", 2167 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2168 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 2169 2170 return (0); 2171 } 2172 2173 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2174 "installing firmware %u.%u.%u.%u on card.\n", 2175 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2176 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 2177 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2178 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2179 2180 return (1); 2181 } 2182 /* 2183 * Establish contact with the firmware and determine if we are the master driver 2184 * or not, and whether we are responsible for chip initialization. 2185 */ 2186 static int 2187 prep_firmware(struct adapter *sc) 2188 { 2189 const struct firmware *fw = NULL, *default_cfg; 2190 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 2191 enum dev_state state; 2192 struct fw_info *fw_info; 2193 struct fw_hdr *card_fw; /* fw on the card */ 2194 const struct fw_hdr *kld_fw; /* fw in the KLD */ 2195 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 2196 against */ 2197 2198 /* Contact firmware. */ 2199 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 2200 if (rc < 0 || state == DEV_STATE_ERR) { 2201 rc = -rc; 2202 device_printf(sc->dev, 2203 "failed to connect to the firmware: %d, %d.\n", rc, state); 2204 return (rc); 2205 } 2206 pf = rc; 2207 if (pf == sc->mbox) 2208 sc->flags |= MASTER_PF; 2209 else if (state == DEV_STATE_UNINIT) { 2210 /* 2211 * We didn't get to be the master so we definitely won't be 2212 * configuring the chip. It's a bug if someone else hasn't 2213 * configured it already. 2214 */ 2215 device_printf(sc->dev, "couldn't be master(%d), " 2216 "device not already initialized either(%d).\n", rc, state); 2217 return (EDOOFUS); 2218 } 2219 2220 /* This is the firmware whose headers the driver was compiled against */ 2221 fw_info = find_fw_info(chip_id(sc)); 2222 if (fw_info == NULL) { 2223 device_printf(sc->dev, 2224 "unable to look up firmware information for chip %d.\n", 2225 chip_id(sc)); 2226 return (EINVAL); 2227 } 2228 drv_fw = &fw_info->fw_hdr; 2229 2230 /* 2231 * The firmware KLD contains many modules. The KLD name is also the 2232 * name of the module that contains the default config file. 2233 */ 2234 default_cfg = firmware_get(fw_info->kld_name); 2235 2236 /* Read the header of the firmware on the card */ 2237 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 2238 rc = -t4_read_flash(sc, FLASH_FW_START, 2239 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 2240 if (rc == 0) 2241 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 2242 else { 2243 device_printf(sc->dev, 2244 "Unable to read card's firmware header: %d\n", rc); 2245 card_fw_usable = 0; 2246 } 2247 2248 /* This is the firmware in the KLD */ 2249 fw = firmware_get(fw_info->fw_mod_name); 2250 if (fw != NULL) { 2251 kld_fw = (const void *)fw->data; 2252 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 2253 } else { 2254 kld_fw = NULL; 2255 kld_fw_usable = 0; 2256 } 2257 2258 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2259 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) { 2260 /* 2261 * Common case: the firmware on the card is an exact match and 2262 * the KLD is an exact match too, or the KLD is 2263 * absent/incompatible. Note that t4_fw_install = 2 is ignored 2264 * here -- use cxgbetool loadfw if you want to reinstall the 2265 * same firmware as the one on the card. 2266 */ 2267 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2268 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2269 be32toh(card_fw->fw_ver))) { 2270 2271 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 2272 if (rc != 0) { 2273 device_printf(sc->dev, 2274 "failed to install firmware: %d\n", rc); 2275 goto done; 2276 } 2277 2278 /* Installed successfully, update the cached header too. */ 2279 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2280 card_fw_usable = 1; 2281 need_fw_reset = 0; /* already reset as part of load_fw */ 2282 } 2283 2284 if (!card_fw_usable) { 2285 uint32_t d, c, k; 2286 2287 d = ntohl(drv_fw->fw_ver); 2288 c = ntohl(card_fw->fw_ver); 2289 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2290 2291 device_printf(sc->dev, "Cannot find a usable firmware: " 2292 "fw_install %d, chip state %d, " 2293 "driver compiled with %d.%d.%d.%d, " 2294 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2295 t4_fw_install, state, 2296 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2297 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2298 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2299 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2300 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2301 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2302 rc = EINVAL; 2303 goto done; 2304 } 2305 2306 /* We're using whatever's on the card and it's known to be good. */ 2307 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2308 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2309 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2310 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2311 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2312 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2313 t4_get_tp_version(sc, &sc->params.tp_vers); 2314 2315 /* Reset device */ 2316 if (need_fw_reset && 2317 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2318 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2319 if (rc != ETIMEDOUT && rc != EIO) 2320 t4_fw_bye(sc, sc->mbox); 2321 goto done; 2322 } 2323 sc->flags |= FW_OK; 2324 2325 rc = get_params__pre_init(sc); 2326 if (rc != 0) 2327 goto done; /* error message displayed already */ 2328 2329 /* Partition adapter resources as specified in the config file. */ 2330 if (state == DEV_STATE_UNINIT) { 2331 2332 KASSERT(sc->flags & MASTER_PF, 2333 ("%s: trying to change chip settings when not master.", 2334 __func__)); 2335 2336 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2337 if (rc != 0) 2338 goto done; /* error message displayed already */ 2339 2340 t4_tweak_chip_settings(sc); 2341 2342 /* get basic stuff going */ 2343 rc = -t4_fw_initialize(sc, sc->mbox); 2344 if (rc != 0) { 2345 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2346 goto done; 2347 } 2348 } else { 2349 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2350 sc->cfcsum = 0; 2351 } 2352 2353 done: 2354 free(card_fw, M_CXGBE); 2355 if (fw != NULL) 2356 firmware_put(fw, FIRMWARE_UNLOAD); 2357 if (default_cfg != NULL) 2358 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2359 2360 return (rc); 2361 } 2362 2363 #define FW_PARAM_DEV(param) \ 2364 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2365 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2366 #define FW_PARAM_PFVF(param) \ 2367 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2368 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2369 2370 /* 2371 * Partition chip resources for use between various PFs, VFs, etc. 2372 */ 2373 static int 2374 partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2375 const char *name_prefix) 2376 { 2377 const struct firmware *cfg = NULL; 2378 int rc = 0; 2379 struct fw_caps_config_cmd caps; 2380 uint32_t mtype, moff, finicsum, cfcsum; 2381 2382 /* 2383 * Figure out what configuration file to use. Pick the default config 2384 * file for the card if the user hasn't specified one explicitly. 2385 */ 2386 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2387 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2388 /* Card specific overrides go here. */ 2389 if (pci_get_device(sc->dev) == 0x440a) 2390 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2391 if (is_fpga(sc)) 2392 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2393 } 2394 2395 /* 2396 * We need to load another module if the profile is anything except 2397 * "default" or "flash". 2398 */ 2399 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2400 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2401 char s[32]; 2402 2403 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2404 cfg = firmware_get(s); 2405 if (cfg == NULL) { 2406 if (default_cfg != NULL) { 2407 device_printf(sc->dev, 2408 "unable to load module \"%s\" for " 2409 "configuration profile \"%s\", will use " 2410 "the default config file instead.\n", 2411 s, sc->cfg_file); 2412 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2413 "%s", DEFAULT_CF); 2414 } else { 2415 device_printf(sc->dev, 2416 "unable to load module \"%s\" for " 2417 "configuration profile \"%s\", will use " 2418 "the config file on the card's flash " 2419 "instead.\n", s, sc->cfg_file); 2420 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2421 "%s", FLASH_CF); 2422 } 2423 } 2424 } 2425 2426 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2427 default_cfg == NULL) { 2428 device_printf(sc->dev, 2429 "default config file not available, will use the config " 2430 "file on the card's flash instead.\n"); 2431 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2432 } 2433 2434 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2435 u_int cflen, i, n; 2436 const uint32_t *cfdata; 2437 uint32_t param, val, addr, off, mw_base, mw_aperture; 2438 2439 KASSERT(cfg != NULL || default_cfg != NULL, 2440 ("%s: no config to upload", __func__)); 2441 2442 /* 2443 * Ask the firmware where it wants us to upload the config file. 2444 */ 2445 param = FW_PARAM_DEV(CF); 2446 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2447 if (rc != 0) { 2448 /* No support for config file? Shouldn't happen. */ 2449 device_printf(sc->dev, 2450 "failed to query config file location: %d.\n", rc); 2451 goto done; 2452 } 2453 mtype = G_FW_PARAMS_PARAM_Y(val); 2454 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2455 2456 /* 2457 * XXX: sheer laziness. We deliberately added 4 bytes of 2458 * useless stuffing/comments at the end of the config file so 2459 * it's ok to simply throw away the last remaining bytes when 2460 * the config file is not an exact multiple of 4. This also 2461 * helps with the validate_mt_off_len check. 2462 */ 2463 if (cfg != NULL) { 2464 cflen = cfg->datasize & ~3; 2465 cfdata = cfg->data; 2466 } else { 2467 cflen = default_cfg->datasize & ~3; 2468 cfdata = default_cfg->data; 2469 } 2470 2471 if (cflen > FLASH_CFG_MAX_SIZE) { 2472 device_printf(sc->dev, 2473 "config file too long (%d, max allowed is %d). " 2474 "Will try to use the config on the card, if any.\n", 2475 cflen, FLASH_CFG_MAX_SIZE); 2476 goto use_config_on_flash; 2477 } 2478 2479 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2480 if (rc != 0) { 2481 device_printf(sc->dev, 2482 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2483 "Will try to use the config on the card, if any.\n", 2484 __func__, mtype, moff, cflen, rc); 2485 goto use_config_on_flash; 2486 } 2487 2488 memwin_info(sc, 2, &mw_base, &mw_aperture); 2489 while (cflen) { 2490 off = position_memwin(sc, 2, addr); 2491 n = min(cflen, mw_aperture - off); 2492 for (i = 0; i < n; i += 4) 2493 t4_write_reg(sc, mw_base + off + i, *cfdata++); 2494 cflen -= n; 2495 addr += n; 2496 } 2497 } else { 2498 use_config_on_flash: 2499 mtype = FW_MEMTYPE_FLASH; 2500 moff = t4_flash_cfg_addr(sc); 2501 } 2502 2503 bzero(&caps, sizeof(caps)); 2504 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2505 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2506 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2507 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2508 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2509 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2510 if (rc != 0) { 2511 device_printf(sc->dev, 2512 "failed to pre-process config file: %d " 2513 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2514 goto done; 2515 } 2516 2517 finicsum = be32toh(caps.finicsum); 2518 cfcsum = be32toh(caps.cfcsum); 2519 if (finicsum != cfcsum) { 2520 device_printf(sc->dev, 2521 "WARNING: config file checksum mismatch: %08x %08x\n", 2522 finicsum, cfcsum); 2523 } 2524 sc->cfcsum = cfcsum; 2525 2526 #define LIMIT_CAPS(x) do { \ 2527 caps.x &= htobe16(t4_##x##_allowed); \ 2528 } while (0) 2529 2530 /* 2531 * Let the firmware know what features will (not) be used so it can tune 2532 * things accordingly. 2533 */ 2534 LIMIT_CAPS(linkcaps); 2535 LIMIT_CAPS(niccaps); 2536 LIMIT_CAPS(toecaps); 2537 LIMIT_CAPS(rdmacaps); 2538 LIMIT_CAPS(iscsicaps); 2539 LIMIT_CAPS(fcoecaps); 2540 #undef LIMIT_CAPS 2541 2542 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2543 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2544 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2545 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2546 if (rc != 0) { 2547 device_printf(sc->dev, 2548 "failed to process config file: %d.\n", rc); 2549 } 2550 done: 2551 if (cfg != NULL) 2552 firmware_put(cfg, FIRMWARE_UNLOAD); 2553 return (rc); 2554 } 2555 2556 /* 2557 * Retrieve parameters that are needed (or nice to have) very early. 2558 */ 2559 static int 2560 get_params__pre_init(struct adapter *sc) 2561 { 2562 int rc; 2563 uint32_t param[2], val[2]; 2564 struct fw_devlog_cmd cmd; 2565 struct devlog_params *dlog = &sc->params.devlog; 2566 2567 param[0] = FW_PARAM_DEV(PORTVEC); 2568 param[1] = FW_PARAM_DEV(CCLK); 2569 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2570 if (rc != 0) { 2571 device_printf(sc->dev, 2572 "failed to query parameters (pre_init): %d.\n", rc); 2573 return (rc); 2574 } 2575 2576 sc->params.portvec = val[0]; 2577 sc->params.nports = bitcount32(val[0]); 2578 sc->params.vpd.cclk = val[1]; 2579 2580 /* Read device log parameters. */ 2581 bzero(&cmd, sizeof(cmd)); 2582 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2583 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2584 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2585 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2586 if (rc != 0) { 2587 device_printf(sc->dev, 2588 "failed to get devlog parameters: %d.\n", rc); 2589 bzero(dlog, sizeof (*dlog)); 2590 rc = 0; /* devlog isn't critical for device operation */ 2591 } else { 2592 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2593 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2594 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2595 dlog->size = be32toh(cmd.memsize_devlog); 2596 } 2597 2598 return (rc); 2599 } 2600 2601 /* 2602 * Retrieve various parameters that are of interest to the driver. The device 2603 * has been initialized by the firmware at this point. 2604 */ 2605 static int 2606 get_params__post_init(struct adapter *sc) 2607 { 2608 int rc; 2609 uint32_t param[7], val[7]; 2610 struct fw_caps_config_cmd caps; 2611 2612 param[0] = FW_PARAM_PFVF(IQFLINT_START); 2613 param[1] = FW_PARAM_PFVF(EQ_START); 2614 param[2] = FW_PARAM_PFVF(FILTER_START); 2615 param[3] = FW_PARAM_PFVF(FILTER_END); 2616 param[4] = FW_PARAM_PFVF(L2T_START); 2617 param[5] = FW_PARAM_PFVF(L2T_END); 2618 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2619 if (rc != 0) { 2620 device_printf(sc->dev, 2621 "failed to query parameters (post_init): %d.\n", rc); 2622 return (rc); 2623 } 2624 2625 sc->sge.iq_start = val[0]; 2626 sc->sge.eq_start = val[1]; 2627 sc->tids.ftid_base = val[2]; 2628 sc->tids.nftids = val[3] - val[2] + 1; 2629 sc->params.ftid_min = val[2]; 2630 sc->params.ftid_max = val[3]; 2631 sc->vres.l2t.start = val[4]; 2632 sc->vres.l2t.size = val[5] - val[4] + 1; 2633 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2634 ("%s: L2 table size (%u) larger than expected (%u)", 2635 __func__, sc->vres.l2t.size, L2T_SIZE)); 2636 2637 /* get capabilites */ 2638 bzero(&caps, sizeof(caps)); 2639 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2640 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2641 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2642 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2643 if (rc != 0) { 2644 device_printf(sc->dev, 2645 "failed to get card capabilities: %d.\n", rc); 2646 return (rc); 2647 } 2648 2649 #define READ_CAPS(x) do { \ 2650 sc->x = htobe16(caps.x); \ 2651 } while (0) 2652 READ_CAPS(linkcaps); 2653 READ_CAPS(niccaps); 2654 READ_CAPS(toecaps); 2655 READ_CAPS(rdmacaps); 2656 READ_CAPS(iscsicaps); 2657 READ_CAPS(fcoecaps); 2658 2659 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { 2660 param[0] = FW_PARAM_PFVF(ETHOFLD_START); 2661 param[1] = FW_PARAM_PFVF(ETHOFLD_END); 2662 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2663 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); 2664 if (rc != 0) { 2665 device_printf(sc->dev, 2666 "failed to query NIC parameters: %d.\n", rc); 2667 return (rc); 2668 } 2669 sc->tids.etid_base = val[0]; 2670 sc->params.etid_min = val[0]; 2671 sc->tids.netids = val[1] - val[0] + 1; 2672 sc->params.netids = sc->tids.netids; 2673 sc->params.eo_wr_cred = val[2]; 2674 sc->params.ethoffload = 1; 2675 } 2676 2677 if (sc->toecaps) { 2678 /* query offload-related parameters */ 2679 param[0] = FW_PARAM_DEV(NTID); 2680 param[1] = FW_PARAM_PFVF(SERVER_START); 2681 param[2] = FW_PARAM_PFVF(SERVER_END); 2682 param[3] = FW_PARAM_PFVF(TDDP_START); 2683 param[4] = FW_PARAM_PFVF(TDDP_END); 2684 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2685 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2686 if (rc != 0) { 2687 device_printf(sc->dev, 2688 "failed to query TOE parameters: %d.\n", rc); 2689 return (rc); 2690 } 2691 sc->tids.ntids = val[0]; 2692 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 2693 sc->tids.stid_base = val[1]; 2694 sc->tids.nstids = val[2] - val[1] + 1; 2695 sc->vres.ddp.start = val[3]; 2696 sc->vres.ddp.size = val[4] - val[3] + 1; 2697 sc->params.ofldq_wr_cred = val[5]; 2698 sc->params.offload = 1; 2699 } 2700 if (sc->rdmacaps) { 2701 param[0] = FW_PARAM_PFVF(STAG_START); 2702 param[1] = FW_PARAM_PFVF(STAG_END); 2703 param[2] = FW_PARAM_PFVF(RQ_START); 2704 param[3] = FW_PARAM_PFVF(RQ_END); 2705 param[4] = FW_PARAM_PFVF(PBL_START); 2706 param[5] = FW_PARAM_PFVF(PBL_END); 2707 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2708 if (rc != 0) { 2709 device_printf(sc->dev, 2710 "failed to query RDMA parameters(1): %d.\n", rc); 2711 return (rc); 2712 } 2713 sc->vres.stag.start = val[0]; 2714 sc->vres.stag.size = val[1] - val[0] + 1; 2715 sc->vres.rq.start = val[2]; 2716 sc->vres.rq.size = val[3] - val[2] + 1; 2717 sc->vres.pbl.start = val[4]; 2718 sc->vres.pbl.size = val[5] - val[4] + 1; 2719 2720 param[0] = FW_PARAM_PFVF(SQRQ_START); 2721 param[1] = FW_PARAM_PFVF(SQRQ_END); 2722 param[2] = FW_PARAM_PFVF(CQ_START); 2723 param[3] = FW_PARAM_PFVF(CQ_END); 2724 param[4] = FW_PARAM_PFVF(OCQ_START); 2725 param[5] = FW_PARAM_PFVF(OCQ_END); 2726 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2727 if (rc != 0) { 2728 device_printf(sc->dev, 2729 "failed to query RDMA parameters(2): %d.\n", rc); 2730 return (rc); 2731 } 2732 sc->vres.qp.start = val[0]; 2733 sc->vres.qp.size = val[1] - val[0] + 1; 2734 sc->vres.cq.start = val[2]; 2735 sc->vres.cq.size = val[3] - val[2] + 1; 2736 sc->vres.ocq.start = val[4]; 2737 sc->vres.ocq.size = val[5] - val[4] + 1; 2738 } 2739 if (sc->iscsicaps) { 2740 param[0] = FW_PARAM_PFVF(ISCSI_START); 2741 param[1] = FW_PARAM_PFVF(ISCSI_END); 2742 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2743 if (rc != 0) { 2744 device_printf(sc->dev, 2745 "failed to query iSCSI parameters: %d.\n", rc); 2746 return (rc); 2747 } 2748 sc->vres.iscsi.start = val[0]; 2749 sc->vres.iscsi.size = val[1] - val[0] + 1; 2750 } 2751 2752 /* 2753 * We've got the params we wanted to query via the firmware. Now grab 2754 * some others directly from the chip. 2755 */ 2756 rc = t4_read_chip_settings(sc); 2757 2758 return (rc); 2759 } 2760 2761 static int 2762 set_params__post_init(struct adapter *sc) 2763 { 2764 uint32_t param, val; 2765 2766 /* ask for encapsulated CPLs */ 2767 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 2768 val = 1; 2769 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2770 2771 return (0); 2772 } 2773 2774 #undef FW_PARAM_PFVF 2775 #undef FW_PARAM_DEV 2776 2777 static void 2778 t4_set_desc(struct adapter *sc) 2779 { 2780 char buf[128]; 2781 struct adapter_params *p = &sc->params; 2782 2783 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, " 2784 "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "", 2785 chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec); 2786 2787 device_set_desc_copy(sc->dev, buf); 2788 } 2789 2790 static void 2791 build_medialist(struct port_info *pi, struct ifmedia *media) 2792 { 2793 int data, m; 2794 2795 PORT_LOCK(pi); 2796 2797 ifmedia_removeall(media); 2798 2799 m = IFM_ETHER | IFM_FDX; 2800 data = (pi->port_type << 8) | pi->mod_type; 2801 2802 switch(pi->port_type) { 2803 case FW_PORT_TYPE_BT_XFI: 2804 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2805 break; 2806 2807 case FW_PORT_TYPE_BT_XAUI: 2808 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2809 /* fall through */ 2810 2811 case FW_PORT_TYPE_BT_SGMII: 2812 ifmedia_add(media, m | IFM_1000_T, data, NULL); 2813 ifmedia_add(media, m | IFM_100_TX, data, NULL); 2814 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL); 2815 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2816 break; 2817 2818 case FW_PORT_TYPE_CX4: 2819 ifmedia_add(media, m | IFM_10G_CX4, data, NULL); 2820 ifmedia_set(media, m | IFM_10G_CX4); 2821 break; 2822 2823 case FW_PORT_TYPE_QSFP_10G: 2824 case FW_PORT_TYPE_SFP: 2825 case FW_PORT_TYPE_FIBER_XFI: 2826 case FW_PORT_TYPE_FIBER_XAUI: 2827 switch (pi->mod_type) { 2828 2829 case FW_PORT_MOD_TYPE_LR: 2830 ifmedia_add(media, m | IFM_10G_LR, data, NULL); 2831 ifmedia_set(media, m | IFM_10G_LR); 2832 break; 2833 2834 case FW_PORT_MOD_TYPE_SR: 2835 ifmedia_add(media, m | IFM_10G_SR, data, NULL); 2836 ifmedia_set(media, m | IFM_10G_SR); 2837 break; 2838 2839 case FW_PORT_MOD_TYPE_LRM: 2840 ifmedia_add(media, m | IFM_10G_LRM, data, NULL); 2841 ifmedia_set(media, m | IFM_10G_LRM); 2842 break; 2843 2844 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2845 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2846 ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL); 2847 ifmedia_set(media, m | IFM_10G_TWINAX); 2848 break; 2849 2850 case FW_PORT_MOD_TYPE_NONE: 2851 m &= ~IFM_FDX; 2852 ifmedia_add(media, m | IFM_NONE, data, NULL); 2853 ifmedia_set(media, m | IFM_NONE); 2854 break; 2855 2856 case FW_PORT_MOD_TYPE_NA: 2857 case FW_PORT_MOD_TYPE_ER: 2858 default: 2859 device_printf(pi->dev, 2860 "unknown port_type (%d), mod_type (%d)\n", 2861 pi->port_type, pi->mod_type); 2862 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2863 ifmedia_set(media, m | IFM_UNKNOWN); 2864 break; 2865 } 2866 break; 2867 2868 case FW_PORT_TYPE_QSFP: 2869 switch (pi->mod_type) { 2870 2871 case FW_PORT_MOD_TYPE_LR: 2872 ifmedia_add(media, m | IFM_40G_LR4, data, NULL); 2873 ifmedia_set(media, m | IFM_40G_LR4); 2874 break; 2875 2876 case FW_PORT_MOD_TYPE_SR: 2877 ifmedia_add(media, m | IFM_40G_SR4, data, NULL); 2878 ifmedia_set(media, m | IFM_40G_SR4); 2879 break; 2880 2881 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2882 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2883 ifmedia_add(media, m | IFM_40G_CR4, data, NULL); 2884 ifmedia_set(media, m | IFM_40G_CR4); 2885 break; 2886 2887 case FW_PORT_MOD_TYPE_NONE: 2888 m &= ~IFM_FDX; 2889 ifmedia_add(media, m | IFM_NONE, data, NULL); 2890 ifmedia_set(media, m | IFM_NONE); 2891 break; 2892 2893 default: 2894 device_printf(pi->dev, 2895 "unknown port_type (%d), mod_type (%d)\n", 2896 pi->port_type, pi->mod_type); 2897 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2898 ifmedia_set(media, m | IFM_UNKNOWN); 2899 break; 2900 } 2901 break; 2902 2903 default: 2904 device_printf(pi->dev, 2905 "unknown port_type (%d), mod_type (%d)\n", pi->port_type, 2906 pi->mod_type); 2907 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2908 ifmedia_set(media, m | IFM_UNKNOWN); 2909 break; 2910 } 2911 2912 PORT_UNLOCK(pi); 2913 } 2914 2915 #define FW_MAC_EXACT_CHUNK 7 2916 2917 /* 2918 * Program the port's XGMAC based on parameters in ifnet. The caller also 2919 * indicates which parameters should be programmed (the rest are left alone). 2920 */ 2921 int 2922 update_mac_settings(struct ifnet *ifp, int flags) 2923 { 2924 int rc = 0; 2925 struct port_info *pi = ifp->if_softc; 2926 struct adapter *sc = pi->adapter; 2927 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 2928 uint16_t viid = 0xffff; 2929 int16_t *xact_addr_filt = NULL; 2930 2931 ASSERT_SYNCHRONIZED_OP(sc); 2932 KASSERT(flags, ("%s: not told what to update.", __func__)); 2933 2934 if (ifp == pi->ifp) { 2935 viid = pi->viid; 2936 xact_addr_filt = &pi->xact_addr_filt; 2937 } 2938 #ifdef DEV_NETMAP 2939 else if (ifp == pi->nm_ifp) { 2940 viid = pi->nm_viid; 2941 xact_addr_filt = &pi->nm_xact_addr_filt; 2942 } 2943 #endif 2944 if (flags & XGMAC_MTU) 2945 mtu = ifp->if_mtu; 2946 2947 if (flags & XGMAC_PROMISC) 2948 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 2949 2950 if (flags & XGMAC_ALLMULTI) 2951 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 2952 2953 if (flags & XGMAC_VLANEX) 2954 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 2955 2956 if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) { 2957 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti, 2958 1, vlanex, false); 2959 if (rc) { 2960 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, 2961 rc); 2962 return (rc); 2963 } 2964 } 2965 2966 if (flags & XGMAC_UCADDR) { 2967 uint8_t ucaddr[ETHER_ADDR_LEN]; 2968 2969 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 2970 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr, 2971 true, true); 2972 if (rc < 0) { 2973 rc = -rc; 2974 if_printf(ifp, "change_mac failed: %d\n", rc); 2975 return (rc); 2976 } else { 2977 *xact_addr_filt = rc; 2978 rc = 0; 2979 } 2980 } 2981 2982 if (flags & XGMAC_MCADDRS) { 2983 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 2984 int del = 1; 2985 uint64_t hash = 0; 2986 struct ifmultiaddr *ifma; 2987 int i = 0, j; 2988 2989 if_maddr_rlock(ifp); 2990 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 2991 if (ifma->ifma_addr->sa_family != AF_LINK) 2992 continue; 2993 mcaddr[i++] = 2994 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 2995 2996 if (i == FW_MAC_EXACT_CHUNK) { 2997 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, 2998 i, mcaddr, NULL, &hash, 0); 2999 if (rc < 0) { 3000 rc = -rc; 3001 for (j = 0; j < i; j++) { 3002 if_printf(ifp, 3003 "failed to add mc address" 3004 " %02x:%02x:%02x:" 3005 "%02x:%02x:%02x rc=%d\n", 3006 mcaddr[j][0], mcaddr[j][1], 3007 mcaddr[j][2], mcaddr[j][3], 3008 mcaddr[j][4], mcaddr[j][5], 3009 rc); 3010 } 3011 goto mcfail; 3012 } 3013 del = 0; 3014 i = 0; 3015 } 3016 } 3017 if (i > 0) { 3018 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i, 3019 mcaddr, NULL, &hash, 0); 3020 if (rc < 0) { 3021 rc = -rc; 3022 for (j = 0; j < i; j++) { 3023 if_printf(ifp, 3024 "failed to add mc address" 3025 " %02x:%02x:%02x:" 3026 "%02x:%02x:%02x rc=%d\n", 3027 mcaddr[j][0], mcaddr[j][1], 3028 mcaddr[j][2], mcaddr[j][3], 3029 mcaddr[j][4], mcaddr[j][5], 3030 rc); 3031 } 3032 goto mcfail; 3033 } 3034 } 3035 3036 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0); 3037 if (rc != 0) 3038 if_printf(ifp, "failed to set mc address hash: %d", rc); 3039 mcfail: 3040 if_maddr_runlock(ifp); 3041 } 3042 3043 return (rc); 3044 } 3045 3046 int 3047 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags, 3048 char *wmesg) 3049 { 3050 int rc, pri; 3051 3052 #ifdef WITNESS 3053 /* the caller thinks it's ok to sleep, but is it really? */ 3054 if (flags & SLEEP_OK) 3055 pause("t4slptst", 1); 3056 #endif 3057 3058 if (INTR_OK) 3059 pri = PCATCH; 3060 else 3061 pri = 0; 3062 3063 ADAPTER_LOCK(sc); 3064 for (;;) { 3065 3066 if (pi && IS_DOOMED(pi)) { 3067 rc = ENXIO; 3068 goto done; 3069 } 3070 3071 if (!IS_BUSY(sc)) { 3072 rc = 0; 3073 break; 3074 } 3075 3076 if (!(flags & SLEEP_OK)) { 3077 rc = EBUSY; 3078 goto done; 3079 } 3080 3081 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 3082 rc = EINTR; 3083 goto done; 3084 } 3085 } 3086 3087 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 3088 SET_BUSY(sc); 3089 #ifdef INVARIANTS 3090 sc->last_op = wmesg; 3091 sc->last_op_thr = curthread; 3092 #endif 3093 3094 done: 3095 if (!(flags & HOLD_LOCK) || rc) 3096 ADAPTER_UNLOCK(sc); 3097 3098 return (rc); 3099 } 3100 3101 void 3102 end_synchronized_op(struct adapter *sc, int flags) 3103 { 3104 3105 if (flags & LOCK_HELD) 3106 ADAPTER_LOCK_ASSERT_OWNED(sc); 3107 else 3108 ADAPTER_LOCK(sc); 3109 3110 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 3111 CLR_BUSY(sc); 3112 wakeup(&sc->flags); 3113 ADAPTER_UNLOCK(sc); 3114 } 3115 3116 static int 3117 cxgbe_init_synchronized(struct port_info *pi) 3118 { 3119 struct adapter *sc = pi->adapter; 3120 struct ifnet *ifp = pi->ifp; 3121 int rc = 0; 3122 3123 ASSERT_SYNCHRONIZED_OP(sc); 3124 3125 if (isset(&sc->open_device_map, pi->port_id)) { 3126 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 3127 ("mismatch between open_device_map and if_drv_flags")); 3128 return (0); /* already running */ 3129 } 3130 3131 if (!(sc->flags & FULL_INIT_DONE) && 3132 ((rc = adapter_full_init(sc)) != 0)) 3133 return (rc); /* error message displayed already */ 3134 3135 if (!(pi->flags & PORT_INIT_DONE) && 3136 ((rc = port_full_init(pi)) != 0)) 3137 return (rc); /* error message displayed already */ 3138 3139 rc = update_mac_settings(ifp, XGMAC_ALL); 3140 if (rc) 3141 goto done; /* error message displayed already */ 3142 3143 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 3144 if (rc != 0) { 3145 if_printf(ifp, "enable_vi failed: %d\n", rc); 3146 goto done; 3147 } 3148 3149 /* 3150 * The first iq of the first port to come up is used for tracing. 3151 */ 3152 if (sc->traceq < 0) { 3153 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id; 3154 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : 3155 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | 3156 V_QUEUENUMBER(sc->traceq)); 3157 pi->flags |= HAS_TRACEQ; 3158 } 3159 3160 /* all ok */ 3161 setbit(&sc->open_device_map, pi->port_id); 3162 PORT_LOCK(pi); 3163 ifp->if_drv_flags |= IFF_DRV_RUNNING; 3164 PORT_UNLOCK(pi); 3165 3166 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 3167 done: 3168 if (rc != 0) 3169 cxgbe_uninit_synchronized(pi); 3170 3171 return (rc); 3172 } 3173 3174 /* 3175 * Idempotent. 3176 */ 3177 static int 3178 cxgbe_uninit_synchronized(struct port_info *pi) 3179 { 3180 struct adapter *sc = pi->adapter; 3181 struct ifnet *ifp = pi->ifp; 3182 int rc; 3183 3184 ASSERT_SYNCHRONIZED_OP(sc); 3185 3186 /* 3187 * Disable the VI so that all its data in either direction is discarded 3188 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 3189 * tick) intact as the TP can deliver negative advice or data that it's 3190 * holding in its RAM (for an offloaded connection) even after the VI is 3191 * disabled. 3192 */ 3193 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 3194 if (rc) { 3195 if_printf(ifp, "disable_vi failed: %d\n", rc); 3196 return (rc); 3197 } 3198 3199 clrbit(&sc->open_device_map, pi->port_id); 3200 PORT_LOCK(pi); 3201 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3202 PORT_UNLOCK(pi); 3203 3204 pi->link_cfg.link_ok = 0; 3205 pi->link_cfg.speed = 0; 3206 pi->linkdnrc = -1; 3207 t4_os_link_changed(sc, pi->port_id, 0, -1); 3208 3209 return (0); 3210 } 3211 3212 /* 3213 * It is ok for this function to fail midway and return right away. t4_detach 3214 * will walk the entire sc->irq list and clean up whatever is valid. 3215 */ 3216 static int 3217 setup_intr_handlers(struct adapter *sc) 3218 { 3219 int rc, rid, p, q; 3220 char s[8]; 3221 struct irq *irq; 3222 struct port_info *pi; 3223 struct sge_rxq *rxq; 3224 #ifdef TCP_OFFLOAD 3225 struct sge_ofld_rxq *ofld_rxq; 3226 #endif 3227 #ifdef DEV_NETMAP 3228 struct sge_nm_rxq *nm_rxq; 3229 #endif 3230 3231 /* 3232 * Setup interrupts. 3233 */ 3234 irq = &sc->irq[0]; 3235 rid = sc->intr_type == INTR_INTX ? 0 : 1; 3236 if (sc->intr_count == 1) 3237 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all")); 3238 3239 /* Multiple interrupts. */ 3240 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 3241 ("%s: too few intr.", __func__)); 3242 3243 /* The first one is always error intr */ 3244 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 3245 if (rc != 0) 3246 return (rc); 3247 irq++; 3248 rid++; 3249 3250 /* The second one is always the firmware event queue */ 3251 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt"); 3252 if (rc != 0) 3253 return (rc); 3254 irq++; 3255 rid++; 3256 3257 for_each_port(sc, p) { 3258 pi = sc->port[p]; 3259 3260 if (pi->flags & INTR_RXQ) { 3261 for_each_rxq(pi, q, rxq) { 3262 snprintf(s, sizeof(s), "%d.%d", p, q); 3263 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq, 3264 s); 3265 if (rc != 0) 3266 return (rc); 3267 irq++; 3268 rid++; 3269 } 3270 } 3271 #ifdef TCP_OFFLOAD 3272 if (pi->flags & INTR_OFLD_RXQ) { 3273 for_each_ofld_rxq(pi, q, ofld_rxq) { 3274 snprintf(s, sizeof(s), "%d,%d", p, q); 3275 rc = t4_alloc_irq(sc, irq, rid, t4_intr, 3276 ofld_rxq, s); 3277 if (rc != 0) 3278 return (rc); 3279 irq++; 3280 rid++; 3281 } 3282 } 3283 #endif 3284 #ifdef DEV_NETMAP 3285 if (pi->flags & INTR_NM_RXQ) { 3286 for_each_nm_rxq(pi, q, nm_rxq) { 3287 snprintf(s, sizeof(s), "%d-%d", p, q); 3288 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr, 3289 nm_rxq, s); 3290 if (rc != 0) 3291 return (rc); 3292 irq++; 3293 rid++; 3294 } 3295 } 3296 #endif 3297 } 3298 MPASS(irq == &sc->irq[sc->intr_count]); 3299 3300 return (0); 3301 } 3302 3303 int 3304 adapter_full_init(struct adapter *sc) 3305 { 3306 int rc, i; 3307 3308 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3309 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 3310 ("%s: FULL_INIT_DONE already", __func__)); 3311 3312 /* 3313 * queues that belong to the adapter (not any particular port). 3314 */ 3315 rc = t4_setup_adapter_queues(sc); 3316 if (rc != 0) 3317 goto done; 3318 3319 for (i = 0; i < nitems(sc->tq); i++) { 3320 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 3321 taskqueue_thread_enqueue, &sc->tq[i]); 3322 if (sc->tq[i] == NULL) { 3323 device_printf(sc->dev, 3324 "failed to allocate task queue %d\n", i); 3325 rc = ENOMEM; 3326 goto done; 3327 } 3328 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3329 device_get_nameunit(sc->dev), i); 3330 } 3331 3332 t4_intr_enable(sc); 3333 sc->flags |= FULL_INIT_DONE; 3334 done: 3335 if (rc != 0) 3336 adapter_full_uninit(sc); 3337 3338 return (rc); 3339 } 3340 3341 int 3342 adapter_full_uninit(struct adapter *sc) 3343 { 3344 int i; 3345 3346 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3347 3348 t4_teardown_adapter_queues(sc); 3349 3350 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3351 taskqueue_free(sc->tq[i]); 3352 sc->tq[i] = NULL; 3353 } 3354 3355 sc->flags &= ~FULL_INIT_DONE; 3356 3357 return (0); 3358 } 3359 3360 int 3361 port_full_init(struct port_info *pi) 3362 { 3363 struct adapter *sc = pi->adapter; 3364 struct ifnet *ifp = pi->ifp; 3365 uint16_t *rss; 3366 struct sge_rxq *rxq; 3367 int rc, i, j; 3368 3369 ASSERT_SYNCHRONIZED_OP(sc); 3370 KASSERT((pi->flags & PORT_INIT_DONE) == 0, 3371 ("%s: PORT_INIT_DONE already", __func__)); 3372 3373 sysctl_ctx_init(&pi->ctx); 3374 pi->flags |= PORT_SYSCTL_CTX; 3375 3376 /* 3377 * Allocate tx/rx/fl queues for this port. 3378 */ 3379 rc = t4_setup_port_queues(pi); 3380 if (rc != 0) 3381 goto done; /* error message displayed already */ 3382 3383 /* 3384 * Setup RSS for this port. Save a copy of the RSS table for later use. 3385 */ 3386 rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); 3387 for (i = 0; i < pi->rss_size;) { 3388 for_each_rxq(pi, j, rxq) { 3389 rss[i++] = rxq->iq.abs_id; 3390 if (i == pi->rss_size) 3391 break; 3392 } 3393 } 3394 3395 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss, 3396 pi->rss_size); 3397 if (rc != 0) { 3398 if_printf(ifp, "rss_config failed: %d\n", rc); 3399 goto done; 3400 } 3401 3402 pi->rss = rss; 3403 pi->flags |= PORT_INIT_DONE; 3404 done: 3405 if (rc != 0) 3406 port_full_uninit(pi); 3407 3408 return (rc); 3409 } 3410 3411 /* 3412 * Idempotent. 3413 */ 3414 int 3415 port_full_uninit(struct port_info *pi) 3416 { 3417 struct adapter *sc = pi->adapter; 3418 int i; 3419 struct sge_rxq *rxq; 3420 struct sge_txq *txq; 3421 #ifdef TCP_OFFLOAD 3422 struct sge_ofld_rxq *ofld_rxq; 3423 struct sge_wrq *ofld_txq; 3424 #endif 3425 3426 if (pi->flags & PORT_INIT_DONE) { 3427 3428 /* Need to quiesce queues. XXX: ctrl queues? */ 3429 3430 for_each_txq(pi, i, txq) { 3431 quiesce_eq(sc, &txq->eq); 3432 } 3433 3434 #ifdef TCP_OFFLOAD 3435 for_each_ofld_txq(pi, i, ofld_txq) { 3436 quiesce_eq(sc, &ofld_txq->eq); 3437 } 3438 #endif 3439 3440 for_each_rxq(pi, i, rxq) { 3441 quiesce_iq(sc, &rxq->iq); 3442 quiesce_fl(sc, &rxq->fl); 3443 } 3444 3445 #ifdef TCP_OFFLOAD 3446 for_each_ofld_rxq(pi, i, ofld_rxq) { 3447 quiesce_iq(sc, &ofld_rxq->iq); 3448 quiesce_fl(sc, &ofld_rxq->fl); 3449 } 3450 #endif 3451 free(pi->rss, M_CXGBE); 3452 } 3453 3454 t4_teardown_port_queues(pi); 3455 pi->flags &= ~PORT_INIT_DONE; 3456 3457 return (0); 3458 } 3459 3460 static void 3461 quiesce_eq(struct adapter *sc, struct sge_eq *eq) 3462 { 3463 EQ_LOCK(eq); 3464 eq->flags |= EQ_DOOMED; 3465 3466 /* 3467 * Wait for the response to a credit flush if one's 3468 * pending. 3469 */ 3470 while (eq->flags & EQ_CRFLUSHED) 3471 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0); 3472 EQ_UNLOCK(eq); 3473 3474 callout_drain(&eq->tx_callout); /* XXX: iffy */ 3475 pause("callout", 10); /* Still iffy */ 3476 3477 taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task); 3478 } 3479 3480 static void 3481 quiesce_iq(struct adapter *sc, struct sge_iq *iq) 3482 { 3483 (void) sc; /* unused */ 3484 3485 /* Synchronize with the interrupt handler */ 3486 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 3487 pause("iqfree", 1); 3488 } 3489 3490 static void 3491 quiesce_fl(struct adapter *sc, struct sge_fl *fl) 3492 { 3493 mtx_lock(&sc->sfl_lock); 3494 FL_LOCK(fl); 3495 fl->flags |= FL_DOOMED; 3496 FL_UNLOCK(fl); 3497 mtx_unlock(&sc->sfl_lock); 3498 3499 callout_drain(&sc->sfl_callout); 3500 KASSERT((fl->flags & FL_STARVING) == 0, 3501 ("%s: still starving", __func__)); 3502 } 3503 3504 static int 3505 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 3506 driver_intr_t *handler, void *arg, char *name) 3507 { 3508 int rc; 3509 3510 irq->rid = rid; 3511 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 3512 RF_SHAREABLE | RF_ACTIVE); 3513 if (irq->res == NULL) { 3514 device_printf(sc->dev, 3515 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 3516 return (ENOMEM); 3517 } 3518 3519 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 3520 NULL, handler, arg, &irq->tag); 3521 if (rc != 0) { 3522 device_printf(sc->dev, 3523 "failed to setup interrupt for rid %d, name %s: %d\n", 3524 rid, name, rc); 3525 } else if (name) 3526 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 3527 3528 return (rc); 3529 } 3530 3531 static int 3532 t4_free_irq(struct adapter *sc, struct irq *irq) 3533 { 3534 if (irq->tag) 3535 bus_teardown_intr(sc->dev, irq->res, irq->tag); 3536 if (irq->res) 3537 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 3538 3539 bzero(irq, sizeof(*irq)); 3540 3541 return (0); 3542 } 3543 3544 static void 3545 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 3546 unsigned int end) 3547 { 3548 uint32_t *p = (uint32_t *)(buf + start); 3549 3550 for ( ; start <= end; start += sizeof(uint32_t)) 3551 *p++ = t4_read_reg(sc, start); 3552 } 3553 3554 static void 3555 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 3556 { 3557 int i, n; 3558 const unsigned int *reg_ranges; 3559 static const unsigned int t4_reg_ranges[] = { 3560 0x1008, 0x1108, 3561 0x1180, 0x11b4, 3562 0x11fc, 0x123c, 3563 0x1300, 0x173c, 3564 0x1800, 0x18fc, 3565 0x3000, 0x30d8, 3566 0x30e0, 0x5924, 3567 0x5960, 0x59d4, 3568 0x5a00, 0x5af8, 3569 0x6000, 0x6098, 3570 0x6100, 0x6150, 3571 0x6200, 0x6208, 3572 0x6240, 0x6248, 3573 0x6280, 0x6338, 3574 0x6370, 0x638c, 3575 0x6400, 0x643c, 3576 0x6500, 0x6524, 3577 0x6a00, 0x6a38, 3578 0x6a60, 0x6a78, 3579 0x6b00, 0x6b84, 3580 0x6bf0, 0x6c84, 3581 0x6cf0, 0x6d84, 3582 0x6df0, 0x6e84, 3583 0x6ef0, 0x6f84, 3584 0x6ff0, 0x7084, 3585 0x70f0, 0x7184, 3586 0x71f0, 0x7284, 3587 0x72f0, 0x7384, 3588 0x73f0, 0x7450, 3589 0x7500, 0x7530, 3590 0x7600, 0x761c, 3591 0x7680, 0x76cc, 3592 0x7700, 0x7798, 3593 0x77c0, 0x77fc, 3594 0x7900, 0x79fc, 3595 0x7b00, 0x7c38, 3596 0x7d00, 0x7efc, 3597 0x8dc0, 0x8e1c, 3598 0x8e30, 0x8e78, 3599 0x8ea0, 0x8f6c, 3600 0x8fc0, 0x9074, 3601 0x90fc, 0x90fc, 3602 0x9400, 0x9458, 3603 0x9600, 0x96bc, 3604 0x9800, 0x9808, 3605 0x9820, 0x983c, 3606 0x9850, 0x9864, 3607 0x9c00, 0x9c6c, 3608 0x9c80, 0x9cec, 3609 0x9d00, 0x9d6c, 3610 0x9d80, 0x9dec, 3611 0x9e00, 0x9e6c, 3612 0x9e80, 0x9eec, 3613 0x9f00, 0x9f6c, 3614 0x9f80, 0x9fec, 3615 0xd004, 0xd03c, 3616 0xdfc0, 0xdfe0, 3617 0xe000, 0xea7c, 3618 0xf000, 0x11110, 3619 0x11118, 0x11190, 3620 0x19040, 0x1906c, 3621 0x19078, 0x19080, 3622 0x1908c, 0x19124, 3623 0x19150, 0x191b0, 3624 0x191d0, 0x191e8, 3625 0x19238, 0x1924c, 3626 0x193f8, 0x19474, 3627 0x19490, 0x194f8, 3628 0x19800, 0x19f30, 3629 0x1a000, 0x1a06c, 3630 0x1a0b0, 0x1a120, 3631 0x1a128, 0x1a138, 3632 0x1a190, 0x1a1c4, 3633 0x1a1fc, 0x1a1fc, 3634 0x1e040, 0x1e04c, 3635 0x1e284, 0x1e28c, 3636 0x1e2c0, 0x1e2c0, 3637 0x1e2e0, 0x1e2e0, 3638 0x1e300, 0x1e384, 3639 0x1e3c0, 0x1e3c8, 3640 0x1e440, 0x1e44c, 3641 0x1e684, 0x1e68c, 3642 0x1e6c0, 0x1e6c0, 3643 0x1e6e0, 0x1e6e0, 3644 0x1e700, 0x1e784, 3645 0x1e7c0, 0x1e7c8, 3646 0x1e840, 0x1e84c, 3647 0x1ea84, 0x1ea8c, 3648 0x1eac0, 0x1eac0, 3649 0x1eae0, 0x1eae0, 3650 0x1eb00, 0x1eb84, 3651 0x1ebc0, 0x1ebc8, 3652 0x1ec40, 0x1ec4c, 3653 0x1ee84, 0x1ee8c, 3654 0x1eec0, 0x1eec0, 3655 0x1eee0, 0x1eee0, 3656 0x1ef00, 0x1ef84, 3657 0x1efc0, 0x1efc8, 3658 0x1f040, 0x1f04c, 3659 0x1f284, 0x1f28c, 3660 0x1f2c0, 0x1f2c0, 3661 0x1f2e0, 0x1f2e0, 3662 0x1f300, 0x1f384, 3663 0x1f3c0, 0x1f3c8, 3664 0x1f440, 0x1f44c, 3665 0x1f684, 0x1f68c, 3666 0x1f6c0, 0x1f6c0, 3667 0x1f6e0, 0x1f6e0, 3668 0x1f700, 0x1f784, 3669 0x1f7c0, 0x1f7c8, 3670 0x1f840, 0x1f84c, 3671 0x1fa84, 0x1fa8c, 3672 0x1fac0, 0x1fac0, 3673 0x1fae0, 0x1fae0, 3674 0x1fb00, 0x1fb84, 3675 0x1fbc0, 0x1fbc8, 3676 0x1fc40, 0x1fc4c, 3677 0x1fe84, 0x1fe8c, 3678 0x1fec0, 0x1fec0, 3679 0x1fee0, 0x1fee0, 3680 0x1ff00, 0x1ff84, 3681 0x1ffc0, 0x1ffc8, 3682 0x20000, 0x2002c, 3683 0x20100, 0x2013c, 3684 0x20190, 0x201c8, 3685 0x20200, 0x20318, 3686 0x20400, 0x20528, 3687 0x20540, 0x20614, 3688 0x21000, 0x21040, 3689 0x2104c, 0x21060, 3690 0x210c0, 0x210ec, 3691 0x21200, 0x21268, 3692 0x21270, 0x21284, 3693 0x212fc, 0x21388, 3694 0x21400, 0x21404, 3695 0x21500, 0x21518, 3696 0x2152c, 0x2153c, 3697 0x21550, 0x21554, 3698 0x21600, 0x21600, 3699 0x21608, 0x21628, 3700 0x21630, 0x2163c, 3701 0x21700, 0x2171c, 3702 0x21780, 0x2178c, 3703 0x21800, 0x21c38, 3704 0x21c80, 0x21d7c, 3705 0x21e00, 0x21e04, 3706 0x22000, 0x2202c, 3707 0x22100, 0x2213c, 3708 0x22190, 0x221c8, 3709 0x22200, 0x22318, 3710 0x22400, 0x22528, 3711 0x22540, 0x22614, 3712 0x23000, 0x23040, 3713 0x2304c, 0x23060, 3714 0x230c0, 0x230ec, 3715 0x23200, 0x23268, 3716 0x23270, 0x23284, 3717 0x232fc, 0x23388, 3718 0x23400, 0x23404, 3719 0x23500, 0x23518, 3720 0x2352c, 0x2353c, 3721 0x23550, 0x23554, 3722 0x23600, 0x23600, 3723 0x23608, 0x23628, 3724 0x23630, 0x2363c, 3725 0x23700, 0x2371c, 3726 0x23780, 0x2378c, 3727 0x23800, 0x23c38, 3728 0x23c80, 0x23d7c, 3729 0x23e00, 0x23e04, 3730 0x24000, 0x2402c, 3731 0x24100, 0x2413c, 3732 0x24190, 0x241c8, 3733 0x24200, 0x24318, 3734 0x24400, 0x24528, 3735 0x24540, 0x24614, 3736 0x25000, 0x25040, 3737 0x2504c, 0x25060, 3738 0x250c0, 0x250ec, 3739 0x25200, 0x25268, 3740 0x25270, 0x25284, 3741 0x252fc, 0x25388, 3742 0x25400, 0x25404, 3743 0x25500, 0x25518, 3744 0x2552c, 0x2553c, 3745 0x25550, 0x25554, 3746 0x25600, 0x25600, 3747 0x25608, 0x25628, 3748 0x25630, 0x2563c, 3749 0x25700, 0x2571c, 3750 0x25780, 0x2578c, 3751 0x25800, 0x25c38, 3752 0x25c80, 0x25d7c, 3753 0x25e00, 0x25e04, 3754 0x26000, 0x2602c, 3755 0x26100, 0x2613c, 3756 0x26190, 0x261c8, 3757 0x26200, 0x26318, 3758 0x26400, 0x26528, 3759 0x26540, 0x26614, 3760 0x27000, 0x27040, 3761 0x2704c, 0x27060, 3762 0x270c0, 0x270ec, 3763 0x27200, 0x27268, 3764 0x27270, 0x27284, 3765 0x272fc, 0x27388, 3766 0x27400, 0x27404, 3767 0x27500, 0x27518, 3768 0x2752c, 0x2753c, 3769 0x27550, 0x27554, 3770 0x27600, 0x27600, 3771 0x27608, 0x27628, 3772 0x27630, 0x2763c, 3773 0x27700, 0x2771c, 3774 0x27780, 0x2778c, 3775 0x27800, 0x27c38, 3776 0x27c80, 0x27d7c, 3777 0x27e00, 0x27e04 3778 }; 3779 static const unsigned int t5_reg_ranges[] = { 3780 0x1008, 0x1148, 3781 0x1180, 0x11b4, 3782 0x11fc, 0x123c, 3783 0x1280, 0x173c, 3784 0x1800, 0x18fc, 3785 0x3000, 0x3028, 3786 0x3060, 0x30d8, 3787 0x30e0, 0x30fc, 3788 0x3140, 0x357c, 3789 0x35a8, 0x35cc, 3790 0x35ec, 0x35ec, 3791 0x3600, 0x5624, 3792 0x56cc, 0x575c, 3793 0x580c, 0x5814, 3794 0x5890, 0x58bc, 3795 0x5940, 0x59dc, 3796 0x59fc, 0x5a18, 3797 0x5a60, 0x5a9c, 3798 0x5b94, 0x5bfc, 3799 0x6000, 0x6040, 3800 0x6058, 0x614c, 3801 0x7700, 0x7798, 3802 0x77c0, 0x78fc, 3803 0x7b00, 0x7c54, 3804 0x7d00, 0x7efc, 3805 0x8dc0, 0x8de0, 3806 0x8df8, 0x8e84, 3807 0x8ea0, 0x8f84, 3808 0x8fc0, 0x90f8, 3809 0x9400, 0x9470, 3810 0x9600, 0x96f4, 3811 0x9800, 0x9808, 3812 0x9820, 0x983c, 3813 0x9850, 0x9864, 3814 0x9c00, 0x9c6c, 3815 0x9c80, 0x9cec, 3816 0x9d00, 0x9d6c, 3817 0x9d80, 0x9dec, 3818 0x9e00, 0x9e6c, 3819 0x9e80, 0x9eec, 3820 0x9f00, 0x9f6c, 3821 0x9f80, 0xa020, 3822 0xd004, 0xd03c, 3823 0xdfc0, 0xdfe0, 3824 0xe000, 0x11088, 3825 0x1109c, 0x11110, 3826 0x11118, 0x1117c, 3827 0x11190, 0x11204, 3828 0x19040, 0x1906c, 3829 0x19078, 0x19080, 3830 0x1908c, 0x19124, 3831 0x19150, 0x191b0, 3832 0x191d0, 0x191e8, 3833 0x19238, 0x19290, 3834 0x193f8, 0x19474, 3835 0x19490, 0x194cc, 3836 0x194f0, 0x194f8, 3837 0x19c00, 0x19c60, 3838 0x19c94, 0x19e10, 3839 0x19e50, 0x19f34, 3840 0x19f40, 0x19f50, 3841 0x19f90, 0x19fe4, 3842 0x1a000, 0x1a06c, 3843 0x1a0b0, 0x1a120, 3844 0x1a128, 0x1a138, 3845 0x1a190, 0x1a1c4, 3846 0x1a1fc, 0x1a1fc, 3847 0x1e008, 0x1e00c, 3848 0x1e040, 0x1e04c, 3849 0x1e284, 0x1e290, 3850 0x1e2c0, 0x1e2c0, 3851 0x1e2e0, 0x1e2e0, 3852 0x1e300, 0x1e384, 3853 0x1e3c0, 0x1e3c8, 3854 0x1e408, 0x1e40c, 3855 0x1e440, 0x1e44c, 3856 0x1e684, 0x1e690, 3857 0x1e6c0, 0x1e6c0, 3858 0x1e6e0, 0x1e6e0, 3859 0x1e700, 0x1e784, 3860 0x1e7c0, 0x1e7c8, 3861 0x1e808, 0x1e80c, 3862 0x1e840, 0x1e84c, 3863 0x1ea84, 0x1ea90, 3864 0x1eac0, 0x1eac0, 3865 0x1eae0, 0x1eae0, 3866 0x1eb00, 0x1eb84, 3867 0x1ebc0, 0x1ebc8, 3868 0x1ec08, 0x1ec0c, 3869 0x1ec40, 0x1ec4c, 3870 0x1ee84, 0x1ee90, 3871 0x1eec0, 0x1eec0, 3872 0x1eee0, 0x1eee0, 3873 0x1ef00, 0x1ef84, 3874 0x1efc0, 0x1efc8, 3875 0x1f008, 0x1f00c, 3876 0x1f040, 0x1f04c, 3877 0x1f284, 0x1f290, 3878 0x1f2c0, 0x1f2c0, 3879 0x1f2e0, 0x1f2e0, 3880 0x1f300, 0x1f384, 3881 0x1f3c0, 0x1f3c8, 3882 0x1f408, 0x1f40c, 3883 0x1f440, 0x1f44c, 3884 0x1f684, 0x1f690, 3885 0x1f6c0, 0x1f6c0, 3886 0x1f6e0, 0x1f6e0, 3887 0x1f700, 0x1f784, 3888 0x1f7c0, 0x1f7c8, 3889 0x1f808, 0x1f80c, 3890 0x1f840, 0x1f84c, 3891 0x1fa84, 0x1fa90, 3892 0x1fac0, 0x1fac0, 3893 0x1fae0, 0x1fae0, 3894 0x1fb00, 0x1fb84, 3895 0x1fbc0, 0x1fbc8, 3896 0x1fc08, 0x1fc0c, 3897 0x1fc40, 0x1fc4c, 3898 0x1fe84, 0x1fe90, 3899 0x1fec0, 0x1fec0, 3900 0x1fee0, 0x1fee0, 3901 0x1ff00, 0x1ff84, 3902 0x1ffc0, 0x1ffc8, 3903 0x30000, 0x30030, 3904 0x30100, 0x30144, 3905 0x30190, 0x301d0, 3906 0x30200, 0x30318, 3907 0x30400, 0x3052c, 3908 0x30540, 0x3061c, 3909 0x30800, 0x30834, 3910 0x308c0, 0x30908, 3911 0x30910, 0x309ac, 3912 0x30a00, 0x30a2c, 3913 0x30a44, 0x30a50, 3914 0x30a74, 0x30c24, 3915 0x30d00, 0x30d00, 3916 0x30d08, 0x30d14, 3917 0x30d1c, 0x30d20, 3918 0x30d3c, 0x30d50, 3919 0x31200, 0x3120c, 3920 0x31220, 0x31220, 3921 0x31240, 0x31240, 3922 0x31600, 0x3160c, 3923 0x31a00, 0x31a1c, 3924 0x31e00, 0x31e20, 3925 0x31e38, 0x31e3c, 3926 0x31e80, 0x31e80, 3927 0x31e88, 0x31ea8, 3928 0x31eb0, 0x31eb4, 3929 0x31ec8, 0x31ed4, 3930 0x31fb8, 0x32004, 3931 0x32200, 0x32200, 3932 0x32208, 0x32240, 3933 0x32248, 0x32280, 3934 0x32288, 0x322c0, 3935 0x322c8, 0x322fc, 3936 0x32600, 0x32630, 3937 0x32a00, 0x32abc, 3938 0x32b00, 0x32b70, 3939 0x33000, 0x33048, 3940 0x33060, 0x3309c, 3941 0x330f0, 0x33148, 3942 0x33160, 0x3319c, 3943 0x331f0, 0x332e4, 3944 0x332f8, 0x333e4, 3945 0x333f8, 0x33448, 3946 0x33460, 0x3349c, 3947 0x334f0, 0x33548, 3948 0x33560, 0x3359c, 3949 0x335f0, 0x336e4, 3950 0x336f8, 0x337e4, 3951 0x337f8, 0x337fc, 3952 0x33814, 0x33814, 3953 0x3382c, 0x3382c, 3954 0x33880, 0x3388c, 3955 0x338e8, 0x338ec, 3956 0x33900, 0x33948, 3957 0x33960, 0x3399c, 3958 0x339f0, 0x33ae4, 3959 0x33af8, 0x33b10, 3960 0x33b28, 0x33b28, 3961 0x33b3c, 0x33b50, 3962 0x33bf0, 0x33c10, 3963 0x33c28, 0x33c28, 3964 0x33c3c, 0x33c50, 3965 0x33cf0, 0x33cfc, 3966 0x34000, 0x34030, 3967 0x34100, 0x34144, 3968 0x34190, 0x341d0, 3969 0x34200, 0x34318, 3970 0x34400, 0x3452c, 3971 0x34540, 0x3461c, 3972 0x34800, 0x34834, 3973 0x348c0, 0x34908, 3974 0x34910, 0x349ac, 3975 0x34a00, 0x34a2c, 3976 0x34a44, 0x34a50, 3977 0x34a74, 0x34c24, 3978 0x34d00, 0x34d00, 3979 0x34d08, 0x34d14, 3980 0x34d1c, 0x34d20, 3981 0x34d3c, 0x34d50, 3982 0x35200, 0x3520c, 3983 0x35220, 0x35220, 3984 0x35240, 0x35240, 3985 0x35600, 0x3560c, 3986 0x35a00, 0x35a1c, 3987 0x35e00, 0x35e20, 3988 0x35e38, 0x35e3c, 3989 0x35e80, 0x35e80, 3990 0x35e88, 0x35ea8, 3991 0x35eb0, 0x35eb4, 3992 0x35ec8, 0x35ed4, 3993 0x35fb8, 0x36004, 3994 0x36200, 0x36200, 3995 0x36208, 0x36240, 3996 0x36248, 0x36280, 3997 0x36288, 0x362c0, 3998 0x362c8, 0x362fc, 3999 0x36600, 0x36630, 4000 0x36a00, 0x36abc, 4001 0x36b00, 0x36b70, 4002 0x37000, 0x37048, 4003 0x37060, 0x3709c, 4004 0x370f0, 0x37148, 4005 0x37160, 0x3719c, 4006 0x371f0, 0x372e4, 4007 0x372f8, 0x373e4, 4008 0x373f8, 0x37448, 4009 0x37460, 0x3749c, 4010 0x374f0, 0x37548, 4011 0x37560, 0x3759c, 4012 0x375f0, 0x376e4, 4013 0x376f8, 0x377e4, 4014 0x377f8, 0x377fc, 4015 0x37814, 0x37814, 4016 0x3782c, 0x3782c, 4017 0x37880, 0x3788c, 4018 0x378e8, 0x378ec, 4019 0x37900, 0x37948, 4020 0x37960, 0x3799c, 4021 0x379f0, 0x37ae4, 4022 0x37af8, 0x37b10, 4023 0x37b28, 0x37b28, 4024 0x37b3c, 0x37b50, 4025 0x37bf0, 0x37c10, 4026 0x37c28, 0x37c28, 4027 0x37c3c, 0x37c50, 4028 0x37cf0, 0x37cfc, 4029 0x38000, 0x38030, 4030 0x38100, 0x38144, 4031 0x38190, 0x381d0, 4032 0x38200, 0x38318, 4033 0x38400, 0x3852c, 4034 0x38540, 0x3861c, 4035 0x38800, 0x38834, 4036 0x388c0, 0x38908, 4037 0x38910, 0x389ac, 4038 0x38a00, 0x38a2c, 4039 0x38a44, 0x38a50, 4040 0x38a74, 0x38c24, 4041 0x38d00, 0x38d00, 4042 0x38d08, 0x38d14, 4043 0x38d1c, 0x38d20, 4044 0x38d3c, 0x38d50, 4045 0x39200, 0x3920c, 4046 0x39220, 0x39220, 4047 0x39240, 0x39240, 4048 0x39600, 0x3960c, 4049 0x39a00, 0x39a1c, 4050 0x39e00, 0x39e20, 4051 0x39e38, 0x39e3c, 4052 0x39e80, 0x39e80, 4053 0x39e88, 0x39ea8, 4054 0x39eb0, 0x39eb4, 4055 0x39ec8, 0x39ed4, 4056 0x39fb8, 0x3a004, 4057 0x3a200, 0x3a200, 4058 0x3a208, 0x3a240, 4059 0x3a248, 0x3a280, 4060 0x3a288, 0x3a2c0, 4061 0x3a2c8, 0x3a2fc, 4062 0x3a600, 0x3a630, 4063 0x3aa00, 0x3aabc, 4064 0x3ab00, 0x3ab70, 4065 0x3b000, 0x3b048, 4066 0x3b060, 0x3b09c, 4067 0x3b0f0, 0x3b148, 4068 0x3b160, 0x3b19c, 4069 0x3b1f0, 0x3b2e4, 4070 0x3b2f8, 0x3b3e4, 4071 0x3b3f8, 0x3b448, 4072 0x3b460, 0x3b49c, 4073 0x3b4f0, 0x3b548, 4074 0x3b560, 0x3b59c, 4075 0x3b5f0, 0x3b6e4, 4076 0x3b6f8, 0x3b7e4, 4077 0x3b7f8, 0x3b7fc, 4078 0x3b814, 0x3b814, 4079 0x3b82c, 0x3b82c, 4080 0x3b880, 0x3b88c, 4081 0x3b8e8, 0x3b8ec, 4082 0x3b900, 0x3b948, 4083 0x3b960, 0x3b99c, 4084 0x3b9f0, 0x3bae4, 4085 0x3baf8, 0x3bb10, 4086 0x3bb28, 0x3bb28, 4087 0x3bb3c, 0x3bb50, 4088 0x3bbf0, 0x3bc10, 4089 0x3bc28, 0x3bc28, 4090 0x3bc3c, 0x3bc50, 4091 0x3bcf0, 0x3bcfc, 4092 0x3c000, 0x3c030, 4093 0x3c100, 0x3c144, 4094 0x3c190, 0x3c1d0, 4095 0x3c200, 0x3c318, 4096 0x3c400, 0x3c52c, 4097 0x3c540, 0x3c61c, 4098 0x3c800, 0x3c834, 4099 0x3c8c0, 0x3c908, 4100 0x3c910, 0x3c9ac, 4101 0x3ca00, 0x3ca2c, 4102 0x3ca44, 0x3ca50, 4103 0x3ca74, 0x3cc24, 4104 0x3cd00, 0x3cd00, 4105 0x3cd08, 0x3cd14, 4106 0x3cd1c, 0x3cd20, 4107 0x3cd3c, 0x3cd50, 4108 0x3d200, 0x3d20c, 4109 0x3d220, 0x3d220, 4110 0x3d240, 0x3d240, 4111 0x3d600, 0x3d60c, 4112 0x3da00, 0x3da1c, 4113 0x3de00, 0x3de20, 4114 0x3de38, 0x3de3c, 4115 0x3de80, 0x3de80, 4116 0x3de88, 0x3dea8, 4117 0x3deb0, 0x3deb4, 4118 0x3dec8, 0x3ded4, 4119 0x3dfb8, 0x3e004, 4120 0x3e200, 0x3e200, 4121 0x3e208, 0x3e240, 4122 0x3e248, 0x3e280, 4123 0x3e288, 0x3e2c0, 4124 0x3e2c8, 0x3e2fc, 4125 0x3e600, 0x3e630, 4126 0x3ea00, 0x3eabc, 4127 0x3eb00, 0x3eb70, 4128 0x3f000, 0x3f048, 4129 0x3f060, 0x3f09c, 4130 0x3f0f0, 0x3f148, 4131 0x3f160, 0x3f19c, 4132 0x3f1f0, 0x3f2e4, 4133 0x3f2f8, 0x3f3e4, 4134 0x3f3f8, 0x3f448, 4135 0x3f460, 0x3f49c, 4136 0x3f4f0, 0x3f548, 4137 0x3f560, 0x3f59c, 4138 0x3f5f0, 0x3f6e4, 4139 0x3f6f8, 0x3f7e4, 4140 0x3f7f8, 0x3f7fc, 4141 0x3f814, 0x3f814, 4142 0x3f82c, 0x3f82c, 4143 0x3f880, 0x3f88c, 4144 0x3f8e8, 0x3f8ec, 4145 0x3f900, 0x3f948, 4146 0x3f960, 0x3f99c, 4147 0x3f9f0, 0x3fae4, 4148 0x3faf8, 0x3fb10, 4149 0x3fb28, 0x3fb28, 4150 0x3fb3c, 0x3fb50, 4151 0x3fbf0, 0x3fc10, 4152 0x3fc28, 0x3fc28, 4153 0x3fc3c, 0x3fc50, 4154 0x3fcf0, 0x3fcfc, 4155 0x40000, 0x4000c, 4156 0x40040, 0x40068, 4157 0x4007c, 0x40144, 4158 0x40180, 0x4018c, 4159 0x40200, 0x40298, 4160 0x402ac, 0x4033c, 4161 0x403f8, 0x403fc, 4162 0x41304, 0x413c4, 4163 0x41400, 0x4141c, 4164 0x41480, 0x414d0, 4165 0x44000, 0x44078, 4166 0x440c0, 0x44278, 4167 0x442c0, 0x44478, 4168 0x444c0, 0x44678, 4169 0x446c0, 0x44878, 4170 0x448c0, 0x449fc, 4171 0x45000, 0x45068, 4172 0x45080, 0x45084, 4173 0x450a0, 0x450b0, 4174 0x45200, 0x45268, 4175 0x45280, 0x45284, 4176 0x452a0, 0x452b0, 4177 0x460c0, 0x460e4, 4178 0x47000, 0x4708c, 4179 0x47200, 0x47250, 4180 0x47400, 0x47420, 4181 0x47600, 0x47618, 4182 0x47800, 0x47814, 4183 0x48000, 0x4800c, 4184 0x48040, 0x48068, 4185 0x4807c, 0x48144, 4186 0x48180, 0x4818c, 4187 0x48200, 0x48298, 4188 0x482ac, 0x4833c, 4189 0x483f8, 0x483fc, 4190 0x49304, 0x493c4, 4191 0x49400, 0x4941c, 4192 0x49480, 0x494d0, 4193 0x4c000, 0x4c078, 4194 0x4c0c0, 0x4c278, 4195 0x4c2c0, 0x4c478, 4196 0x4c4c0, 0x4c678, 4197 0x4c6c0, 0x4c878, 4198 0x4c8c0, 0x4c9fc, 4199 0x4d000, 0x4d068, 4200 0x4d080, 0x4d084, 4201 0x4d0a0, 0x4d0b0, 4202 0x4d200, 0x4d268, 4203 0x4d280, 0x4d284, 4204 0x4d2a0, 0x4d2b0, 4205 0x4e0c0, 0x4e0e4, 4206 0x4f000, 0x4f08c, 4207 0x4f200, 0x4f250, 4208 0x4f400, 0x4f420, 4209 0x4f600, 0x4f618, 4210 0x4f800, 0x4f814, 4211 0x50000, 0x500cc, 4212 0x50400, 0x50400, 4213 0x50800, 0x508cc, 4214 0x50c00, 0x50c00, 4215 0x51000, 0x5101c, 4216 0x51300, 0x51308, 4217 }; 4218 4219 if (is_t4(sc)) { 4220 reg_ranges = &t4_reg_ranges[0]; 4221 n = nitems(t4_reg_ranges); 4222 } else { 4223 reg_ranges = &t5_reg_ranges[0]; 4224 n = nitems(t5_reg_ranges); 4225 } 4226 4227 regs->version = chip_id(sc) | chip_rev(sc) << 10; 4228 for (i = 0; i < n; i += 2) 4229 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 4230 } 4231 4232 static void 4233 cxgbe_tick(void *arg) 4234 { 4235 struct port_info *pi = arg; 4236 struct adapter *sc = pi->adapter; 4237 struct ifnet *ifp = pi->ifp; 4238 struct sge_txq *txq; 4239 int i, drops; 4240 struct port_stats *s = &pi->stats; 4241 4242 PORT_LOCK(pi); 4243 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 4244 PORT_UNLOCK(pi); 4245 return; /* without scheduling another callout */ 4246 } 4247 4248 t4_get_port_stats(sc, pi->tx_chan, s); 4249 4250 ifp->if_opackets = s->tx_frames - s->tx_pause; 4251 ifp->if_ipackets = s->rx_frames - s->rx_pause; 4252 ifp->if_obytes = s->tx_octets - s->tx_pause * 64; 4253 ifp->if_ibytes = s->rx_octets - s->rx_pause * 64; 4254 ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause; 4255 ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause; 4256 ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 4257 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 4258 s->rx_trunc3; 4259 for (i = 0; i < 4; i++) { 4260 if (pi->rx_chan_map & (1 << i)) { 4261 uint32_t v; 4262 4263 /* 4264 * XXX: indirect reads from the same ADDR/DATA pair can 4265 * race with each other. 4266 */ 4267 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 4268 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 4269 ifp->if_iqdrops += v; 4270 } 4271 } 4272 4273 drops = s->tx_drop; 4274 for_each_txq(pi, i, txq) 4275 drops += txq->br->br_drops; 4276 ifp->if_oqdrops = drops; 4277 4278 ifp->if_oerrors = s->tx_error_frames; 4279 ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long + 4280 s->rx_fcs_err + s->rx_len_err; 4281 4282 callout_schedule(&pi->tick, hz); 4283 PORT_UNLOCK(pi); 4284 } 4285 4286 static void 4287 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 4288 { 4289 struct ifnet *vlan; 4290 4291 if (arg != ifp || ifp->if_type != IFT_ETHER) 4292 return; 4293 4294 vlan = VLAN_DEVAT(ifp, vid); 4295 VLAN_SETCOOKIE(vlan, ifp); 4296 } 4297 4298 static int 4299 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 4300 { 4301 4302 #ifdef INVARIANTS 4303 panic("%s: opcode 0x%02x on iq %p with payload %p", 4304 __func__, rss->opcode, iq, m); 4305 #else 4306 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 4307 __func__, rss->opcode, iq, m); 4308 m_freem(m); 4309 #endif 4310 return (EDOOFUS); 4311 } 4312 4313 int 4314 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 4315 { 4316 uintptr_t *loc, new; 4317 4318 if (opcode >= nitems(sc->cpl_handler)) 4319 return (EINVAL); 4320 4321 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 4322 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 4323 atomic_store_rel_ptr(loc, new); 4324 4325 return (0); 4326 } 4327 4328 static int 4329 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 4330 { 4331 4332 #ifdef INVARIANTS 4333 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 4334 #else 4335 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 4336 __func__, iq, ctrl); 4337 #endif 4338 return (EDOOFUS); 4339 } 4340 4341 int 4342 t4_register_an_handler(struct adapter *sc, an_handler_t h) 4343 { 4344 uintptr_t *loc, new; 4345 4346 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 4347 loc = (uintptr_t *) &sc->an_handler; 4348 atomic_store_rel_ptr(loc, new); 4349 4350 return (0); 4351 } 4352 4353 static int 4354 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 4355 { 4356 const struct cpl_fw6_msg *cpl = 4357 __containerof(rpl, struct cpl_fw6_msg, data[0]); 4358 4359 #ifdef INVARIANTS 4360 panic("%s: fw_msg type %d", __func__, cpl->type); 4361 #else 4362 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 4363 #endif 4364 return (EDOOFUS); 4365 } 4366 4367 int 4368 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 4369 { 4370 uintptr_t *loc, new; 4371 4372 if (type >= nitems(sc->fw_msg_handler)) 4373 return (EINVAL); 4374 4375 /* 4376 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 4377 * handler dispatch table. Reject any attempt to install a handler for 4378 * this subtype. 4379 */ 4380 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 4381 return (EINVAL); 4382 4383 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 4384 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 4385 atomic_store_rel_ptr(loc, new); 4386 4387 return (0); 4388 } 4389 4390 static int 4391 t4_sysctls(struct adapter *sc) 4392 { 4393 struct sysctl_ctx_list *ctx; 4394 struct sysctl_oid *oid; 4395 struct sysctl_oid_list *children, *c0; 4396 static char *caps[] = { 4397 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 4398 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL" /* caps[1] niccaps */ 4399 "\6HASHFILTER\7ETHOFLD", 4400 "\20\1TOE", /* caps[2] toecaps */ 4401 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 4402 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 4403 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 4404 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 4405 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 4406 "\4PO_INITIAOR\5PO_TARGET" 4407 }; 4408 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 4409 4410 ctx = device_get_sysctl_ctx(sc->dev); 4411 4412 /* 4413 * dev.t4nex.X. 4414 */ 4415 oid = device_get_sysctl_tree(sc->dev); 4416 c0 = children = SYSCTL_CHILDREN(oid); 4417 4418 sc->sc_do_rxcopy = 1; 4419 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, 4420 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); 4421 4422 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 4423 sc->params.nports, "# of ports"); 4424 4425 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 4426 NULL, chip_rev(sc), "chip hardware revision"); 4427 4428 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 4429 CTLFLAG_RD, &sc->fw_version, 0, "firmware version"); 4430 4431 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 4432 CTLFLAG_RD, &sc->cfg_file, 0, "configuration file"); 4433 4434 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 4435 sc->cfcsum, "config file checksum"); 4436 4437 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 4438 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 4439 sysctl_bitfield, "A", "available doorbells"); 4440 4441 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 4442 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 4443 sysctl_bitfield, "A", "available link capabilities"); 4444 4445 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 4446 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 4447 sysctl_bitfield, "A", "available NIC capabilities"); 4448 4449 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 4450 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 4451 sysctl_bitfield, "A", "available TCP offload capabilities"); 4452 4453 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 4454 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 4455 sysctl_bitfield, "A", "available RDMA capabilities"); 4456 4457 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 4458 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 4459 sysctl_bitfield, "A", "available iSCSI capabilities"); 4460 4461 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 4462 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 4463 sysctl_bitfield, "A", "available FCoE capabilities"); 4464 4465 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 4466 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 4467 4468 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 4469 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 4470 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 4471 "interrupt holdoff timer values (us)"); 4472 4473 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 4474 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 4475 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 4476 "interrupt holdoff packet counter values"); 4477 4478 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, 4479 NULL, sc->tids.nftids, "number of filters"); 4480 4481 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | 4482 CTLFLAG_RD, sc, 0, sysctl_temperature, "I", 4483 "chip temperature (in Celsius)"); 4484 4485 t4_sge_sysctls(sc, ctx, children); 4486 4487 sc->lro_timeout = 100; 4488 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, 4489 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); 4490 4491 #ifdef SBUF_DRAIN 4492 /* 4493 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 4494 */ 4495 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 4496 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 4497 "logs and miscellaneous information"); 4498 children = SYSCTL_CHILDREN(oid); 4499 4500 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 4501 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4502 sysctl_cctrl, "A", "congestion control"); 4503 4504 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 4505 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4506 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 4507 4508 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 4509 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 4510 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 4511 4512 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 4513 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 4514 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 4515 4516 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 4517 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 4518 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 4519 4520 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 4521 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 4522 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 4523 4524 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 4525 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 4526 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 4527 4528 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 4529 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4530 sysctl_cim_la, "A", "CIM logic analyzer"); 4531 4532 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", 4533 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4534 sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); 4535 4536 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 4537 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 4538 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 4539 4540 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 4541 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 4542 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 4543 4544 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 4545 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 4546 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 4547 4548 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 4549 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 4550 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 4551 4552 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 4553 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 4554 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 4555 4556 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 4557 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 4558 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 4559 4560 if (is_t5(sc)) { 4561 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 4562 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 4563 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 4564 4565 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 4566 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 4567 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 4568 } 4569 4570 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", 4571 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4572 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); 4573 4574 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 4575 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4576 sysctl_cim_qcfg, "A", "CIM queue configuration"); 4577 4578 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 4579 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4580 sysctl_cpl_stats, "A", "CPL statistics"); 4581 4582 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 4583 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4584 sysctl_ddp_stats, "A", "DDP statistics"); 4585 4586 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 4587 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4588 sysctl_devlog, "A", "firmware's device log"); 4589 4590 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 4591 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4592 sysctl_fcoe_stats, "A", "FCoE statistics"); 4593 4594 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 4595 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4596 sysctl_hw_sched, "A", "hardware scheduler "); 4597 4598 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 4599 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4600 sysctl_l2t, "A", "hardware L2 table"); 4601 4602 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 4603 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4604 sysctl_lb_stats, "A", "loopback statistics"); 4605 4606 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 4607 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4608 sysctl_meminfo, "A", "memory regions"); 4609 4610 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", 4611 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4612 sysctl_mps_tcam, "A", "MPS TCAM entries"); 4613 4614 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 4615 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4616 sysctl_path_mtus, "A", "path MTUs"); 4617 4618 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 4619 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4620 sysctl_pm_stats, "A", "PM statistics"); 4621 4622 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 4623 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4624 sysctl_rdma_stats, "A", "RDMA statistics"); 4625 4626 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 4627 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4628 sysctl_tcp_stats, "A", "TCP statistics"); 4629 4630 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 4631 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4632 sysctl_tids, "A", "TID information"); 4633 4634 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 4635 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4636 sysctl_tp_err_stats, "A", "TP error statistics"); 4637 4638 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", 4639 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4640 sysctl_tp_la, "A", "TP logic analyzer"); 4641 4642 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 4643 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4644 sysctl_tx_rate, "A", "Tx rate"); 4645 4646 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", 4647 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4648 sysctl_ulprx_la, "A", "ULPRX logic analyzer"); 4649 4650 if (is_t5(sc)) { 4651 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 4652 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4653 sysctl_wcwr_stats, "A", "write combined work requests"); 4654 } 4655 #endif 4656 4657 #ifdef TCP_OFFLOAD 4658 if (is_offload(sc)) { 4659 /* 4660 * dev.t4nex.X.toe. 4661 */ 4662 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 4663 NULL, "TOE parameters"); 4664 children = SYSCTL_CHILDREN(oid); 4665 4666 sc->tt.sndbuf = 256 * 1024; 4667 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 4668 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 4669 4670 sc->tt.ddp = 0; 4671 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 4672 &sc->tt.ddp, 0, "DDP allowed"); 4673 4674 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 4675 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 4676 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 4677 4678 sc->tt.ddp_thres = 4679 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 4680 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 4681 &sc->tt.ddp_thres, 0, "DDP threshold"); 4682 4683 sc->tt.rx_coalesce = 1; 4684 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", 4685 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); 4686 } 4687 #endif 4688 4689 4690 return (0); 4691 } 4692 4693 static int 4694 cxgbe_sysctls(struct port_info *pi) 4695 { 4696 struct sysctl_ctx_list *ctx; 4697 struct sysctl_oid *oid; 4698 struct sysctl_oid_list *children; 4699 struct adapter *sc = pi->adapter; 4700 4701 ctx = device_get_sysctl_ctx(pi->dev); 4702 4703 /* 4704 * dev.cxgbe.X. 4705 */ 4706 oid = device_get_sysctl_tree(pi->dev); 4707 children = SYSCTL_CHILDREN(oid); 4708 4709 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | 4710 CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); 4711 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { 4712 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 4713 CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I", 4714 "PHY temperature (in Celsius)"); 4715 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", 4716 CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I", 4717 "PHY firmware version"); 4718 } 4719 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 4720 &pi->nrxq, 0, "# of rx queues"); 4721 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 4722 &pi->ntxq, 0, "# of tx queues"); 4723 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 4724 &pi->first_rxq, 0, "index of first rx queue"); 4725 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 4726 &pi->first_txq, 0, "index of first tx queue"); 4727 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT | 4728 CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU", 4729 "Reserve queue 0 for non-flowid packets"); 4730 4731 #ifdef TCP_OFFLOAD 4732 if (is_offload(sc)) { 4733 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 4734 &pi->nofldrxq, 0, 4735 "# of rx queues for offloaded TCP connections"); 4736 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 4737 &pi->nofldtxq, 0, 4738 "# of tx queues for offloaded TCP connections"); 4739 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 4740 CTLFLAG_RD, &pi->first_ofld_rxq, 0, 4741 "index of first TOE rx queue"); 4742 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 4743 CTLFLAG_RD, &pi->first_ofld_txq, 0, 4744 "index of first TOE tx queue"); 4745 } 4746 #endif 4747 #ifdef DEV_NETMAP 4748 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD, 4749 &pi->nnmrxq, 0, "# of rx queues for netmap"); 4750 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD, 4751 &pi->nnmtxq, 0, "# of tx queues for netmap"); 4752 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq", 4753 CTLFLAG_RD, &pi->first_nm_rxq, 0, 4754 "index of first netmap rx queue"); 4755 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq", 4756 CTLFLAG_RD, &pi->first_nm_txq, 0, 4757 "index of first netmap tx queue"); 4758 #endif 4759 4760 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 4761 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 4762 "holdoff timer index"); 4763 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 4764 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 4765 "holdoff packet counter index"); 4766 4767 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 4768 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 4769 "rx queue size"); 4770 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 4771 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 4772 "tx queue size"); 4773 4774 /* 4775 * dev.cxgbe.X.stats. 4776 */ 4777 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 4778 NULL, "port statistics"); 4779 children = SYSCTL_CHILDREN(oid); 4780 4781 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 4782 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 4783 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ 4784 sysctl_handle_t4_reg64, "QU", desc) 4785 4786 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 4787 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 4788 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 4789 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 4790 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 4791 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 4792 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 4793 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 4794 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 4795 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 4796 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 4797 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 4798 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 4799 "# of tx frames in this range", 4800 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 4801 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 4802 "# of tx frames in this range", 4803 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 4804 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 4805 "# of tx frames in this range", 4806 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 4807 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 4808 "# of tx frames in this range", 4809 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 4810 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 4811 "# of tx frames in this range", 4812 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 4813 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 4814 "# of tx frames in this range", 4815 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 4816 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 4817 "# of tx frames in this range", 4818 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 4819 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 4820 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 4821 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 4822 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 4823 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 4824 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 4825 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 4826 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 4827 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 4828 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 4829 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 4830 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 4831 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 4832 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 4833 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 4834 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 4835 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 4836 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 4837 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 4838 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 4839 4840 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 4841 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 4842 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 4843 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 4844 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 4845 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 4846 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 4847 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 4848 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 4849 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 4850 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 4851 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 4852 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 4853 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 4854 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 4855 "# of frames received with bad FCS", 4856 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 4857 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 4858 "# of frames received with length error", 4859 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 4860 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 4861 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 4862 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 4863 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 4864 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 4865 "# of rx frames in this range", 4866 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 4867 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 4868 "# of rx frames in this range", 4869 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 4870 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 4871 "# of rx frames in this range", 4872 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 4873 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 4874 "# of rx frames in this range", 4875 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 4876 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 4877 "# of rx frames in this range", 4878 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 4879 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 4880 "# of rx frames in this range", 4881 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 4882 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 4883 "# of rx frames in this range", 4884 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 4885 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 4886 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 4887 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 4888 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 4889 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 4890 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 4891 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 4892 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 4893 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 4894 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 4895 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 4896 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 4897 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 4898 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 4899 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 4900 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 4901 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 4902 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 4903 4904 #undef SYSCTL_ADD_T4_REG64 4905 4906 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 4907 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 4908 &pi->stats.name, desc) 4909 4910 /* We get these from port_stats and they may be stale by upto 1s */ 4911 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 4912 "# drops due to buffer-group 0 overflows"); 4913 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 4914 "# drops due to buffer-group 1 overflows"); 4915 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 4916 "# drops due to buffer-group 2 overflows"); 4917 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 4918 "# drops due to buffer-group 3 overflows"); 4919 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 4920 "# of buffer-group 0 truncated packets"); 4921 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 4922 "# of buffer-group 1 truncated packets"); 4923 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 4924 "# of buffer-group 2 truncated packets"); 4925 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 4926 "# of buffer-group 3 truncated packets"); 4927 4928 #undef SYSCTL_ADD_T4_PORTSTAT 4929 4930 return (0); 4931 } 4932 4933 static int 4934 sysctl_int_array(SYSCTL_HANDLER_ARGS) 4935 { 4936 int rc, *i; 4937 struct sbuf sb; 4938 4939 sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND); 4940 for (i = arg1; arg2; arg2 -= sizeof(int), i++) 4941 sbuf_printf(&sb, "%d ", *i); 4942 sbuf_trim(&sb); 4943 sbuf_finish(&sb); 4944 rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 4945 sbuf_delete(&sb); 4946 return (rc); 4947 } 4948 4949 static int 4950 sysctl_bitfield(SYSCTL_HANDLER_ARGS) 4951 { 4952 int rc; 4953 struct sbuf *sb; 4954 4955 rc = sysctl_wire_old_buffer(req, 0); 4956 if (rc != 0) 4957 return(rc); 4958 4959 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 4960 if (sb == NULL) 4961 return (ENOMEM); 4962 4963 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 4964 rc = sbuf_finish(sb); 4965 sbuf_delete(sb); 4966 4967 return (rc); 4968 } 4969 4970 static int 4971 sysctl_btphy(SYSCTL_HANDLER_ARGS) 4972 { 4973 struct port_info *pi = arg1; 4974 int op = arg2; 4975 struct adapter *sc = pi->adapter; 4976 u_int v; 4977 int rc; 4978 4979 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt"); 4980 if (rc) 4981 return (rc); 4982 /* XXX: magic numbers */ 4983 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, 4984 &v); 4985 end_synchronized_op(sc, 0); 4986 if (rc) 4987 return (rc); 4988 if (op == 0) 4989 v /= 256; 4990 4991 rc = sysctl_handle_int(oidp, &v, 0, req); 4992 return (rc); 4993 } 4994 4995 static int 4996 sysctl_noflowq(SYSCTL_HANDLER_ARGS) 4997 { 4998 struct port_info *pi = arg1; 4999 int rc, val; 5000 5001 val = pi->rsrv_noflowq; 5002 rc = sysctl_handle_int(oidp, &val, 0, req); 5003 if (rc != 0 || req->newptr == NULL) 5004 return (rc); 5005 5006 if ((val >= 1) && (pi->ntxq > 1)) 5007 pi->rsrv_noflowq = 1; 5008 else 5009 pi->rsrv_noflowq = 0; 5010 5011 return (rc); 5012 } 5013 5014 static int 5015 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 5016 { 5017 struct port_info *pi = arg1; 5018 struct adapter *sc = pi->adapter; 5019 int idx, rc, i; 5020 struct sge_rxq *rxq; 5021 #ifdef TCP_OFFLOAD 5022 struct sge_ofld_rxq *ofld_rxq; 5023 #endif 5024 uint8_t v; 5025 5026 idx = pi->tmr_idx; 5027 5028 rc = sysctl_handle_int(oidp, &idx, 0, req); 5029 if (rc != 0 || req->newptr == NULL) 5030 return (rc); 5031 5032 if (idx < 0 || idx >= SGE_NTIMERS) 5033 return (EINVAL); 5034 5035 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5036 "t4tmr"); 5037 if (rc) 5038 return (rc); 5039 5040 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 5041 for_each_rxq(pi, i, rxq) { 5042 #ifdef atomic_store_rel_8 5043 atomic_store_rel_8(&rxq->iq.intr_params, v); 5044 #else 5045 rxq->iq.intr_params = v; 5046 #endif 5047 } 5048 #ifdef TCP_OFFLOAD 5049 for_each_ofld_rxq(pi, i, ofld_rxq) { 5050 #ifdef atomic_store_rel_8 5051 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); 5052 #else 5053 ofld_rxq->iq.intr_params = v; 5054 #endif 5055 } 5056 #endif 5057 pi->tmr_idx = idx; 5058 5059 end_synchronized_op(sc, LOCK_HELD); 5060 return (0); 5061 } 5062 5063 static int 5064 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 5065 { 5066 struct port_info *pi = arg1; 5067 struct adapter *sc = pi->adapter; 5068 int idx, rc; 5069 5070 idx = pi->pktc_idx; 5071 5072 rc = sysctl_handle_int(oidp, &idx, 0, req); 5073 if (rc != 0 || req->newptr == NULL) 5074 return (rc); 5075 5076 if (idx < -1 || idx >= SGE_NCOUNTERS) 5077 return (EINVAL); 5078 5079 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5080 "t4pktc"); 5081 if (rc) 5082 return (rc); 5083 5084 if (pi->flags & PORT_INIT_DONE) 5085 rc = EBUSY; /* cannot be changed once the queues are created */ 5086 else 5087 pi->pktc_idx = idx; 5088 5089 end_synchronized_op(sc, LOCK_HELD); 5090 return (rc); 5091 } 5092 5093 static int 5094 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 5095 { 5096 struct port_info *pi = arg1; 5097 struct adapter *sc = pi->adapter; 5098 int qsize, rc; 5099 5100 qsize = pi->qsize_rxq; 5101 5102 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5103 if (rc != 0 || req->newptr == NULL) 5104 return (rc); 5105 5106 if (qsize < 128 || (qsize & 7)) 5107 return (EINVAL); 5108 5109 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5110 "t4rxqs"); 5111 if (rc) 5112 return (rc); 5113 5114 if (pi->flags & PORT_INIT_DONE) 5115 rc = EBUSY; /* cannot be changed once the queues are created */ 5116 else 5117 pi->qsize_rxq = qsize; 5118 5119 end_synchronized_op(sc, LOCK_HELD); 5120 return (rc); 5121 } 5122 5123 static int 5124 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 5125 { 5126 struct port_info *pi = arg1; 5127 struct adapter *sc = pi->adapter; 5128 int qsize, rc; 5129 5130 qsize = pi->qsize_txq; 5131 5132 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5133 if (rc != 0 || req->newptr == NULL) 5134 return (rc); 5135 5136 /* bufring size must be powerof2 */ 5137 if (qsize < 128 || !powerof2(qsize)) 5138 return (EINVAL); 5139 5140 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5141 "t4txqs"); 5142 if (rc) 5143 return (rc); 5144 5145 if (pi->flags & PORT_INIT_DONE) 5146 rc = EBUSY; /* cannot be changed once the queues are created */ 5147 else 5148 pi->qsize_txq = qsize; 5149 5150 end_synchronized_op(sc, LOCK_HELD); 5151 return (rc); 5152 } 5153 5154 static int 5155 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 5156 { 5157 struct adapter *sc = arg1; 5158 int reg = arg2; 5159 uint64_t val; 5160 5161 val = t4_read_reg64(sc, reg); 5162 5163 return (sysctl_handle_64(oidp, &val, 0, req)); 5164 } 5165 5166 static int 5167 sysctl_temperature(SYSCTL_HANDLER_ARGS) 5168 { 5169 struct adapter *sc = arg1; 5170 int rc, t; 5171 uint32_t param, val; 5172 5173 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); 5174 if (rc) 5175 return (rc); 5176 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 5177 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 5178 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); 5179 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 5180 end_synchronized_op(sc, 0); 5181 if (rc) 5182 return (rc); 5183 5184 /* unknown is returned as 0 but we display -1 in that case */ 5185 t = val == 0 ? -1 : val; 5186 5187 rc = sysctl_handle_int(oidp, &t, 0, req); 5188 return (rc); 5189 } 5190 5191 #ifdef SBUF_DRAIN 5192 static int 5193 sysctl_cctrl(SYSCTL_HANDLER_ARGS) 5194 { 5195 struct adapter *sc = arg1; 5196 struct sbuf *sb; 5197 int rc, i; 5198 uint16_t incr[NMTUS][NCCTRL_WIN]; 5199 static const char *dec_fac[] = { 5200 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 5201 "0.9375" 5202 }; 5203 5204 rc = sysctl_wire_old_buffer(req, 0); 5205 if (rc != 0) 5206 return (rc); 5207 5208 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5209 if (sb == NULL) 5210 return (ENOMEM); 5211 5212 t4_read_cong_tbl(sc, incr); 5213 5214 for (i = 0; i < NCCTRL_WIN; ++i) { 5215 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 5216 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 5217 incr[5][i], incr[6][i], incr[7][i]); 5218 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 5219 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 5220 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 5221 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 5222 } 5223 5224 rc = sbuf_finish(sb); 5225 sbuf_delete(sb); 5226 5227 return (rc); 5228 } 5229 5230 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 5231 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 5232 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 5233 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 5234 }; 5235 5236 static int 5237 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 5238 { 5239 struct adapter *sc = arg1; 5240 struct sbuf *sb; 5241 int rc, i, n, qid = arg2; 5242 uint32_t *buf, *p; 5243 char *qtype; 5244 u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5; 5245 5246 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 5247 ("%s: bad qid %d\n", __func__, qid)); 5248 5249 if (qid < CIM_NUM_IBQ) { 5250 /* inbound queue */ 5251 qtype = "IBQ"; 5252 n = 4 * CIM_IBQ_SIZE; 5253 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5254 rc = t4_read_cim_ibq(sc, qid, buf, n); 5255 } else { 5256 /* outbound queue */ 5257 qtype = "OBQ"; 5258 qid -= CIM_NUM_IBQ; 5259 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 5260 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5261 rc = t4_read_cim_obq(sc, qid, buf, n); 5262 } 5263 5264 if (rc < 0) { 5265 rc = -rc; 5266 goto done; 5267 } 5268 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 5269 5270 rc = sysctl_wire_old_buffer(req, 0); 5271 if (rc != 0) 5272 goto done; 5273 5274 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5275 if (sb == NULL) { 5276 rc = ENOMEM; 5277 goto done; 5278 } 5279 5280 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 5281 for (i = 0, p = buf; i < n; i += 16, p += 4) 5282 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 5283 p[2], p[3]); 5284 5285 rc = sbuf_finish(sb); 5286 sbuf_delete(sb); 5287 done: 5288 free(buf, M_CXGBE); 5289 return (rc); 5290 } 5291 5292 static int 5293 sysctl_cim_la(SYSCTL_HANDLER_ARGS) 5294 { 5295 struct adapter *sc = arg1; 5296 u_int cfg; 5297 struct sbuf *sb; 5298 uint32_t *buf, *p; 5299 int rc; 5300 5301 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 5302 if (rc != 0) 5303 return (rc); 5304 5305 rc = sysctl_wire_old_buffer(req, 0); 5306 if (rc != 0) 5307 return (rc); 5308 5309 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5310 if (sb == NULL) 5311 return (ENOMEM); 5312 5313 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 5314 M_ZERO | M_WAITOK); 5315 5316 rc = -t4_cim_read_la(sc, buf, NULL); 5317 if (rc != 0) 5318 goto done; 5319 5320 sbuf_printf(sb, "Status Data PC%s", 5321 cfg & F_UPDBGLACAPTPCONLY ? "" : 5322 " LS0Stat LS0Addr LS0Data"); 5323 5324 KASSERT((sc->params.cim_la_size & 7) == 0, 5325 ("%s: p will walk off the end of buf", __func__)); 5326 5327 for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) { 5328 if (cfg & F_UPDBGLACAPTPCONLY) { 5329 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 5330 p[6], p[7]); 5331 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 5332 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 5333 p[4] & 0xff, p[5] >> 8); 5334 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 5335 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5336 p[1] & 0xf, p[2] >> 4); 5337 } else { 5338 sbuf_printf(sb, 5339 "\n %02x %x%07x %x%07x %08x %08x " 5340 "%08x%08x%08x%08x", 5341 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5342 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 5343 p[6], p[7]); 5344 } 5345 } 5346 5347 rc = sbuf_finish(sb); 5348 sbuf_delete(sb); 5349 done: 5350 free(buf, M_CXGBE); 5351 return (rc); 5352 } 5353 5354 static int 5355 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) 5356 { 5357 struct adapter *sc = arg1; 5358 u_int i; 5359 struct sbuf *sb; 5360 uint32_t *buf, *p; 5361 int rc; 5362 5363 rc = sysctl_wire_old_buffer(req, 0); 5364 if (rc != 0) 5365 return (rc); 5366 5367 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5368 if (sb == NULL) 5369 return (ENOMEM); 5370 5371 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, 5372 M_ZERO | M_WAITOK); 5373 5374 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); 5375 p = buf; 5376 5377 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5378 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], 5379 p[1], p[0]); 5380 } 5381 5382 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); 5383 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5384 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", 5385 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, 5386 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, 5387 (p[1] >> 2) | ((p[2] & 3) << 30), 5388 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, 5389 p[0] & 1); 5390 } 5391 5392 rc = sbuf_finish(sb); 5393 sbuf_delete(sb); 5394 free(buf, M_CXGBE); 5395 return (rc); 5396 } 5397 5398 static int 5399 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) 5400 { 5401 struct adapter *sc = arg1; 5402 u_int i; 5403 struct sbuf *sb; 5404 uint32_t *buf, *p; 5405 int rc; 5406 5407 rc = sysctl_wire_old_buffer(req, 0); 5408 if (rc != 0) 5409 return (rc); 5410 5411 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5412 if (sb == NULL) 5413 return (ENOMEM); 5414 5415 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, 5416 M_ZERO | M_WAITOK); 5417 5418 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); 5419 p = buf; 5420 5421 sbuf_printf(sb, "Cntl ID DataBE Addr Data"); 5422 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5423 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", 5424 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, 5425 p[4], p[3], p[2], p[1], p[0]); 5426 } 5427 5428 sbuf_printf(sb, "\n\nCntl ID Data"); 5429 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5430 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", 5431 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); 5432 } 5433 5434 rc = sbuf_finish(sb); 5435 sbuf_delete(sb); 5436 free(buf, M_CXGBE); 5437 return (rc); 5438 } 5439 5440 static int 5441 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 5442 { 5443 struct adapter *sc = arg1; 5444 struct sbuf *sb; 5445 int rc, i; 5446 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5447 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5448 uint16_t thres[CIM_NUM_IBQ]; 5449 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 5450 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 5451 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 5452 5453 if (is_t4(sc)) { 5454 cim_num_obq = CIM_NUM_OBQ; 5455 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 5456 obq_rdaddr = A_UP_OBQ_0_REALADDR; 5457 } else { 5458 cim_num_obq = CIM_NUM_OBQ_T5; 5459 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 5460 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 5461 } 5462 nq = CIM_NUM_IBQ + cim_num_obq; 5463 5464 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 5465 if (rc == 0) 5466 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 5467 if (rc != 0) 5468 return (rc); 5469 5470 t4_read_cimq_cfg(sc, base, size, thres); 5471 5472 rc = sysctl_wire_old_buffer(req, 0); 5473 if (rc != 0) 5474 return (rc); 5475 5476 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5477 if (sb == NULL) 5478 return (ENOMEM); 5479 5480 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 5481 5482 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 5483 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 5484 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 5485 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5486 G_QUEREMFLITS(p[2]) * 16); 5487 for ( ; i < nq; i++, p += 4, wr += 2) 5488 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 5489 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 5490 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5491 G_QUEREMFLITS(p[2]) * 16); 5492 5493 rc = sbuf_finish(sb); 5494 sbuf_delete(sb); 5495 5496 return (rc); 5497 } 5498 5499 static int 5500 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 5501 { 5502 struct adapter *sc = arg1; 5503 struct sbuf *sb; 5504 int rc; 5505 struct tp_cpl_stats stats; 5506 5507 rc = sysctl_wire_old_buffer(req, 0); 5508 if (rc != 0) 5509 return (rc); 5510 5511 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5512 if (sb == NULL) 5513 return (ENOMEM); 5514 5515 t4_tp_get_cpl_stats(sc, &stats); 5516 5517 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 5518 "channel 3\n"); 5519 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 5520 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 5521 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 5522 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 5523 5524 rc = sbuf_finish(sb); 5525 sbuf_delete(sb); 5526 5527 return (rc); 5528 } 5529 5530 static int 5531 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 5532 { 5533 struct adapter *sc = arg1; 5534 struct sbuf *sb; 5535 int rc; 5536 struct tp_usm_stats stats; 5537 5538 rc = sysctl_wire_old_buffer(req, 0); 5539 if (rc != 0) 5540 return(rc); 5541 5542 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5543 if (sb == NULL) 5544 return (ENOMEM); 5545 5546 t4_get_usm_stats(sc, &stats); 5547 5548 sbuf_printf(sb, "Frames: %u\n", stats.frames); 5549 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 5550 sbuf_printf(sb, "Drops: %u", stats.drops); 5551 5552 rc = sbuf_finish(sb); 5553 sbuf_delete(sb); 5554 5555 return (rc); 5556 } 5557 5558 const char *devlog_level_strings[] = { 5559 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 5560 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 5561 [FW_DEVLOG_LEVEL_ERR] = "ERR", 5562 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 5563 [FW_DEVLOG_LEVEL_INFO] = "INFO", 5564 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 5565 }; 5566 5567 const char *devlog_facility_strings[] = { 5568 [FW_DEVLOG_FACILITY_CORE] = "CORE", 5569 [FW_DEVLOG_FACILITY_CF] = "CF", 5570 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 5571 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 5572 [FW_DEVLOG_FACILITY_RES] = "RES", 5573 [FW_DEVLOG_FACILITY_HW] = "HW", 5574 [FW_DEVLOG_FACILITY_FLR] = "FLR", 5575 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 5576 [FW_DEVLOG_FACILITY_PHY] = "PHY", 5577 [FW_DEVLOG_FACILITY_MAC] = "MAC", 5578 [FW_DEVLOG_FACILITY_PORT] = "PORT", 5579 [FW_DEVLOG_FACILITY_VI] = "VI", 5580 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 5581 [FW_DEVLOG_FACILITY_ACL] = "ACL", 5582 [FW_DEVLOG_FACILITY_TM] = "TM", 5583 [FW_DEVLOG_FACILITY_QFC] = "QFC", 5584 [FW_DEVLOG_FACILITY_DCB] = "DCB", 5585 [FW_DEVLOG_FACILITY_ETH] = "ETH", 5586 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 5587 [FW_DEVLOG_FACILITY_RI] = "RI", 5588 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 5589 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 5590 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 5591 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 5592 }; 5593 5594 static int 5595 sysctl_devlog(SYSCTL_HANDLER_ARGS) 5596 { 5597 struct adapter *sc = arg1; 5598 struct devlog_params *dparams = &sc->params.devlog; 5599 struct fw_devlog_e *buf, *e; 5600 int i, j, rc, nentries, first = 0, m; 5601 struct sbuf *sb; 5602 uint64_t ftstamp = UINT64_MAX; 5603 5604 if (dparams->start == 0) { 5605 dparams->memtype = FW_MEMTYPE_EDC0; 5606 dparams->start = 0x84000; 5607 dparams->size = 32768; 5608 } 5609 5610 nentries = dparams->size / sizeof(struct fw_devlog_e); 5611 5612 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 5613 if (buf == NULL) 5614 return (ENOMEM); 5615 5616 m = fwmtype_to_hwmtype(dparams->memtype); 5617 rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf); 5618 if (rc != 0) 5619 goto done; 5620 5621 for (i = 0; i < nentries; i++) { 5622 e = &buf[i]; 5623 5624 if (e->timestamp == 0) 5625 break; /* end */ 5626 5627 e->timestamp = be64toh(e->timestamp); 5628 e->seqno = be32toh(e->seqno); 5629 for (j = 0; j < 8; j++) 5630 e->params[j] = be32toh(e->params[j]); 5631 5632 if (e->timestamp < ftstamp) { 5633 ftstamp = e->timestamp; 5634 first = i; 5635 } 5636 } 5637 5638 if (buf[first].timestamp == 0) 5639 goto done; /* nothing in the log */ 5640 5641 rc = sysctl_wire_old_buffer(req, 0); 5642 if (rc != 0) 5643 goto done; 5644 5645 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5646 if (sb == NULL) { 5647 rc = ENOMEM; 5648 goto done; 5649 } 5650 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 5651 "Seq#", "Tstamp", "Level", "Facility", "Message"); 5652 5653 i = first; 5654 do { 5655 e = &buf[i]; 5656 if (e->timestamp == 0) 5657 break; /* end */ 5658 5659 sbuf_printf(sb, "%10d %15ju %8s %8s ", 5660 e->seqno, e->timestamp, 5661 (e->level < nitems(devlog_level_strings) ? 5662 devlog_level_strings[e->level] : "UNKNOWN"), 5663 (e->facility < nitems(devlog_facility_strings) ? 5664 devlog_facility_strings[e->facility] : "UNKNOWN")); 5665 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 5666 e->params[2], e->params[3], e->params[4], 5667 e->params[5], e->params[6], e->params[7]); 5668 5669 if (++i == nentries) 5670 i = 0; 5671 } while (i != first); 5672 5673 rc = sbuf_finish(sb); 5674 sbuf_delete(sb); 5675 done: 5676 free(buf, M_CXGBE); 5677 return (rc); 5678 } 5679 5680 static int 5681 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 5682 { 5683 struct adapter *sc = arg1; 5684 struct sbuf *sb; 5685 int rc; 5686 struct tp_fcoe_stats stats[4]; 5687 5688 rc = sysctl_wire_old_buffer(req, 0); 5689 if (rc != 0) 5690 return (rc); 5691 5692 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5693 if (sb == NULL) 5694 return (ENOMEM); 5695 5696 t4_get_fcoe_stats(sc, 0, &stats[0]); 5697 t4_get_fcoe_stats(sc, 1, &stats[1]); 5698 t4_get_fcoe_stats(sc, 2, &stats[2]); 5699 t4_get_fcoe_stats(sc, 3, &stats[3]); 5700 5701 sbuf_printf(sb, " channel 0 channel 1 " 5702 "channel 2 channel 3\n"); 5703 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 5704 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 5705 stats[3].octetsDDP); 5706 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 5707 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 5708 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 5709 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 5710 stats[3].framesDrop); 5711 5712 rc = sbuf_finish(sb); 5713 sbuf_delete(sb); 5714 5715 return (rc); 5716 } 5717 5718 static int 5719 sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 5720 { 5721 struct adapter *sc = arg1; 5722 struct sbuf *sb; 5723 int rc, i; 5724 unsigned int map, kbps, ipg, mode; 5725 unsigned int pace_tab[NTX_SCHED]; 5726 5727 rc = sysctl_wire_old_buffer(req, 0); 5728 if (rc != 0) 5729 return (rc); 5730 5731 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5732 if (sb == NULL) 5733 return (ENOMEM); 5734 5735 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 5736 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 5737 t4_read_pace_tbl(sc, pace_tab); 5738 5739 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 5740 "Class IPG (0.1 ns) Flow IPG (us)"); 5741 5742 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 5743 t4_get_tx_sched(sc, i, &kbps, &ipg); 5744 sbuf_printf(sb, "\n %u %-5s %u ", i, 5745 (mode & (1 << i)) ? "flow" : "class", map & 3); 5746 if (kbps) 5747 sbuf_printf(sb, "%9u ", kbps); 5748 else 5749 sbuf_printf(sb, " disabled "); 5750 5751 if (ipg) 5752 sbuf_printf(sb, "%13u ", ipg); 5753 else 5754 sbuf_printf(sb, " disabled "); 5755 5756 if (pace_tab[i]) 5757 sbuf_printf(sb, "%10u", pace_tab[i]); 5758 else 5759 sbuf_printf(sb, " disabled"); 5760 } 5761 5762 rc = sbuf_finish(sb); 5763 sbuf_delete(sb); 5764 5765 return (rc); 5766 } 5767 5768 static int 5769 sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 5770 { 5771 struct adapter *sc = arg1; 5772 struct sbuf *sb; 5773 int rc, i, j; 5774 uint64_t *p0, *p1; 5775 struct lb_port_stats s[2]; 5776 static const char *stat_name[] = { 5777 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 5778 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 5779 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 5780 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 5781 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 5782 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 5783 "BG2FramesTrunc:", "BG3FramesTrunc:" 5784 }; 5785 5786 rc = sysctl_wire_old_buffer(req, 0); 5787 if (rc != 0) 5788 return (rc); 5789 5790 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5791 if (sb == NULL) 5792 return (ENOMEM); 5793 5794 memset(s, 0, sizeof(s)); 5795 5796 for (i = 0; i < 4; i += 2) { 5797 t4_get_lb_stats(sc, i, &s[0]); 5798 t4_get_lb_stats(sc, i + 1, &s[1]); 5799 5800 p0 = &s[0].octets; 5801 p1 = &s[1].octets; 5802 sbuf_printf(sb, "%s Loopback %u" 5803 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 5804 5805 for (j = 0; j < nitems(stat_name); j++) 5806 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 5807 *p0++, *p1++); 5808 } 5809 5810 rc = sbuf_finish(sb); 5811 sbuf_delete(sb); 5812 5813 return (rc); 5814 } 5815 5816 static int 5817 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) 5818 { 5819 int rc = 0; 5820 struct port_info *pi = arg1; 5821 struct sbuf *sb; 5822 static const char *linkdnreasons[] = { 5823 "non-specific", "remote fault", "autoneg failed", "reserved3", 5824 "PHY overheated", "unknown", "rx los", "reserved7" 5825 }; 5826 5827 rc = sysctl_wire_old_buffer(req, 0); 5828 if (rc != 0) 5829 return(rc); 5830 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); 5831 if (sb == NULL) 5832 return (ENOMEM); 5833 5834 if (pi->linkdnrc < 0) 5835 sbuf_printf(sb, "n/a"); 5836 else if (pi->linkdnrc < nitems(linkdnreasons)) 5837 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]); 5838 else 5839 sbuf_printf(sb, "%d", pi->linkdnrc); 5840 5841 rc = sbuf_finish(sb); 5842 sbuf_delete(sb); 5843 5844 return (rc); 5845 } 5846 5847 struct mem_desc { 5848 unsigned int base; 5849 unsigned int limit; 5850 unsigned int idx; 5851 }; 5852 5853 static int 5854 mem_desc_cmp(const void *a, const void *b) 5855 { 5856 return ((const struct mem_desc *)a)->base - 5857 ((const struct mem_desc *)b)->base; 5858 } 5859 5860 static void 5861 mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 5862 unsigned int to) 5863 { 5864 unsigned int size; 5865 5866 size = to - from + 1; 5867 if (size == 0) 5868 return; 5869 5870 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 5871 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 5872 } 5873 5874 static int 5875 sysctl_meminfo(SYSCTL_HANDLER_ARGS) 5876 { 5877 struct adapter *sc = arg1; 5878 struct sbuf *sb; 5879 int rc, i, n; 5880 uint32_t lo, hi, used, alloc; 5881 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 5882 static const char *region[] = { 5883 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 5884 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 5885 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 5886 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 5887 "RQUDP region:", "PBL region:", "TXPBL region:", 5888 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 5889 "On-chip queues:" 5890 }; 5891 struct mem_desc avail[4]; 5892 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 5893 struct mem_desc *md = mem; 5894 5895 rc = sysctl_wire_old_buffer(req, 0); 5896 if (rc != 0) 5897 return (rc); 5898 5899 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5900 if (sb == NULL) 5901 return (ENOMEM); 5902 5903 for (i = 0; i < nitems(mem); i++) { 5904 mem[i].limit = 0; 5905 mem[i].idx = i; 5906 } 5907 5908 /* Find and sort the populated memory ranges */ 5909 i = 0; 5910 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 5911 if (lo & F_EDRAM0_ENABLE) { 5912 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 5913 avail[i].base = G_EDRAM0_BASE(hi) << 20; 5914 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 5915 avail[i].idx = 0; 5916 i++; 5917 } 5918 if (lo & F_EDRAM1_ENABLE) { 5919 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 5920 avail[i].base = G_EDRAM1_BASE(hi) << 20; 5921 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 5922 avail[i].idx = 1; 5923 i++; 5924 } 5925 if (lo & F_EXT_MEM_ENABLE) { 5926 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 5927 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 5928 avail[i].limit = avail[i].base + 5929 (G_EXT_MEM_SIZE(hi) << 20); 5930 avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */ 5931 i++; 5932 } 5933 if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) { 5934 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 5935 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 5936 avail[i].limit = avail[i].base + 5937 (G_EXT_MEM1_SIZE(hi) << 20); 5938 avail[i].idx = 4; 5939 i++; 5940 } 5941 if (!i) /* no memory available */ 5942 return 0; 5943 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 5944 5945 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 5946 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 5947 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 5948 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 5949 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 5950 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 5951 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 5952 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 5953 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 5954 5955 /* the next few have explicit upper bounds */ 5956 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 5957 md->limit = md->base - 1 + 5958 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 5959 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 5960 md++; 5961 5962 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 5963 md->limit = md->base - 1 + 5964 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 5965 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 5966 md++; 5967 5968 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 5969 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 5970 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 5971 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 5972 } else { 5973 md->base = 0; 5974 md->idx = nitems(region); /* hide it */ 5975 } 5976 md++; 5977 5978 #define ulp_region(reg) \ 5979 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 5980 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 5981 5982 ulp_region(RX_ISCSI); 5983 ulp_region(RX_TDDP); 5984 ulp_region(TX_TPT); 5985 ulp_region(RX_STAG); 5986 ulp_region(RX_RQ); 5987 ulp_region(RX_RQUDP); 5988 ulp_region(RX_PBL); 5989 ulp_region(TX_PBL); 5990 #undef ulp_region 5991 5992 md->base = 0; 5993 md->idx = nitems(region); 5994 if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) { 5995 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); 5996 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc, 5997 A_SGE_DBVFIFO_SIZE))) << 2) - 1; 5998 } 5999 md++; 6000 6001 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 6002 md->limit = md->base + sc->tids.ntids - 1; 6003 md++; 6004 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 6005 md->limit = md->base + sc->tids.ntids - 1; 6006 md++; 6007 6008 md->base = sc->vres.ocq.start; 6009 if (sc->vres.ocq.size) 6010 md->limit = md->base + sc->vres.ocq.size - 1; 6011 else 6012 md->idx = nitems(region); /* hide it */ 6013 md++; 6014 6015 /* add any address-space holes, there can be up to 3 */ 6016 for (n = 0; n < i - 1; n++) 6017 if (avail[n].limit < avail[n + 1].base) 6018 (md++)->base = avail[n].limit; 6019 if (avail[n].limit) 6020 (md++)->base = avail[n].limit; 6021 6022 n = md - mem; 6023 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 6024 6025 for (lo = 0; lo < i; lo++) 6026 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 6027 avail[lo].limit - 1); 6028 6029 sbuf_printf(sb, "\n"); 6030 for (i = 0; i < n; i++) { 6031 if (mem[i].idx >= nitems(region)) 6032 continue; /* skip holes */ 6033 if (!mem[i].limit) 6034 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 6035 mem_region_show(sb, region[mem[i].idx], mem[i].base, 6036 mem[i].limit); 6037 } 6038 6039 sbuf_printf(sb, "\n"); 6040 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 6041 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 6042 mem_region_show(sb, "uP RAM:", lo, hi); 6043 6044 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 6045 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 6046 mem_region_show(sb, "uP Extmem2:", lo, hi); 6047 6048 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 6049 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 6050 G_PMRXMAXPAGE(lo), 6051 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 6052 (lo & F_PMRXNUMCHN) ? 2 : 1); 6053 6054 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 6055 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 6056 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 6057 G_PMTXMAXPAGE(lo), 6058 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 6059 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 6060 sbuf_printf(sb, "%u p-structs\n", 6061 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 6062 6063 for (i = 0; i < 4; i++) { 6064 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 6065 if (is_t4(sc)) { 6066 used = G_USED(lo); 6067 alloc = G_ALLOC(lo); 6068 } else { 6069 used = G_T5_USED(lo); 6070 alloc = G_T5_ALLOC(lo); 6071 } 6072 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 6073 i, used, alloc); 6074 } 6075 for (i = 0; i < 4; i++) { 6076 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 6077 if (is_t4(sc)) { 6078 used = G_USED(lo); 6079 alloc = G_ALLOC(lo); 6080 } else { 6081 used = G_T5_USED(lo); 6082 alloc = G_T5_ALLOC(lo); 6083 } 6084 sbuf_printf(sb, 6085 "\nLoopback %d using %u pages out of %u allocated", 6086 i, used, alloc); 6087 } 6088 6089 rc = sbuf_finish(sb); 6090 sbuf_delete(sb); 6091 6092 return (rc); 6093 } 6094 6095 static inline void 6096 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) 6097 { 6098 *mask = x | y; 6099 y = htobe64(y); 6100 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); 6101 } 6102 6103 static int 6104 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) 6105 { 6106 struct adapter *sc = arg1; 6107 struct sbuf *sb; 6108 int rc, i, n; 6109 6110 rc = sysctl_wire_old_buffer(req, 0); 6111 if (rc != 0) 6112 return (rc); 6113 6114 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6115 if (sb == NULL) 6116 return (ENOMEM); 6117 6118 sbuf_printf(sb, 6119 "Idx Ethernet address Mask Vld Ports PF" 6120 " VF Replication P0 P1 P2 P3 ML"); 6121 n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES : 6122 NUM_MPS_T5_CLS_SRAM_L_INSTANCES; 6123 for (i = 0; i < n; i++) { 6124 uint64_t tcamx, tcamy, mask; 6125 uint32_t cls_lo, cls_hi; 6126 uint8_t addr[ETHER_ADDR_LEN]; 6127 6128 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); 6129 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); 6130 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 6131 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 6132 6133 if (tcamx & tcamy) 6134 continue; 6135 6136 tcamxy2valmask(tcamx, tcamy, addr, &mask); 6137 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" 6138 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], 6139 addr[3], addr[4], addr[5], (uintmax_t)mask, 6140 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', 6141 G_PORTMAP(cls_hi), G_PF(cls_lo), 6142 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); 6143 6144 if (cls_lo & F_REPLICATE) { 6145 struct fw_ldst_cmd ldst_cmd; 6146 6147 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 6148 ldst_cmd.op_to_addrspace = 6149 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 6150 F_FW_CMD_REQUEST | F_FW_CMD_READ | 6151 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 6152 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 6153 ldst_cmd.u.mps.fid_ctl = 6154 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 6155 V_FW_LDST_CMD_CTL(i)); 6156 6157 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 6158 "t4mps"); 6159 if (rc) 6160 break; 6161 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 6162 sizeof(ldst_cmd), &ldst_cmd); 6163 end_synchronized_op(sc, 0); 6164 6165 if (rc != 0) { 6166 sbuf_printf(sb, 6167 " ------------ error %3u ------------", rc); 6168 rc = 0; 6169 } else { 6170 sbuf_printf(sb, " %08x %08x %08x %08x", 6171 be32toh(ldst_cmd.u.mps.rplc127_96), 6172 be32toh(ldst_cmd.u.mps.rplc95_64), 6173 be32toh(ldst_cmd.u.mps.rplc63_32), 6174 be32toh(ldst_cmd.u.mps.rplc31_0)); 6175 } 6176 } else 6177 sbuf_printf(sb, "%36s", ""); 6178 6179 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), 6180 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), 6181 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); 6182 } 6183 6184 if (rc) 6185 (void) sbuf_finish(sb); 6186 else 6187 rc = sbuf_finish(sb); 6188 sbuf_delete(sb); 6189 6190 return (rc); 6191 } 6192 6193 static int 6194 sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 6195 { 6196 struct adapter *sc = arg1; 6197 struct sbuf *sb; 6198 int rc; 6199 uint16_t mtus[NMTUS]; 6200 6201 rc = sysctl_wire_old_buffer(req, 0); 6202 if (rc != 0) 6203 return (rc); 6204 6205 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6206 if (sb == NULL) 6207 return (ENOMEM); 6208 6209 t4_read_mtu_tbl(sc, mtus, NULL); 6210 6211 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 6212 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 6213 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 6214 mtus[14], mtus[15]); 6215 6216 rc = sbuf_finish(sb); 6217 sbuf_delete(sb); 6218 6219 return (rc); 6220 } 6221 6222 static int 6223 sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 6224 { 6225 struct adapter *sc = arg1; 6226 struct sbuf *sb; 6227 int rc, i; 6228 uint32_t cnt[PM_NSTATS]; 6229 uint64_t cyc[PM_NSTATS]; 6230 static const char *rx_stats[] = { 6231 "Read:", "Write bypass:", "Write mem:", "Flush:" 6232 }; 6233 static const char *tx_stats[] = { 6234 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:" 6235 }; 6236 6237 rc = sysctl_wire_old_buffer(req, 0); 6238 if (rc != 0) 6239 return (rc); 6240 6241 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6242 if (sb == NULL) 6243 return (ENOMEM); 6244 6245 t4_pmtx_get_stats(sc, cnt, cyc); 6246 sbuf_printf(sb, " Tx pcmds Tx bytes"); 6247 for (i = 0; i < ARRAY_SIZE(tx_stats); i++) 6248 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i], 6249 cyc[i]); 6250 6251 t4_pmrx_get_stats(sc, cnt, cyc); 6252 sbuf_printf(sb, "\n Rx pcmds Rx bytes"); 6253 for (i = 0; i < ARRAY_SIZE(rx_stats); i++) 6254 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i], 6255 cyc[i]); 6256 6257 rc = sbuf_finish(sb); 6258 sbuf_delete(sb); 6259 6260 return (rc); 6261 } 6262 6263 static int 6264 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 6265 { 6266 struct adapter *sc = arg1; 6267 struct sbuf *sb; 6268 int rc; 6269 struct tp_rdma_stats stats; 6270 6271 rc = sysctl_wire_old_buffer(req, 0); 6272 if (rc != 0) 6273 return (rc); 6274 6275 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6276 if (sb == NULL) 6277 return (ENOMEM); 6278 6279 t4_tp_get_rdma_stats(sc, &stats); 6280 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 6281 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 6282 6283 rc = sbuf_finish(sb); 6284 sbuf_delete(sb); 6285 6286 return (rc); 6287 } 6288 6289 static int 6290 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 6291 { 6292 struct adapter *sc = arg1; 6293 struct sbuf *sb; 6294 int rc; 6295 struct tp_tcp_stats v4, v6; 6296 6297 rc = sysctl_wire_old_buffer(req, 0); 6298 if (rc != 0) 6299 return (rc); 6300 6301 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6302 if (sb == NULL) 6303 return (ENOMEM); 6304 6305 t4_tp_get_tcp_stats(sc, &v4, &v6); 6306 sbuf_printf(sb, 6307 " IP IPv6\n"); 6308 sbuf_printf(sb, "OutRsts: %20u %20u\n", 6309 v4.tcpOutRsts, v6.tcpOutRsts); 6310 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 6311 v4.tcpInSegs, v6.tcpInSegs); 6312 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 6313 v4.tcpOutSegs, v6.tcpOutSegs); 6314 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 6315 v4.tcpRetransSegs, v6.tcpRetransSegs); 6316 6317 rc = sbuf_finish(sb); 6318 sbuf_delete(sb); 6319 6320 return (rc); 6321 } 6322 6323 static int 6324 sysctl_tids(SYSCTL_HANDLER_ARGS) 6325 { 6326 struct adapter *sc = arg1; 6327 struct sbuf *sb; 6328 int rc; 6329 struct tid_info *t = &sc->tids; 6330 6331 rc = sysctl_wire_old_buffer(req, 0); 6332 if (rc != 0) 6333 return (rc); 6334 6335 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6336 if (sb == NULL) 6337 return (ENOMEM); 6338 6339 if (t->natids) { 6340 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 6341 t->atids_in_use); 6342 } 6343 6344 if (t->ntids) { 6345 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6346 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 6347 6348 if (b) { 6349 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 6350 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6351 t->ntids - 1); 6352 } else { 6353 sbuf_printf(sb, "TID range: %u-%u", 6354 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6355 t->ntids - 1); 6356 } 6357 } else 6358 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 6359 sbuf_printf(sb, ", in use: %u\n", 6360 atomic_load_acq_int(&t->tids_in_use)); 6361 } 6362 6363 if (t->nstids) { 6364 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 6365 t->stid_base + t->nstids - 1, t->stids_in_use); 6366 } 6367 6368 if (t->nftids) { 6369 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 6370 t->ftid_base + t->nftids - 1); 6371 } 6372 6373 if (t->netids) { 6374 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base, 6375 t->etid_base + t->netids - 1); 6376 } 6377 6378 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 6379 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 6380 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 6381 6382 rc = sbuf_finish(sb); 6383 sbuf_delete(sb); 6384 6385 return (rc); 6386 } 6387 6388 static int 6389 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 6390 { 6391 struct adapter *sc = arg1; 6392 struct sbuf *sb; 6393 int rc; 6394 struct tp_err_stats stats; 6395 6396 rc = sysctl_wire_old_buffer(req, 0); 6397 if (rc != 0) 6398 return (rc); 6399 6400 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6401 if (sb == NULL) 6402 return (ENOMEM); 6403 6404 t4_tp_get_err_stats(sc, &stats); 6405 6406 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6407 "channel 3\n"); 6408 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 6409 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 6410 stats.macInErrs[3]); 6411 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 6412 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 6413 stats.hdrInErrs[3]); 6414 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 6415 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 6416 stats.tcpInErrs[3]); 6417 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 6418 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 6419 stats.tcp6InErrs[3]); 6420 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 6421 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 6422 stats.tnlCongDrops[3]); 6423 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 6424 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 6425 stats.tnlTxDrops[3]); 6426 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 6427 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 6428 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 6429 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 6430 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 6431 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 6432 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 6433 stats.ofldNoNeigh, stats.ofldCongDefer); 6434 6435 rc = sbuf_finish(sb); 6436 sbuf_delete(sb); 6437 6438 return (rc); 6439 } 6440 6441 struct field_desc { 6442 const char *name; 6443 u_int start; 6444 u_int width; 6445 }; 6446 6447 static void 6448 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) 6449 { 6450 char buf[32]; 6451 int line_size = 0; 6452 6453 while (f->name) { 6454 uint64_t mask = (1ULL << f->width) - 1; 6455 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, 6456 ((uintmax_t)v >> f->start) & mask); 6457 6458 if (line_size + len >= 79) { 6459 line_size = 8; 6460 sbuf_printf(sb, "\n "); 6461 } 6462 sbuf_printf(sb, "%s ", buf); 6463 line_size += len + 1; 6464 f++; 6465 } 6466 sbuf_printf(sb, "\n"); 6467 } 6468 6469 static struct field_desc tp_la0[] = { 6470 { "RcfOpCodeOut", 60, 4 }, 6471 { "State", 56, 4 }, 6472 { "WcfState", 52, 4 }, 6473 { "RcfOpcSrcOut", 50, 2 }, 6474 { "CRxError", 49, 1 }, 6475 { "ERxError", 48, 1 }, 6476 { "SanityFailed", 47, 1 }, 6477 { "SpuriousMsg", 46, 1 }, 6478 { "FlushInputMsg", 45, 1 }, 6479 { "FlushInputCpl", 44, 1 }, 6480 { "RssUpBit", 43, 1 }, 6481 { "RssFilterHit", 42, 1 }, 6482 { "Tid", 32, 10 }, 6483 { "InitTcb", 31, 1 }, 6484 { "LineNumber", 24, 7 }, 6485 { "Emsg", 23, 1 }, 6486 { "EdataOut", 22, 1 }, 6487 { "Cmsg", 21, 1 }, 6488 { "CdataOut", 20, 1 }, 6489 { "EreadPdu", 19, 1 }, 6490 { "CreadPdu", 18, 1 }, 6491 { "TunnelPkt", 17, 1 }, 6492 { "RcfPeerFin", 16, 1 }, 6493 { "RcfReasonOut", 12, 4 }, 6494 { "TxCchannel", 10, 2 }, 6495 { "RcfTxChannel", 8, 2 }, 6496 { "RxEchannel", 6, 2 }, 6497 { "RcfRxChannel", 5, 1 }, 6498 { "RcfDataOutSrdy", 4, 1 }, 6499 { "RxDvld", 3, 1 }, 6500 { "RxOoDvld", 2, 1 }, 6501 { "RxCongestion", 1, 1 }, 6502 { "TxCongestion", 0, 1 }, 6503 { NULL } 6504 }; 6505 6506 static struct field_desc tp_la1[] = { 6507 { "CplCmdIn", 56, 8 }, 6508 { "CplCmdOut", 48, 8 }, 6509 { "ESynOut", 47, 1 }, 6510 { "EAckOut", 46, 1 }, 6511 { "EFinOut", 45, 1 }, 6512 { "ERstOut", 44, 1 }, 6513 { "SynIn", 43, 1 }, 6514 { "AckIn", 42, 1 }, 6515 { "FinIn", 41, 1 }, 6516 { "RstIn", 40, 1 }, 6517 { "DataIn", 39, 1 }, 6518 { "DataInVld", 38, 1 }, 6519 { "PadIn", 37, 1 }, 6520 { "RxBufEmpty", 36, 1 }, 6521 { "RxDdp", 35, 1 }, 6522 { "RxFbCongestion", 34, 1 }, 6523 { "TxFbCongestion", 33, 1 }, 6524 { "TxPktSumSrdy", 32, 1 }, 6525 { "RcfUlpType", 28, 4 }, 6526 { "Eread", 27, 1 }, 6527 { "Ebypass", 26, 1 }, 6528 { "Esave", 25, 1 }, 6529 { "Static0", 24, 1 }, 6530 { "Cread", 23, 1 }, 6531 { "Cbypass", 22, 1 }, 6532 { "Csave", 21, 1 }, 6533 { "CPktOut", 20, 1 }, 6534 { "RxPagePoolFull", 18, 2 }, 6535 { "RxLpbkPkt", 17, 1 }, 6536 { "TxLpbkPkt", 16, 1 }, 6537 { "RxVfValid", 15, 1 }, 6538 { "SynLearned", 14, 1 }, 6539 { "SetDelEntry", 13, 1 }, 6540 { "SetInvEntry", 12, 1 }, 6541 { "CpcmdDvld", 11, 1 }, 6542 { "CpcmdSave", 10, 1 }, 6543 { "RxPstructsFull", 8, 2 }, 6544 { "EpcmdDvld", 7, 1 }, 6545 { "EpcmdFlush", 6, 1 }, 6546 { "EpcmdTrimPrefix", 5, 1 }, 6547 { "EpcmdTrimPostfix", 4, 1 }, 6548 { "ERssIp4Pkt", 3, 1 }, 6549 { "ERssIp6Pkt", 2, 1 }, 6550 { "ERssTcpUdpPkt", 1, 1 }, 6551 { "ERssFceFipPkt", 0, 1 }, 6552 { NULL } 6553 }; 6554 6555 static struct field_desc tp_la2[] = { 6556 { "CplCmdIn", 56, 8 }, 6557 { "MpsVfVld", 55, 1 }, 6558 { "MpsPf", 52, 3 }, 6559 { "MpsVf", 44, 8 }, 6560 { "SynIn", 43, 1 }, 6561 { "AckIn", 42, 1 }, 6562 { "FinIn", 41, 1 }, 6563 { "RstIn", 40, 1 }, 6564 { "DataIn", 39, 1 }, 6565 { "DataInVld", 38, 1 }, 6566 { "PadIn", 37, 1 }, 6567 { "RxBufEmpty", 36, 1 }, 6568 { "RxDdp", 35, 1 }, 6569 { "RxFbCongestion", 34, 1 }, 6570 { "TxFbCongestion", 33, 1 }, 6571 { "TxPktSumSrdy", 32, 1 }, 6572 { "RcfUlpType", 28, 4 }, 6573 { "Eread", 27, 1 }, 6574 { "Ebypass", 26, 1 }, 6575 { "Esave", 25, 1 }, 6576 { "Static0", 24, 1 }, 6577 { "Cread", 23, 1 }, 6578 { "Cbypass", 22, 1 }, 6579 { "Csave", 21, 1 }, 6580 { "CPktOut", 20, 1 }, 6581 { "RxPagePoolFull", 18, 2 }, 6582 { "RxLpbkPkt", 17, 1 }, 6583 { "TxLpbkPkt", 16, 1 }, 6584 { "RxVfValid", 15, 1 }, 6585 { "SynLearned", 14, 1 }, 6586 { "SetDelEntry", 13, 1 }, 6587 { "SetInvEntry", 12, 1 }, 6588 { "CpcmdDvld", 11, 1 }, 6589 { "CpcmdSave", 10, 1 }, 6590 { "RxPstructsFull", 8, 2 }, 6591 { "EpcmdDvld", 7, 1 }, 6592 { "EpcmdFlush", 6, 1 }, 6593 { "EpcmdTrimPrefix", 5, 1 }, 6594 { "EpcmdTrimPostfix", 4, 1 }, 6595 { "ERssIp4Pkt", 3, 1 }, 6596 { "ERssIp6Pkt", 2, 1 }, 6597 { "ERssTcpUdpPkt", 1, 1 }, 6598 { "ERssFceFipPkt", 0, 1 }, 6599 { NULL } 6600 }; 6601 6602 static void 6603 tp_la_show(struct sbuf *sb, uint64_t *p, int idx) 6604 { 6605 6606 field_desc_show(sb, *p, tp_la0); 6607 } 6608 6609 static void 6610 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) 6611 { 6612 6613 if (idx) 6614 sbuf_printf(sb, "\n"); 6615 field_desc_show(sb, p[0], tp_la0); 6616 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6617 field_desc_show(sb, p[1], tp_la0); 6618 } 6619 6620 static void 6621 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) 6622 { 6623 6624 if (idx) 6625 sbuf_printf(sb, "\n"); 6626 field_desc_show(sb, p[0], tp_la0); 6627 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6628 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); 6629 } 6630 6631 static int 6632 sysctl_tp_la(SYSCTL_HANDLER_ARGS) 6633 { 6634 struct adapter *sc = arg1; 6635 struct sbuf *sb; 6636 uint64_t *buf, *p; 6637 int rc; 6638 u_int i, inc; 6639 void (*show_func)(struct sbuf *, uint64_t *, int); 6640 6641 rc = sysctl_wire_old_buffer(req, 0); 6642 if (rc != 0) 6643 return (rc); 6644 6645 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6646 if (sb == NULL) 6647 return (ENOMEM); 6648 6649 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); 6650 6651 t4_tp_read_la(sc, buf, NULL); 6652 p = buf; 6653 6654 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { 6655 case 2: 6656 inc = 2; 6657 show_func = tp_la_show2; 6658 break; 6659 case 3: 6660 inc = 2; 6661 show_func = tp_la_show3; 6662 break; 6663 default: 6664 inc = 1; 6665 show_func = tp_la_show; 6666 } 6667 6668 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) 6669 (*show_func)(sb, p, i); 6670 6671 rc = sbuf_finish(sb); 6672 sbuf_delete(sb); 6673 free(buf, M_CXGBE); 6674 return (rc); 6675 } 6676 6677 static int 6678 sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 6679 { 6680 struct adapter *sc = arg1; 6681 struct sbuf *sb; 6682 int rc; 6683 u64 nrate[NCHAN], orate[NCHAN]; 6684 6685 rc = sysctl_wire_old_buffer(req, 0); 6686 if (rc != 0) 6687 return (rc); 6688 6689 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6690 if (sb == NULL) 6691 return (ENOMEM); 6692 6693 t4_get_chan_txrate(sc, nrate, orate); 6694 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6695 "channel 3\n"); 6696 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 6697 nrate[0], nrate[1], nrate[2], nrate[3]); 6698 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 6699 orate[0], orate[1], orate[2], orate[3]); 6700 6701 rc = sbuf_finish(sb); 6702 sbuf_delete(sb); 6703 6704 return (rc); 6705 } 6706 6707 static int 6708 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) 6709 { 6710 struct adapter *sc = arg1; 6711 struct sbuf *sb; 6712 uint32_t *buf, *p; 6713 int rc, i; 6714 6715 rc = sysctl_wire_old_buffer(req, 0); 6716 if (rc != 0) 6717 return (rc); 6718 6719 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6720 if (sb == NULL) 6721 return (ENOMEM); 6722 6723 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, 6724 M_ZERO | M_WAITOK); 6725 6726 t4_ulprx_read_la(sc, buf); 6727 p = buf; 6728 6729 sbuf_printf(sb, " Pcmd Type Message" 6730 " Data"); 6731 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { 6732 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", 6733 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); 6734 } 6735 6736 rc = sbuf_finish(sb); 6737 sbuf_delete(sb); 6738 free(buf, M_CXGBE); 6739 return (rc); 6740 } 6741 6742 static int 6743 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 6744 { 6745 struct adapter *sc = arg1; 6746 struct sbuf *sb; 6747 int rc, v; 6748 6749 rc = sysctl_wire_old_buffer(req, 0); 6750 if (rc != 0) 6751 return (rc); 6752 6753 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6754 if (sb == NULL) 6755 return (ENOMEM); 6756 6757 v = t4_read_reg(sc, A_SGE_STAT_CFG); 6758 if (G_STATSOURCE_T5(v) == 7) { 6759 if (G_STATMODE(v) == 0) { 6760 sbuf_printf(sb, "total %d, incomplete %d", 6761 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6762 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6763 } else if (G_STATMODE(v) == 1) { 6764 sbuf_printf(sb, "total %d, data overflow %d", 6765 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6766 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6767 } 6768 } 6769 rc = sbuf_finish(sb); 6770 sbuf_delete(sb); 6771 6772 return (rc); 6773 } 6774 #endif 6775 6776 static inline void 6777 txq_start(struct ifnet *ifp, struct sge_txq *txq) 6778 { 6779 struct buf_ring *br; 6780 struct mbuf *m; 6781 6782 TXQ_LOCK_ASSERT_OWNED(txq); 6783 6784 br = txq->br; 6785 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 6786 if (m) 6787 t4_eth_tx(ifp, txq, m); 6788 } 6789 6790 void 6791 t4_tx_callout(void *arg) 6792 { 6793 struct sge_eq *eq = arg; 6794 struct adapter *sc; 6795 6796 if (EQ_TRYLOCK(eq) == 0) 6797 goto reschedule; 6798 6799 if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) { 6800 EQ_UNLOCK(eq); 6801 reschedule: 6802 if (__predict_true(!(eq->flags && EQ_DOOMED))) 6803 callout_schedule(&eq->tx_callout, 1); 6804 return; 6805 } 6806 6807 EQ_LOCK_ASSERT_OWNED(eq); 6808 6809 if (__predict_true((eq->flags & EQ_DOOMED) == 0)) { 6810 6811 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 6812 struct sge_txq *txq = arg; 6813 struct port_info *pi = txq->ifp->if_softc; 6814 6815 sc = pi->adapter; 6816 } else { 6817 struct sge_wrq *wrq = arg; 6818 6819 sc = wrq->adapter; 6820 } 6821 6822 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task); 6823 } 6824 6825 EQ_UNLOCK(eq); 6826 } 6827 6828 void 6829 t4_tx_task(void *arg, int count) 6830 { 6831 struct sge_eq *eq = arg; 6832 6833 EQ_LOCK(eq); 6834 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 6835 struct sge_txq *txq = arg; 6836 txq_start(txq->ifp, txq); 6837 } else { 6838 struct sge_wrq *wrq = arg; 6839 t4_wrq_tx_locked(wrq->adapter, wrq, NULL); 6840 } 6841 EQ_UNLOCK(eq); 6842 } 6843 6844 static uint32_t 6845 fconf_to_mode(uint32_t fconf) 6846 { 6847 uint32_t mode; 6848 6849 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 6850 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 6851 6852 if (fconf & F_FRAGMENTATION) 6853 mode |= T4_FILTER_IP_FRAGMENT; 6854 6855 if (fconf & F_MPSHITTYPE) 6856 mode |= T4_FILTER_MPS_HIT_TYPE; 6857 6858 if (fconf & F_MACMATCH) 6859 mode |= T4_FILTER_MAC_IDX; 6860 6861 if (fconf & F_ETHERTYPE) 6862 mode |= T4_FILTER_ETH_TYPE; 6863 6864 if (fconf & F_PROTOCOL) 6865 mode |= T4_FILTER_IP_PROTO; 6866 6867 if (fconf & F_TOS) 6868 mode |= T4_FILTER_IP_TOS; 6869 6870 if (fconf & F_VLAN) 6871 mode |= T4_FILTER_VLAN; 6872 6873 if (fconf & F_VNIC_ID) 6874 mode |= T4_FILTER_VNIC; 6875 6876 if (fconf & F_PORT) 6877 mode |= T4_FILTER_PORT; 6878 6879 if (fconf & F_FCOE) 6880 mode |= T4_FILTER_FCoE; 6881 6882 return (mode); 6883 } 6884 6885 static uint32_t 6886 mode_to_fconf(uint32_t mode) 6887 { 6888 uint32_t fconf = 0; 6889 6890 if (mode & T4_FILTER_IP_FRAGMENT) 6891 fconf |= F_FRAGMENTATION; 6892 6893 if (mode & T4_FILTER_MPS_HIT_TYPE) 6894 fconf |= F_MPSHITTYPE; 6895 6896 if (mode & T4_FILTER_MAC_IDX) 6897 fconf |= F_MACMATCH; 6898 6899 if (mode & T4_FILTER_ETH_TYPE) 6900 fconf |= F_ETHERTYPE; 6901 6902 if (mode & T4_FILTER_IP_PROTO) 6903 fconf |= F_PROTOCOL; 6904 6905 if (mode & T4_FILTER_IP_TOS) 6906 fconf |= F_TOS; 6907 6908 if (mode & T4_FILTER_VLAN) 6909 fconf |= F_VLAN; 6910 6911 if (mode & T4_FILTER_VNIC) 6912 fconf |= F_VNIC_ID; 6913 6914 if (mode & T4_FILTER_PORT) 6915 fconf |= F_PORT; 6916 6917 if (mode & T4_FILTER_FCoE) 6918 fconf |= F_FCOE; 6919 6920 return (fconf); 6921 } 6922 6923 static uint32_t 6924 fspec_to_fconf(struct t4_filter_specification *fs) 6925 { 6926 uint32_t fconf = 0; 6927 6928 if (fs->val.frag || fs->mask.frag) 6929 fconf |= F_FRAGMENTATION; 6930 6931 if (fs->val.matchtype || fs->mask.matchtype) 6932 fconf |= F_MPSHITTYPE; 6933 6934 if (fs->val.macidx || fs->mask.macidx) 6935 fconf |= F_MACMATCH; 6936 6937 if (fs->val.ethtype || fs->mask.ethtype) 6938 fconf |= F_ETHERTYPE; 6939 6940 if (fs->val.proto || fs->mask.proto) 6941 fconf |= F_PROTOCOL; 6942 6943 if (fs->val.tos || fs->mask.tos) 6944 fconf |= F_TOS; 6945 6946 if (fs->val.vlan_vld || fs->mask.vlan_vld) 6947 fconf |= F_VLAN; 6948 6949 if (fs->val.vnic_vld || fs->mask.vnic_vld) 6950 fconf |= F_VNIC_ID; 6951 6952 if (fs->val.iport || fs->mask.iport) 6953 fconf |= F_PORT; 6954 6955 if (fs->val.fcoe || fs->mask.fcoe) 6956 fconf |= F_FCOE; 6957 6958 return (fconf); 6959 } 6960 6961 static int 6962 get_filter_mode(struct adapter *sc, uint32_t *mode) 6963 { 6964 int rc; 6965 uint32_t fconf; 6966 6967 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 6968 "t4getfm"); 6969 if (rc) 6970 return (rc); 6971 6972 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 6973 A_TP_VLAN_PRI_MAP); 6974 6975 if (sc->params.tp.vlan_pri_map != fconf) { 6976 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 6977 device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map, 6978 fconf); 6979 sc->params.tp.vlan_pri_map = fconf; 6980 } 6981 6982 *mode = fconf_to_mode(sc->params.tp.vlan_pri_map); 6983 6984 end_synchronized_op(sc, LOCK_HELD); 6985 return (0); 6986 } 6987 6988 static int 6989 set_filter_mode(struct adapter *sc, uint32_t mode) 6990 { 6991 uint32_t fconf; 6992 int rc; 6993 6994 fconf = mode_to_fconf(mode); 6995 6996 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 6997 "t4setfm"); 6998 if (rc) 6999 return (rc); 7000 7001 if (sc->tids.ftids_in_use > 0) { 7002 rc = EBUSY; 7003 goto done; 7004 } 7005 7006 #ifdef TCP_OFFLOAD 7007 if (sc->offload_map) { 7008 rc = EBUSY; 7009 goto done; 7010 } 7011 #endif 7012 7013 #ifdef notyet 7014 rc = -t4_set_filter_mode(sc, fconf); 7015 if (rc == 0) 7016 sc->filter_mode = fconf; 7017 #else 7018 rc = ENOTSUP; 7019 #endif 7020 7021 done: 7022 end_synchronized_op(sc, LOCK_HELD); 7023 return (rc); 7024 } 7025 7026 static inline uint64_t 7027 get_filter_hits(struct adapter *sc, uint32_t fid) 7028 { 7029 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 7030 uint64_t hits; 7031 7032 memwin_info(sc, 0, &mw_base, NULL); 7033 off = position_memwin(sc, 0, 7034 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 7035 if (is_t4(sc)) { 7036 hits = t4_read_reg64(sc, mw_base + off + 16); 7037 hits = be64toh(hits); 7038 } else { 7039 hits = t4_read_reg(sc, mw_base + off + 24); 7040 hits = be32toh(hits); 7041 } 7042 7043 return (hits); 7044 } 7045 7046 static int 7047 get_filter(struct adapter *sc, struct t4_filter *t) 7048 { 7049 int i, rc, nfilters = sc->tids.nftids; 7050 struct filter_entry *f; 7051 7052 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7053 "t4getf"); 7054 if (rc) 7055 return (rc); 7056 7057 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 7058 t->idx >= nfilters) { 7059 t->idx = 0xffffffff; 7060 goto done; 7061 } 7062 7063 f = &sc->tids.ftid_tab[t->idx]; 7064 for (i = t->idx; i < nfilters; i++, f++) { 7065 if (f->valid) { 7066 t->idx = i; 7067 t->l2tidx = f->l2t ? f->l2t->idx : 0; 7068 t->smtidx = f->smtidx; 7069 if (f->fs.hitcnts) 7070 t->hits = get_filter_hits(sc, t->idx); 7071 else 7072 t->hits = UINT64_MAX; 7073 t->fs = f->fs; 7074 7075 goto done; 7076 } 7077 } 7078 7079 t->idx = 0xffffffff; 7080 done: 7081 end_synchronized_op(sc, LOCK_HELD); 7082 return (0); 7083 } 7084 7085 static int 7086 set_filter(struct adapter *sc, struct t4_filter *t) 7087 { 7088 unsigned int nfilters, nports; 7089 struct filter_entry *f; 7090 int i, rc; 7091 7092 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 7093 if (rc) 7094 return (rc); 7095 7096 nfilters = sc->tids.nftids; 7097 nports = sc->params.nports; 7098 7099 if (nfilters == 0) { 7100 rc = ENOTSUP; 7101 goto done; 7102 } 7103 7104 if (!(sc->flags & FULL_INIT_DONE)) { 7105 rc = EAGAIN; 7106 goto done; 7107 } 7108 7109 if (t->idx >= nfilters) { 7110 rc = EINVAL; 7111 goto done; 7112 } 7113 7114 /* Validate against the global filter mode */ 7115 if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) != 7116 sc->params.tp.vlan_pri_map) { 7117 rc = E2BIG; 7118 goto done; 7119 } 7120 7121 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 7122 rc = EINVAL; 7123 goto done; 7124 } 7125 7126 if (t->fs.val.iport >= nports) { 7127 rc = EINVAL; 7128 goto done; 7129 } 7130 7131 /* Can't specify an iq if not steering to it */ 7132 if (!t->fs.dirsteer && t->fs.iq) { 7133 rc = EINVAL; 7134 goto done; 7135 } 7136 7137 /* IPv6 filter idx must be 4 aligned */ 7138 if (t->fs.type == 1 && 7139 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 7140 rc = EINVAL; 7141 goto done; 7142 } 7143 7144 if (sc->tids.ftid_tab == NULL) { 7145 KASSERT(sc->tids.ftids_in_use == 0, 7146 ("%s: no memory allocated but filters_in_use > 0", 7147 __func__)); 7148 7149 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 7150 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 7151 if (sc->tids.ftid_tab == NULL) { 7152 rc = ENOMEM; 7153 goto done; 7154 } 7155 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 7156 } 7157 7158 for (i = 0; i < 4; i++) { 7159 f = &sc->tids.ftid_tab[t->idx + i]; 7160 7161 if (f->pending || f->valid) { 7162 rc = EBUSY; 7163 goto done; 7164 } 7165 if (f->locked) { 7166 rc = EPERM; 7167 goto done; 7168 } 7169 7170 if (t->fs.type == 0) 7171 break; 7172 } 7173 7174 f = &sc->tids.ftid_tab[t->idx]; 7175 f->fs = t->fs; 7176 7177 rc = set_filter_wr(sc, t->idx); 7178 done: 7179 end_synchronized_op(sc, 0); 7180 7181 if (rc == 0) { 7182 mtx_lock(&sc->tids.ftid_lock); 7183 for (;;) { 7184 if (f->pending == 0) { 7185 rc = f->valid ? 0 : EIO; 7186 break; 7187 } 7188 7189 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7190 PCATCH, "t4setfw", 0)) { 7191 rc = EINPROGRESS; 7192 break; 7193 } 7194 } 7195 mtx_unlock(&sc->tids.ftid_lock); 7196 } 7197 return (rc); 7198 } 7199 7200 static int 7201 del_filter(struct adapter *sc, struct t4_filter *t) 7202 { 7203 unsigned int nfilters; 7204 struct filter_entry *f; 7205 int rc; 7206 7207 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 7208 if (rc) 7209 return (rc); 7210 7211 nfilters = sc->tids.nftids; 7212 7213 if (nfilters == 0) { 7214 rc = ENOTSUP; 7215 goto done; 7216 } 7217 7218 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 7219 t->idx >= nfilters) { 7220 rc = EINVAL; 7221 goto done; 7222 } 7223 7224 if (!(sc->flags & FULL_INIT_DONE)) { 7225 rc = EAGAIN; 7226 goto done; 7227 } 7228 7229 f = &sc->tids.ftid_tab[t->idx]; 7230 7231 if (f->pending) { 7232 rc = EBUSY; 7233 goto done; 7234 } 7235 if (f->locked) { 7236 rc = EPERM; 7237 goto done; 7238 } 7239 7240 if (f->valid) { 7241 t->fs = f->fs; /* extra info for the caller */ 7242 rc = del_filter_wr(sc, t->idx); 7243 } 7244 7245 done: 7246 end_synchronized_op(sc, 0); 7247 7248 if (rc == 0) { 7249 mtx_lock(&sc->tids.ftid_lock); 7250 for (;;) { 7251 if (f->pending == 0) { 7252 rc = f->valid ? EIO : 0; 7253 break; 7254 } 7255 7256 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7257 PCATCH, "t4delfw", 0)) { 7258 rc = EINPROGRESS; 7259 break; 7260 } 7261 } 7262 mtx_unlock(&sc->tids.ftid_lock); 7263 } 7264 7265 return (rc); 7266 } 7267 7268 static void 7269 clear_filter(struct filter_entry *f) 7270 { 7271 if (f->l2t) 7272 t4_l2t_release(f->l2t); 7273 7274 bzero(f, sizeof (*f)); 7275 } 7276 7277 static int 7278 set_filter_wr(struct adapter *sc, int fidx) 7279 { 7280 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7281 struct wrqe *wr; 7282 struct fw_filter_wr *fwr; 7283 unsigned int ftid; 7284 7285 ASSERT_SYNCHRONIZED_OP(sc); 7286 7287 if (f->fs.newdmac || f->fs.newvlan) { 7288 /* This filter needs an L2T entry; allocate one. */ 7289 f->l2t = t4_l2t_alloc_switching(sc->l2t); 7290 if (f->l2t == NULL) 7291 return (EAGAIN); 7292 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 7293 f->fs.dmac)) { 7294 t4_l2t_release(f->l2t); 7295 f->l2t = NULL; 7296 return (ENOMEM); 7297 } 7298 } 7299 7300 ftid = sc->tids.ftid_base + fidx; 7301 7302 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 7303 if (wr == NULL) 7304 return (ENOMEM); 7305 7306 fwr = wrtod(wr); 7307 bzero(fwr, sizeof (*fwr)); 7308 7309 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 7310 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 7311 fwr->tid_to_iq = 7312 htobe32(V_FW_FILTER_WR_TID(ftid) | 7313 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 7314 V_FW_FILTER_WR_NOREPLY(0) | 7315 V_FW_FILTER_WR_IQ(f->fs.iq)); 7316 fwr->del_filter_to_l2tix = 7317 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 7318 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 7319 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 7320 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 7321 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 7322 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 7323 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 7324 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 7325 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 7326 f->fs.newvlan == VLAN_REWRITE) | 7327 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 7328 f->fs.newvlan == VLAN_REWRITE) | 7329 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 7330 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 7331 V_FW_FILTER_WR_PRIO(f->fs.prio) | 7332 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 7333 fwr->ethtype = htobe16(f->fs.val.ethtype); 7334 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 7335 fwr->frag_to_ovlan_vldm = 7336 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 7337 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 7338 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 7339 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 7340 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 7341 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 7342 fwr->smac_sel = 0; 7343 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 7344 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 7345 fwr->maci_to_matchtypem = 7346 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 7347 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 7348 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 7349 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 7350 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 7351 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 7352 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 7353 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 7354 fwr->ptcl = f->fs.val.proto; 7355 fwr->ptclm = f->fs.mask.proto; 7356 fwr->ttyp = f->fs.val.tos; 7357 fwr->ttypm = f->fs.mask.tos; 7358 fwr->ivlan = htobe16(f->fs.val.vlan); 7359 fwr->ivlanm = htobe16(f->fs.mask.vlan); 7360 fwr->ovlan = htobe16(f->fs.val.vnic); 7361 fwr->ovlanm = htobe16(f->fs.mask.vnic); 7362 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 7363 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 7364 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 7365 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 7366 fwr->lp = htobe16(f->fs.val.dport); 7367 fwr->lpm = htobe16(f->fs.mask.dport); 7368 fwr->fp = htobe16(f->fs.val.sport); 7369 fwr->fpm = htobe16(f->fs.mask.sport); 7370 if (f->fs.newsmac) 7371 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 7372 7373 f->pending = 1; 7374 sc->tids.ftids_in_use++; 7375 7376 t4_wrq_tx(sc, wr); 7377 return (0); 7378 } 7379 7380 static int 7381 del_filter_wr(struct adapter *sc, int fidx) 7382 { 7383 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7384 struct wrqe *wr; 7385 struct fw_filter_wr *fwr; 7386 unsigned int ftid; 7387 7388 ftid = sc->tids.ftid_base + fidx; 7389 7390 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 7391 if (wr == NULL) 7392 return (ENOMEM); 7393 fwr = wrtod(wr); 7394 bzero(fwr, sizeof (*fwr)); 7395 7396 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 7397 7398 f->pending = 1; 7399 t4_wrq_tx(sc, wr); 7400 return (0); 7401 } 7402 7403 int 7404 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 7405 { 7406 struct adapter *sc = iq->adapter; 7407 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 7408 unsigned int idx = GET_TID(rpl); 7409 unsigned int rc; 7410 struct filter_entry *f; 7411 7412 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 7413 rss->opcode)); 7414 7415 if (is_ftid(sc, idx)) { 7416 7417 idx -= sc->tids.ftid_base; 7418 f = &sc->tids.ftid_tab[idx]; 7419 rc = G_COOKIE(rpl->cookie); 7420 7421 mtx_lock(&sc->tids.ftid_lock); 7422 if (rc == FW_FILTER_WR_FLT_ADDED) { 7423 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 7424 __func__, idx)); 7425 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 7426 f->pending = 0; /* asynchronous setup completed */ 7427 f->valid = 1; 7428 } else { 7429 if (rc != FW_FILTER_WR_FLT_DELETED) { 7430 /* Add or delete failed, display an error */ 7431 log(LOG_ERR, 7432 "filter %u setup failed with error %u\n", 7433 idx, rc); 7434 } 7435 7436 clear_filter(f); 7437 sc->tids.ftids_in_use--; 7438 } 7439 wakeup(&sc->tids.ftid_tab); 7440 mtx_unlock(&sc->tids.ftid_lock); 7441 } 7442 7443 return (0); 7444 } 7445 7446 static int 7447 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 7448 { 7449 int rc; 7450 7451 if (cntxt->cid > M_CTXTQID) 7452 return (EINVAL); 7453 7454 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 7455 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 7456 return (EINVAL); 7457 7458 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 7459 if (rc) 7460 return (rc); 7461 7462 if (sc->flags & FW_OK) { 7463 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 7464 &cntxt->data[0]); 7465 if (rc == 0) 7466 goto done; 7467 } 7468 7469 /* 7470 * Read via firmware failed or wasn't even attempted. Read directly via 7471 * the backdoor. 7472 */ 7473 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 7474 done: 7475 end_synchronized_op(sc, 0); 7476 return (rc); 7477 } 7478 7479 static int 7480 load_fw(struct adapter *sc, struct t4_data *fw) 7481 { 7482 int rc; 7483 uint8_t *fw_data; 7484 7485 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 7486 if (rc) 7487 return (rc); 7488 7489 if (sc->flags & FULL_INIT_DONE) { 7490 rc = EBUSY; 7491 goto done; 7492 } 7493 7494 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 7495 if (fw_data == NULL) { 7496 rc = ENOMEM; 7497 goto done; 7498 } 7499 7500 rc = copyin(fw->data, fw_data, fw->len); 7501 if (rc == 0) 7502 rc = -t4_load_fw(sc, fw_data, fw->len); 7503 7504 free(fw_data, M_CXGBE); 7505 done: 7506 end_synchronized_op(sc, 0); 7507 return (rc); 7508 } 7509 7510 static int 7511 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 7512 { 7513 uint32_t addr, off, remaining, i, n; 7514 uint32_t *buf, *b; 7515 uint32_t mw_base, mw_aperture; 7516 int rc; 7517 uint8_t *dst; 7518 7519 rc = validate_mem_range(sc, mr->addr, mr->len); 7520 if (rc != 0) 7521 return (rc); 7522 7523 memwin_info(sc, win, &mw_base, &mw_aperture); 7524 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 7525 addr = mr->addr; 7526 remaining = mr->len; 7527 dst = (void *)mr->data; 7528 7529 while (remaining) { 7530 off = position_memwin(sc, win, addr); 7531 7532 /* number of bytes that we'll copy in the inner loop */ 7533 n = min(remaining, mw_aperture - off); 7534 for (i = 0; i < n; i += 4) 7535 *b++ = t4_read_reg(sc, mw_base + off + i); 7536 7537 rc = copyout(buf, dst, n); 7538 if (rc != 0) 7539 break; 7540 7541 b = buf; 7542 dst += n; 7543 remaining -= n; 7544 addr += n; 7545 } 7546 7547 free(buf, M_CXGBE); 7548 return (rc); 7549 } 7550 7551 static int 7552 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 7553 { 7554 int rc; 7555 7556 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 7557 return (EINVAL); 7558 7559 if (i2cd->len > sizeof(i2cd->data)) 7560 return (EFBIG); 7561 7562 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 7563 if (rc) 7564 return (rc); 7565 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 7566 i2cd->offset, i2cd->len, &i2cd->data[0]); 7567 end_synchronized_op(sc, 0); 7568 7569 return (rc); 7570 } 7571 7572 static int 7573 in_range(int val, int lo, int hi) 7574 { 7575 7576 return (val < 0 || (val <= hi && val >= lo)); 7577 } 7578 7579 static int 7580 set_sched_class(struct adapter *sc, struct t4_sched_params *p) 7581 { 7582 int fw_subcmd, fw_type, rc; 7583 7584 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc"); 7585 if (rc) 7586 return (rc); 7587 7588 if (!(sc->flags & FULL_INIT_DONE)) { 7589 rc = EAGAIN; 7590 goto done; 7591 } 7592 7593 /* 7594 * Translate the cxgbetool parameters into T4 firmware parameters. (The 7595 * sub-command and type are in common locations.) 7596 */ 7597 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG) 7598 fw_subcmd = FW_SCHED_SC_CONFIG; 7599 else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS) 7600 fw_subcmd = FW_SCHED_SC_PARAMS; 7601 else { 7602 rc = EINVAL; 7603 goto done; 7604 } 7605 if (p->type == SCHED_CLASS_TYPE_PACKET) 7606 fw_type = FW_SCHED_TYPE_PKTSCHED; 7607 else { 7608 rc = EINVAL; 7609 goto done; 7610 } 7611 7612 if (fw_subcmd == FW_SCHED_SC_CONFIG) { 7613 /* Vet our parameters ..*/ 7614 if (p->u.config.minmax < 0) { 7615 rc = EINVAL; 7616 goto done; 7617 } 7618 7619 /* And pass the request to the firmware ...*/ 7620 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1); 7621 goto done; 7622 } 7623 7624 if (fw_subcmd == FW_SCHED_SC_PARAMS) { 7625 int fw_level; 7626 int fw_mode; 7627 int fw_rateunit; 7628 int fw_ratemode; 7629 7630 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL) 7631 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL; 7632 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) 7633 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR; 7634 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) 7635 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL; 7636 else { 7637 rc = EINVAL; 7638 goto done; 7639 } 7640 7641 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS) 7642 fw_mode = FW_SCHED_PARAMS_MODE_CLASS; 7643 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW) 7644 fw_mode = FW_SCHED_PARAMS_MODE_FLOW; 7645 else { 7646 rc = EINVAL; 7647 goto done; 7648 } 7649 7650 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS) 7651 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 7652 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS) 7653 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE; 7654 else { 7655 rc = EINVAL; 7656 goto done; 7657 } 7658 7659 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL) 7660 fw_ratemode = FW_SCHED_PARAMS_RATE_REL; 7661 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS) 7662 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS; 7663 else { 7664 rc = EINVAL; 7665 goto done; 7666 } 7667 7668 /* Vet our parameters ... */ 7669 if (!in_range(p->u.params.channel, 0, 3) || 7670 !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) || 7671 !in_range(p->u.params.minrate, 0, 10000000) || 7672 !in_range(p->u.params.maxrate, 0, 10000000) || 7673 !in_range(p->u.params.weight, 0, 100)) { 7674 rc = ERANGE; 7675 goto done; 7676 } 7677 7678 /* 7679 * Translate any unset parameters into the firmware's 7680 * nomenclature and/or fail the call if the parameters 7681 * are required ... 7682 */ 7683 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 || 7684 p->u.params.channel < 0 || p->u.params.cl < 0) { 7685 rc = EINVAL; 7686 goto done; 7687 } 7688 if (p->u.params.minrate < 0) 7689 p->u.params.minrate = 0; 7690 if (p->u.params.maxrate < 0) { 7691 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7692 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7693 rc = EINVAL; 7694 goto done; 7695 } else 7696 p->u.params.maxrate = 0; 7697 } 7698 if (p->u.params.weight < 0) { 7699 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) { 7700 rc = EINVAL; 7701 goto done; 7702 } else 7703 p->u.params.weight = 0; 7704 } 7705 if (p->u.params.pktsize < 0) { 7706 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7707 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7708 rc = EINVAL; 7709 goto done; 7710 } else 7711 p->u.params.pktsize = 0; 7712 } 7713 7714 /* See what the firmware thinks of the request ... */ 7715 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode, 7716 fw_rateunit, fw_ratemode, p->u.params.channel, 7717 p->u.params.cl, p->u.params.minrate, p->u.params.maxrate, 7718 p->u.params.weight, p->u.params.pktsize, 1); 7719 goto done; 7720 } 7721 7722 rc = EINVAL; 7723 done: 7724 end_synchronized_op(sc, 0); 7725 return (rc); 7726 } 7727 7728 static int 7729 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p) 7730 { 7731 struct port_info *pi = NULL; 7732 struct sge_txq *txq; 7733 uint32_t fw_mnem, fw_queue, fw_class; 7734 int i, rc; 7735 7736 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq"); 7737 if (rc) 7738 return (rc); 7739 7740 if (!(sc->flags & FULL_INIT_DONE)) { 7741 rc = EAGAIN; 7742 goto done; 7743 } 7744 7745 if (p->port >= sc->params.nports) { 7746 rc = EINVAL; 7747 goto done; 7748 } 7749 7750 pi = sc->port[p->port]; 7751 if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) { 7752 rc = EINVAL; 7753 goto done; 7754 } 7755 7756 /* 7757 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX 7758 * Scheduling Class in this case). 7759 */ 7760 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 7761 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH)); 7762 fw_class = p->cl < 0 ? 0xffffffff : p->cl; 7763 7764 /* 7765 * If op.queue is non-negative, then we're only changing the scheduling 7766 * on a single specified TX queue. 7767 */ 7768 if (p->queue >= 0) { 7769 txq = &sc->sge.txq[pi->first_txq + p->queue]; 7770 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7771 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7772 &fw_class); 7773 goto done; 7774 } 7775 7776 /* 7777 * Change the scheduling on all the TX queues for the 7778 * interface. 7779 */ 7780 for_each_txq(pi, i, txq) { 7781 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7782 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7783 &fw_class); 7784 if (rc) 7785 goto done; 7786 } 7787 7788 rc = 0; 7789 done: 7790 end_synchronized_op(sc, 0); 7791 return (rc); 7792 } 7793 7794 int 7795 t4_os_find_pci_capability(struct adapter *sc, int cap) 7796 { 7797 int i; 7798 7799 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 7800 } 7801 7802 int 7803 t4_os_pci_save_state(struct adapter *sc) 7804 { 7805 device_t dev; 7806 struct pci_devinfo *dinfo; 7807 7808 dev = sc->dev; 7809 dinfo = device_get_ivars(dev); 7810 7811 pci_cfg_save(dev, dinfo, 0); 7812 return (0); 7813 } 7814 7815 int 7816 t4_os_pci_restore_state(struct adapter *sc) 7817 { 7818 device_t dev; 7819 struct pci_devinfo *dinfo; 7820 7821 dev = sc->dev; 7822 dinfo = device_get_ivars(dev); 7823 7824 pci_cfg_restore(dev, dinfo); 7825 return (0); 7826 } 7827 7828 void 7829 t4_os_portmod_changed(const struct adapter *sc, int idx) 7830 { 7831 struct port_info *pi = sc->port[idx]; 7832 static const char *mod_str[] = { 7833 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 7834 }; 7835 7836 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 7837 if_printf(pi->ifp, "transceiver unplugged.\n"); 7838 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 7839 if_printf(pi->ifp, "unknown transceiver inserted.\n"); 7840 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 7841 if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 7842 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 7843 if_printf(pi->ifp, "%s transceiver inserted.\n", 7844 mod_str[pi->mod_type]); 7845 } else { 7846 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 7847 pi->mod_type); 7848 } 7849 } 7850 7851 void 7852 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason) 7853 { 7854 struct port_info *pi = sc->port[idx]; 7855 struct ifnet *ifp = pi->ifp; 7856 7857 if (link_stat) { 7858 pi->linkdnrc = -1; 7859 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 7860 if_link_state_change(ifp, LINK_STATE_UP); 7861 } else { 7862 if (reason >= 0) 7863 pi->linkdnrc = reason; 7864 if_link_state_change(ifp, LINK_STATE_DOWN); 7865 } 7866 } 7867 7868 void 7869 t4_iterate(void (*func)(struct adapter *, void *), void *arg) 7870 { 7871 struct adapter *sc; 7872 7873 sx_slock(&t4_list_lock); 7874 SLIST_FOREACH(sc, &t4_list, link) { 7875 /* 7876 * func should not make any assumptions about what state sc is 7877 * in - the only guarantee is that sc->sc_lock is a valid lock. 7878 */ 7879 func(sc, arg); 7880 } 7881 sx_sunlock(&t4_list_lock); 7882 } 7883 7884 static int 7885 t4_open(struct cdev *dev, int flags, int type, struct thread *td) 7886 { 7887 return (0); 7888 } 7889 7890 static int 7891 t4_close(struct cdev *dev, int flags, int type, struct thread *td) 7892 { 7893 return (0); 7894 } 7895 7896 static int 7897 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 7898 struct thread *td) 7899 { 7900 int rc; 7901 struct adapter *sc = dev->si_drv1; 7902 7903 rc = priv_check(td, PRIV_DRIVER); 7904 if (rc != 0) 7905 return (rc); 7906 7907 switch (cmd) { 7908 case CHELSIO_T4_GETREG: { 7909 struct t4_reg *edata = (struct t4_reg *)data; 7910 7911 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 7912 return (EFAULT); 7913 7914 if (edata->size == 4) 7915 edata->val = t4_read_reg(sc, edata->addr); 7916 else if (edata->size == 8) 7917 edata->val = t4_read_reg64(sc, edata->addr); 7918 else 7919 return (EINVAL); 7920 7921 break; 7922 } 7923 case CHELSIO_T4_SETREG: { 7924 struct t4_reg *edata = (struct t4_reg *)data; 7925 7926 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 7927 return (EFAULT); 7928 7929 if (edata->size == 4) { 7930 if (edata->val & 0xffffffff00000000) 7931 return (EINVAL); 7932 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 7933 } else if (edata->size == 8) 7934 t4_write_reg64(sc, edata->addr, edata->val); 7935 else 7936 return (EINVAL); 7937 break; 7938 } 7939 case CHELSIO_T4_REGDUMP: { 7940 struct t4_regdump *regs = (struct t4_regdump *)data; 7941 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 7942 uint8_t *buf; 7943 7944 if (regs->len < reglen) { 7945 regs->len = reglen; /* hint to the caller */ 7946 return (ENOBUFS); 7947 } 7948 7949 regs->len = reglen; 7950 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 7951 t4_get_regs(sc, regs, buf); 7952 rc = copyout(buf, regs->data, reglen); 7953 free(buf, M_CXGBE); 7954 break; 7955 } 7956 case CHELSIO_T4_GET_FILTER_MODE: 7957 rc = get_filter_mode(sc, (uint32_t *)data); 7958 break; 7959 case CHELSIO_T4_SET_FILTER_MODE: 7960 rc = set_filter_mode(sc, *(uint32_t *)data); 7961 break; 7962 case CHELSIO_T4_GET_FILTER: 7963 rc = get_filter(sc, (struct t4_filter *)data); 7964 break; 7965 case CHELSIO_T4_SET_FILTER: 7966 rc = set_filter(sc, (struct t4_filter *)data); 7967 break; 7968 case CHELSIO_T4_DEL_FILTER: 7969 rc = del_filter(sc, (struct t4_filter *)data); 7970 break; 7971 case CHELSIO_T4_GET_SGE_CONTEXT: 7972 rc = get_sge_context(sc, (struct t4_sge_context *)data); 7973 break; 7974 case CHELSIO_T4_LOAD_FW: 7975 rc = load_fw(sc, (struct t4_data *)data); 7976 break; 7977 case CHELSIO_T4_GET_MEM: 7978 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 7979 break; 7980 case CHELSIO_T4_GET_I2C: 7981 rc = read_i2c(sc, (struct t4_i2c_data *)data); 7982 break; 7983 case CHELSIO_T4_CLEAR_STATS: { 7984 int i; 7985 u_int port_id = *(uint32_t *)data; 7986 struct port_info *pi; 7987 7988 if (port_id >= sc->params.nports) 7989 return (EINVAL); 7990 pi = sc->port[port_id]; 7991 7992 /* MAC stats */ 7993 t4_clr_port_stats(sc, pi->tx_chan); 7994 7995 if (pi->flags & PORT_INIT_DONE) { 7996 struct sge_rxq *rxq; 7997 struct sge_txq *txq; 7998 struct sge_wrq *wrq; 7999 8000 for_each_rxq(pi, i, rxq) { 8001 #if defined(INET) || defined(INET6) 8002 rxq->lro.lro_queued = 0; 8003 rxq->lro.lro_flushed = 0; 8004 #endif 8005 rxq->rxcsum = 0; 8006 rxq->vlan_extraction = 0; 8007 } 8008 8009 for_each_txq(pi, i, txq) { 8010 txq->txcsum = 0; 8011 txq->tso_wrs = 0; 8012 txq->vlan_insertion = 0; 8013 txq->imm_wrs = 0; 8014 txq->sgl_wrs = 0; 8015 txq->txpkt_wrs = 0; 8016 txq->txpkts_wrs = 0; 8017 txq->txpkts_pkts = 0; 8018 txq->br->br_drops = 0; 8019 txq->no_dmamap = 0; 8020 txq->no_desc = 0; 8021 } 8022 8023 #ifdef TCP_OFFLOAD 8024 /* nothing to clear for each ofld_rxq */ 8025 8026 for_each_ofld_txq(pi, i, wrq) { 8027 wrq->tx_wrs = 0; 8028 wrq->no_desc = 0; 8029 } 8030 #endif 8031 wrq = &sc->sge.ctrlq[pi->port_id]; 8032 wrq->tx_wrs = 0; 8033 wrq->no_desc = 0; 8034 } 8035 break; 8036 } 8037 case CHELSIO_T4_SCHED_CLASS: 8038 rc = set_sched_class(sc, (struct t4_sched_params *)data); 8039 break; 8040 case CHELSIO_T4_SCHED_QUEUE: 8041 rc = set_sched_queue(sc, (struct t4_sched_queue *)data); 8042 break; 8043 case CHELSIO_T4_GET_TRACER: 8044 rc = t4_get_tracer(sc, (struct t4_tracer *)data); 8045 break; 8046 case CHELSIO_T4_SET_TRACER: 8047 rc = t4_set_tracer(sc, (struct t4_tracer *)data); 8048 break; 8049 default: 8050 rc = EINVAL; 8051 } 8052 8053 return (rc); 8054 } 8055 8056 #ifdef TCP_OFFLOAD 8057 void 8058 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask, 8059 const unsigned int *pgsz_order) 8060 { 8061 struct port_info *pi = ifp->if_softc; 8062 struct adapter *sc = pi->adapter; 8063 8064 t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask); 8065 t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) | 8066 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) | 8067 V_HPZ3(pgsz_order[3])); 8068 } 8069 8070 static int 8071 toe_capability(struct port_info *pi, int enable) 8072 { 8073 int rc; 8074 struct adapter *sc = pi->adapter; 8075 8076 ASSERT_SYNCHRONIZED_OP(sc); 8077 8078 if (!is_offload(sc)) 8079 return (ENODEV); 8080 8081 if (enable) { 8082 if (!(sc->flags & FULL_INIT_DONE)) { 8083 rc = cxgbe_init_synchronized(pi); 8084 if (rc) 8085 return (rc); 8086 } 8087 8088 if (isset(&sc->offload_map, pi->port_id)) 8089 return (0); 8090 8091 if (!(sc->flags & TOM_INIT_DONE)) { 8092 rc = t4_activate_uld(sc, ULD_TOM); 8093 if (rc == EAGAIN) { 8094 log(LOG_WARNING, 8095 "You must kldload t4_tom.ko before trying " 8096 "to enable TOE on a cxgbe interface.\n"); 8097 } 8098 if (rc != 0) 8099 return (rc); 8100 KASSERT(sc->tom_softc != NULL, 8101 ("%s: TOM activated but softc NULL", __func__)); 8102 KASSERT(sc->flags & TOM_INIT_DONE, 8103 ("%s: TOM activated but flag not set", __func__)); 8104 } 8105 8106 setbit(&sc->offload_map, pi->port_id); 8107 } else { 8108 if (!isset(&sc->offload_map, pi->port_id)) 8109 return (0); 8110 8111 KASSERT(sc->flags & TOM_INIT_DONE, 8112 ("%s: TOM never initialized?", __func__)); 8113 clrbit(&sc->offload_map, pi->port_id); 8114 } 8115 8116 return (0); 8117 } 8118 8119 /* 8120 * Add an upper layer driver to the global list. 8121 */ 8122 int 8123 t4_register_uld(struct uld_info *ui) 8124 { 8125 int rc = 0; 8126 struct uld_info *u; 8127 8128 sx_xlock(&t4_uld_list_lock); 8129 SLIST_FOREACH(u, &t4_uld_list, link) { 8130 if (u->uld_id == ui->uld_id) { 8131 rc = EEXIST; 8132 goto done; 8133 } 8134 } 8135 8136 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 8137 ui->refcount = 0; 8138 done: 8139 sx_xunlock(&t4_uld_list_lock); 8140 return (rc); 8141 } 8142 8143 int 8144 t4_unregister_uld(struct uld_info *ui) 8145 { 8146 int rc = EINVAL; 8147 struct uld_info *u; 8148 8149 sx_xlock(&t4_uld_list_lock); 8150 8151 SLIST_FOREACH(u, &t4_uld_list, link) { 8152 if (u == ui) { 8153 if (ui->refcount > 0) { 8154 rc = EBUSY; 8155 goto done; 8156 } 8157 8158 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 8159 rc = 0; 8160 goto done; 8161 } 8162 } 8163 done: 8164 sx_xunlock(&t4_uld_list_lock); 8165 return (rc); 8166 } 8167 8168 int 8169 t4_activate_uld(struct adapter *sc, int id) 8170 { 8171 int rc = EAGAIN; 8172 struct uld_info *ui; 8173 8174 ASSERT_SYNCHRONIZED_OP(sc); 8175 8176 sx_slock(&t4_uld_list_lock); 8177 8178 SLIST_FOREACH(ui, &t4_uld_list, link) { 8179 if (ui->uld_id == id) { 8180 rc = ui->activate(sc); 8181 if (rc == 0) 8182 ui->refcount++; 8183 goto done; 8184 } 8185 } 8186 done: 8187 sx_sunlock(&t4_uld_list_lock); 8188 8189 return (rc); 8190 } 8191 8192 int 8193 t4_deactivate_uld(struct adapter *sc, int id) 8194 { 8195 int rc = EINVAL; 8196 struct uld_info *ui; 8197 8198 ASSERT_SYNCHRONIZED_OP(sc); 8199 8200 sx_slock(&t4_uld_list_lock); 8201 8202 SLIST_FOREACH(ui, &t4_uld_list, link) { 8203 if (ui->uld_id == id) { 8204 rc = ui->deactivate(sc); 8205 if (rc == 0) 8206 ui->refcount--; 8207 goto done; 8208 } 8209 } 8210 done: 8211 sx_sunlock(&t4_uld_list_lock); 8212 8213 return (rc); 8214 } 8215 #endif 8216 8217 /* 8218 * Come up with reasonable defaults for some of the tunables, provided they're 8219 * not set by the user (in which case we'll use the values as is). 8220 */ 8221 static void 8222 tweak_tunables(void) 8223 { 8224 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 8225 8226 if (t4_ntxq10g < 1) 8227 t4_ntxq10g = min(nc, NTXQ_10G); 8228 8229 if (t4_ntxq1g < 1) 8230 t4_ntxq1g = min(nc, NTXQ_1G); 8231 8232 if (t4_nrxq10g < 1) 8233 t4_nrxq10g = min(nc, NRXQ_10G); 8234 8235 if (t4_nrxq1g < 1) 8236 t4_nrxq1g = min(nc, NRXQ_1G); 8237 8238 #ifdef TCP_OFFLOAD 8239 if (t4_nofldtxq10g < 1) 8240 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 8241 8242 if (t4_nofldtxq1g < 1) 8243 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 8244 8245 if (t4_nofldrxq10g < 1) 8246 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 8247 8248 if (t4_nofldrxq1g < 1) 8249 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 8250 8251 if (t4_toecaps_allowed == -1) 8252 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 8253 #else 8254 if (t4_toecaps_allowed == -1) 8255 t4_toecaps_allowed = 0; 8256 #endif 8257 8258 #ifdef DEV_NETMAP 8259 if (t4_nnmtxq10g < 1) 8260 t4_nnmtxq10g = min(nc, NNMTXQ_10G); 8261 8262 if (t4_nnmtxq1g < 1) 8263 t4_nnmtxq1g = min(nc, NNMTXQ_1G); 8264 8265 if (t4_nnmrxq10g < 1) 8266 t4_nnmrxq10g = min(nc, NNMRXQ_10G); 8267 8268 if (t4_nnmrxq1g < 1) 8269 t4_nnmrxq1g = min(nc, NNMRXQ_1G); 8270 #endif 8271 8272 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 8273 t4_tmr_idx_10g = TMR_IDX_10G; 8274 8275 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 8276 t4_pktc_idx_10g = PKTC_IDX_10G; 8277 8278 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 8279 t4_tmr_idx_1g = TMR_IDX_1G; 8280 8281 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 8282 t4_pktc_idx_1g = PKTC_IDX_1G; 8283 8284 if (t4_qsize_txq < 128) 8285 t4_qsize_txq = 128; 8286 8287 if (t4_qsize_rxq < 128) 8288 t4_qsize_rxq = 128; 8289 while (t4_qsize_rxq & 7) 8290 t4_qsize_rxq++; 8291 8292 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 8293 } 8294 8295 static struct sx mlu; /* mod load unload */ 8296 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload"); 8297 8298 static int 8299 mod_event(module_t mod, int cmd, void *arg) 8300 { 8301 int rc = 0; 8302 static int loaded = 0; 8303 8304 switch (cmd) { 8305 case MOD_LOAD: 8306 sx_xlock(&mlu); 8307 if (loaded++ == 0) { 8308 t4_sge_modload(); 8309 sx_init(&t4_list_lock, "T4/T5 adapters"); 8310 SLIST_INIT(&t4_list); 8311 #ifdef TCP_OFFLOAD 8312 sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); 8313 SLIST_INIT(&t4_uld_list); 8314 #endif 8315 t4_tracer_modload(); 8316 tweak_tunables(); 8317 } 8318 sx_xunlock(&mlu); 8319 break; 8320 8321 case MOD_UNLOAD: 8322 sx_xlock(&mlu); 8323 if (--loaded == 0) { 8324 int tries; 8325 8326 sx_slock(&t4_list_lock); 8327 if (!SLIST_EMPTY(&t4_list)) { 8328 rc = EBUSY; 8329 sx_sunlock(&t4_list_lock); 8330 goto done_unload; 8331 } 8332 #ifdef TCP_OFFLOAD 8333 sx_slock(&t4_uld_list_lock); 8334 if (!SLIST_EMPTY(&t4_uld_list)) { 8335 rc = EBUSY; 8336 sx_sunlock(&t4_uld_list_lock); 8337 sx_sunlock(&t4_list_lock); 8338 goto done_unload; 8339 } 8340 #endif 8341 tries = 0; 8342 while (tries++ < 5 && t4_sge_extfree_refs() != 0) { 8343 uprintf("%ju clusters with custom free routine " 8344 "still is use.\n", t4_sge_extfree_refs()); 8345 pause("t4unload", 2 * hz); 8346 } 8347 #ifdef TCP_OFFLOAD 8348 sx_sunlock(&t4_uld_list_lock); 8349 #endif 8350 sx_sunlock(&t4_list_lock); 8351 8352 if (t4_sge_extfree_refs() == 0) { 8353 t4_tracer_modunload(); 8354 #ifdef TCP_OFFLOAD 8355 sx_destroy(&t4_uld_list_lock); 8356 #endif 8357 sx_destroy(&t4_list_lock); 8358 t4_sge_modunload(); 8359 loaded = 0; 8360 } else { 8361 rc = EBUSY; 8362 loaded++; /* undo earlier decrement */ 8363 } 8364 } 8365 done_unload: 8366 sx_xunlock(&mlu); 8367 break; 8368 } 8369 8370 return (rc); 8371 } 8372 8373 static devclass_t t4_devclass, t5_devclass; 8374 static devclass_t cxgbe_devclass, cxl_devclass; 8375 8376 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 8377 MODULE_VERSION(t4nex, 1); 8378 MODULE_DEPEND(t4nex, firmware, 1, 1, 1); 8379 8380 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 8381 MODULE_VERSION(t5nex, 1); 8382 MODULE_DEPEND(t5nex, firmware, 1, 1, 1); 8383 8384 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 8385 MODULE_VERSION(cxgbe, 1); 8386 8387 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 8388 MODULE_VERSION(cxl, 1); 8389