1 /*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_inet.h" 32 #include "opt_inet6.h" 33 34 #include <sys/param.h> 35 #include <sys/conf.h> 36 #include <sys/priv.h> 37 #include <sys/kernel.h> 38 #include <sys/bus.h> 39 #include <sys/module.h> 40 #include <sys/malloc.h> 41 #include <sys/queue.h> 42 #include <sys/taskqueue.h> 43 #include <sys/pciio.h> 44 #include <dev/pci/pcireg.h> 45 #include <dev/pci/pcivar.h> 46 #include <dev/pci/pci_private.h> 47 #include <sys/firmware.h> 48 #include <sys/sbuf.h> 49 #include <sys/smp.h> 50 #include <sys/socket.h> 51 #include <sys/sockio.h> 52 #include <sys/sysctl.h> 53 #include <net/ethernet.h> 54 #include <net/if.h> 55 #include <net/if_types.h> 56 #include <net/if_dl.h> 57 #include <net/if_vlan_var.h> 58 #if defined(__i386__) || defined(__amd64__) 59 #include <vm/vm.h> 60 #include <vm/pmap.h> 61 #endif 62 63 #include "common/common.h" 64 #include "common/t4_msg.h" 65 #include "common/t4_regs.h" 66 #include "common/t4_regs_values.h" 67 #include "t4_ioctl.h" 68 #include "t4_l2t.h" 69 #include "t4_mp_ring.h" 70 71 /* T4 bus driver interface */ 72 static int t4_probe(device_t); 73 static int t4_attach(device_t); 74 static int t4_detach(device_t); 75 static device_method_t t4_methods[] = { 76 DEVMETHOD(device_probe, t4_probe), 77 DEVMETHOD(device_attach, t4_attach), 78 DEVMETHOD(device_detach, t4_detach), 79 80 DEVMETHOD_END 81 }; 82 static driver_t t4_driver = { 83 "t4nex", 84 t4_methods, 85 sizeof(struct adapter) 86 }; 87 88 89 /* T4 port (cxgbe) interface */ 90 static int cxgbe_probe(device_t); 91 static int cxgbe_attach(device_t); 92 static int cxgbe_detach(device_t); 93 static device_method_t cxgbe_methods[] = { 94 DEVMETHOD(device_probe, cxgbe_probe), 95 DEVMETHOD(device_attach, cxgbe_attach), 96 DEVMETHOD(device_detach, cxgbe_detach), 97 { 0, 0 } 98 }; 99 static driver_t cxgbe_driver = { 100 "cxgbe", 101 cxgbe_methods, 102 sizeof(struct port_info) 103 }; 104 105 static d_ioctl_t t4_ioctl; 106 static d_open_t t4_open; 107 static d_close_t t4_close; 108 109 static struct cdevsw t4_cdevsw = { 110 .d_version = D_VERSION, 111 .d_flags = 0, 112 .d_open = t4_open, 113 .d_close = t4_close, 114 .d_ioctl = t4_ioctl, 115 .d_name = "t4nex", 116 }; 117 118 /* T5 bus driver interface */ 119 static int t5_probe(device_t); 120 static device_method_t t5_methods[] = { 121 DEVMETHOD(device_probe, t5_probe), 122 DEVMETHOD(device_attach, t4_attach), 123 DEVMETHOD(device_detach, t4_detach), 124 125 DEVMETHOD_END 126 }; 127 static driver_t t5_driver = { 128 "t5nex", 129 t5_methods, 130 sizeof(struct adapter) 131 }; 132 133 134 /* T5 port (cxl) interface */ 135 static driver_t cxl_driver = { 136 "cxl", 137 cxgbe_methods, 138 sizeof(struct port_info) 139 }; 140 141 static struct cdevsw t5_cdevsw = { 142 .d_version = D_VERSION, 143 .d_flags = 0, 144 .d_open = t4_open, 145 .d_close = t4_close, 146 .d_ioctl = t4_ioctl, 147 .d_name = "t5nex", 148 }; 149 150 /* ifnet + media interface */ 151 static void cxgbe_init(void *); 152 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 153 static int cxgbe_transmit(struct ifnet *, struct mbuf *); 154 static void cxgbe_qflush(struct ifnet *); 155 static uint64_t cxgbe_get_counter(struct ifnet *, ift_counter); 156 static int cxgbe_media_change(struct ifnet *); 157 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 158 159 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 160 161 /* 162 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 163 * then ADAPTER_LOCK, then t4_uld_list_lock. 164 */ 165 static struct sx t4_list_lock; 166 SLIST_HEAD(, adapter) t4_list; 167 #ifdef TCP_OFFLOAD 168 static struct sx t4_uld_list_lock; 169 SLIST_HEAD(, uld_info) t4_uld_list; 170 #endif 171 172 /* 173 * Tunables. See tweak_tunables() too. 174 * 175 * Each tunable is set to a default value here if it's known at compile-time. 176 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 177 * provide a reasonable default when the driver is loaded. 178 * 179 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 180 * T5 are under hw.cxl. 181 */ 182 183 /* 184 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 185 */ 186 #define NTXQ_10G 16 187 static int t4_ntxq10g = -1; 188 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 189 190 #define NRXQ_10G 8 191 static int t4_nrxq10g = -1; 192 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 193 194 #define NTXQ_1G 4 195 static int t4_ntxq1g = -1; 196 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 197 198 #define NRXQ_1G 2 199 static int t4_nrxq1g = -1; 200 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 201 202 static int t4_rsrv_noflowq = 0; 203 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq); 204 205 #ifdef TCP_OFFLOAD 206 #define NOFLDTXQ_10G 8 207 static int t4_nofldtxq10g = -1; 208 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 209 210 #define NOFLDRXQ_10G 2 211 static int t4_nofldrxq10g = -1; 212 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 213 214 #define NOFLDTXQ_1G 2 215 static int t4_nofldtxq1g = -1; 216 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 217 218 #define NOFLDRXQ_1G 1 219 static int t4_nofldrxq1g = -1; 220 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 221 #endif 222 223 #ifdef DEV_NETMAP 224 #define NNMTXQ_10G 2 225 static int t4_nnmtxq10g = -1; 226 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g); 227 228 #define NNMRXQ_10G 2 229 static int t4_nnmrxq10g = -1; 230 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g); 231 232 #define NNMTXQ_1G 1 233 static int t4_nnmtxq1g = -1; 234 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g); 235 236 #define NNMRXQ_1G 1 237 static int t4_nnmrxq1g = -1; 238 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g); 239 #endif 240 241 /* 242 * Holdoff parameters for 10G and 1G ports. 243 */ 244 #define TMR_IDX_10G 1 245 static int t4_tmr_idx_10g = TMR_IDX_10G; 246 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 247 248 #define PKTC_IDX_10G (-1) 249 static int t4_pktc_idx_10g = PKTC_IDX_10G; 250 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 251 252 #define TMR_IDX_1G 1 253 static int t4_tmr_idx_1g = TMR_IDX_1G; 254 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 255 256 #define PKTC_IDX_1G (-1) 257 static int t4_pktc_idx_1g = PKTC_IDX_1G; 258 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 259 260 /* 261 * Size (# of entries) of each tx and rx queue. 262 */ 263 static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 264 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 265 266 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 267 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 268 269 /* 270 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 271 */ 272 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 273 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 274 275 /* 276 * Configuration file. 277 */ 278 #define DEFAULT_CF "default" 279 #define FLASH_CF "flash" 280 #define UWIRE_CF "uwire" 281 #define FPGA_CF "fpga" 282 static char t4_cfg_file[32] = DEFAULT_CF; 283 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 284 285 /* 286 * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively). 287 * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them. 288 * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water 289 * mark or when signalled to do so, 0 to never emit PAUSE. 290 */ 291 static int t4_pause_settings = PAUSE_TX | PAUSE_RX; 292 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings); 293 294 /* 295 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 296 * encouraged respectively). 297 */ 298 static unsigned int t4_fw_install = 1; 299 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 300 301 /* 302 * ASIC features that will be used. Disable the ones you don't want so that the 303 * chip resources aren't wasted on features that will not be used. 304 */ 305 static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 306 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 307 308 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 309 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 310 311 static int t4_toecaps_allowed = -1; 312 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 313 314 static int t4_rdmacaps_allowed = 0; 315 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 316 317 static int t4_iscsicaps_allowed = 0; 318 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 319 320 static int t4_fcoecaps_allowed = 0; 321 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 322 323 static int t5_write_combine = 0; 324 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 325 326 struct intrs_and_queues { 327 uint16_t intr_type; /* INTx, MSI, or MSI-X */ 328 uint16_t nirq; /* Total # of vectors */ 329 uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */ 330 uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */ 331 uint16_t ntxq10g; /* # of NIC txq's for each 10G port */ 332 uint16_t nrxq10g; /* # of NIC rxq's for each 10G port */ 333 uint16_t ntxq1g; /* # of NIC txq's for each 1G port */ 334 uint16_t nrxq1g; /* # of NIC rxq's for each 1G port */ 335 uint16_t rsrv_noflowq; /* Flag whether to reserve queue 0 */ 336 #ifdef TCP_OFFLOAD 337 uint16_t nofldtxq10g; /* # of TOE txq's for each 10G port */ 338 uint16_t nofldrxq10g; /* # of TOE rxq's for each 10G port */ 339 uint16_t nofldtxq1g; /* # of TOE txq's for each 1G port */ 340 uint16_t nofldrxq1g; /* # of TOE rxq's for each 1G port */ 341 #endif 342 #ifdef DEV_NETMAP 343 uint16_t nnmtxq10g; /* # of netmap txq's for each 10G port */ 344 uint16_t nnmrxq10g; /* # of netmap rxq's for each 10G port */ 345 uint16_t nnmtxq1g; /* # of netmap txq's for each 1G port */ 346 uint16_t nnmrxq1g; /* # of netmap rxq's for each 1G port */ 347 #endif 348 }; 349 350 struct filter_entry { 351 uint32_t valid:1; /* filter allocated and valid */ 352 uint32_t locked:1; /* filter is administratively locked */ 353 uint32_t pending:1; /* filter action is pending firmware reply */ 354 uint32_t smtidx:8; /* Source MAC Table index for smac */ 355 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 356 357 struct t4_filter_specification fs; 358 }; 359 360 static int map_bars_0_and_4(struct adapter *); 361 static int map_bar_2(struct adapter *); 362 static void setup_memwin(struct adapter *); 363 static int validate_mem_range(struct adapter *, uint32_t, int); 364 static int fwmtype_to_hwmtype(int); 365 static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 366 uint32_t *); 367 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 368 static uint32_t position_memwin(struct adapter *, int, uint32_t); 369 static int cfg_itype_and_nqueues(struct adapter *, int, int, 370 struct intrs_and_queues *); 371 static int prep_firmware(struct adapter *); 372 static int partition_resources(struct adapter *, const struct firmware *, 373 const char *); 374 static int get_params__pre_init(struct adapter *); 375 static int get_params__post_init(struct adapter *); 376 static int set_params__post_init(struct adapter *); 377 static void t4_set_desc(struct adapter *); 378 static void build_medialist(struct port_info *, struct ifmedia *); 379 static int cxgbe_init_synchronized(struct port_info *); 380 static int cxgbe_uninit_synchronized(struct port_info *); 381 static int setup_intr_handlers(struct adapter *); 382 static void quiesce_txq(struct adapter *, struct sge_txq *); 383 static void quiesce_wrq(struct adapter *, struct sge_wrq *); 384 static void quiesce_iq(struct adapter *, struct sge_iq *); 385 static void quiesce_fl(struct adapter *, struct sge_fl *); 386 static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 387 driver_intr_t *, void *, char *); 388 static int t4_free_irq(struct adapter *, struct irq *); 389 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 390 unsigned int); 391 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 392 static void cxgbe_refresh_stats(struct adapter *, struct port_info *); 393 static void cxgbe_tick(void *); 394 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 395 static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 396 struct mbuf *); 397 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 398 static int fw_msg_not_handled(struct adapter *, const __be64 *); 399 static int t4_sysctls(struct adapter *); 400 static int cxgbe_sysctls(struct port_info *); 401 static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 402 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 403 static int sysctl_btphy(SYSCTL_HANDLER_ARGS); 404 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); 405 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 406 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 407 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 408 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 409 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS); 410 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 411 static int sysctl_temperature(SYSCTL_HANDLER_ARGS); 412 #ifdef SBUF_DRAIN 413 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 414 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 415 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 416 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); 417 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); 418 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 419 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 420 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 421 static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 422 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 423 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 424 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 425 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); 426 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 427 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); 428 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 429 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 430 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 431 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 432 static int sysctl_tids(SYSCTL_HANDLER_ARGS); 433 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 434 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 435 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 436 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 437 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 438 #endif 439 static uint32_t fconf_to_mode(uint32_t); 440 static uint32_t mode_to_fconf(uint32_t); 441 static uint32_t fspec_to_fconf(struct t4_filter_specification *); 442 static int get_filter_mode(struct adapter *, uint32_t *); 443 static int set_filter_mode(struct adapter *, uint32_t); 444 static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 445 static int get_filter(struct adapter *, struct t4_filter *); 446 static int set_filter(struct adapter *, struct t4_filter *); 447 static int del_filter(struct adapter *, struct t4_filter *); 448 static void clear_filter(struct filter_entry *); 449 static int set_filter_wr(struct adapter *, int); 450 static int del_filter_wr(struct adapter *, int); 451 static int get_sge_context(struct adapter *, struct t4_sge_context *); 452 static int load_fw(struct adapter *, struct t4_data *); 453 static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 454 static int read_i2c(struct adapter *, struct t4_i2c_data *); 455 static int set_sched_class(struct adapter *, struct t4_sched_params *); 456 static int set_sched_queue(struct adapter *, struct t4_sched_queue *); 457 #ifdef TCP_OFFLOAD 458 static int toe_capability(struct port_info *, int); 459 #endif 460 static int mod_event(module_t, int, void *); 461 462 struct { 463 uint16_t device; 464 char *desc; 465 } t4_pciids[] = { 466 {0xa000, "Chelsio Terminator 4 FPGA"}, 467 {0x4400, "Chelsio T440-dbg"}, 468 {0x4401, "Chelsio T420-CR"}, 469 {0x4402, "Chelsio T422-CR"}, 470 {0x4403, "Chelsio T440-CR"}, 471 {0x4404, "Chelsio T420-BCH"}, 472 {0x4405, "Chelsio T440-BCH"}, 473 {0x4406, "Chelsio T440-CH"}, 474 {0x4407, "Chelsio T420-SO"}, 475 {0x4408, "Chelsio T420-CX"}, 476 {0x4409, "Chelsio T420-BT"}, 477 {0x440a, "Chelsio T404-BT"}, 478 {0x440e, "Chelsio T440-LP-CR"}, 479 }, t5_pciids[] = { 480 {0xb000, "Chelsio Terminator 5 FPGA"}, 481 {0x5400, "Chelsio T580-dbg"}, 482 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ 483 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ 484 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ 485 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ 486 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ 487 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ 488 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ 489 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ 490 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 491 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ 492 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ 493 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ 494 {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */ 495 #ifdef notyet 496 {0x5404, "Chelsio T520-BCH"}, 497 {0x5405, "Chelsio T540-BCH"}, 498 {0x5406, "Chelsio T540-CH"}, 499 {0x5408, "Chelsio T520-CX"}, 500 {0x540b, "Chelsio B520-SR"}, 501 {0x540c, "Chelsio B504-BT"}, 502 {0x540f, "Chelsio Amsterdam"}, 503 {0x5413, "Chelsio T580-CHR"}, 504 #endif 505 }; 506 507 #ifdef TCP_OFFLOAD 508 /* 509 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 510 * exactly the same for both rxq and ofld_rxq. 511 */ 512 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 513 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 514 #endif 515 516 /* No easy way to include t4_msg.h before adapter.h so we check this way */ 517 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 518 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 519 520 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); 521 522 static int 523 t4_probe(device_t dev) 524 { 525 int i; 526 uint16_t v = pci_get_vendor(dev); 527 uint16_t d = pci_get_device(dev); 528 uint8_t f = pci_get_function(dev); 529 530 if (v != PCI_VENDOR_ID_CHELSIO) 531 return (ENXIO); 532 533 /* Attach only to PF0 of the FPGA */ 534 if (d == 0xa000 && f != 0) 535 return (ENXIO); 536 537 for (i = 0; i < nitems(t4_pciids); i++) { 538 if (d == t4_pciids[i].device) { 539 device_set_desc(dev, t4_pciids[i].desc); 540 return (BUS_PROBE_DEFAULT); 541 } 542 } 543 544 return (ENXIO); 545 } 546 547 static int 548 t5_probe(device_t dev) 549 { 550 int i; 551 uint16_t v = pci_get_vendor(dev); 552 uint16_t d = pci_get_device(dev); 553 uint8_t f = pci_get_function(dev); 554 555 if (v != PCI_VENDOR_ID_CHELSIO) 556 return (ENXIO); 557 558 /* Attach only to PF0 of the FPGA */ 559 if (d == 0xb000 && f != 0) 560 return (ENXIO); 561 562 for (i = 0; i < nitems(t5_pciids); i++) { 563 if (d == t5_pciids[i].device) { 564 device_set_desc(dev, t5_pciids[i].desc); 565 return (BUS_PROBE_DEFAULT); 566 } 567 } 568 569 return (ENXIO); 570 } 571 572 static int 573 t4_attach(device_t dev) 574 { 575 struct adapter *sc; 576 int rc = 0, i, n10g, n1g, rqidx, tqidx; 577 struct intrs_and_queues iaq; 578 struct sge *s; 579 #ifdef TCP_OFFLOAD 580 int ofld_rqidx, ofld_tqidx; 581 #endif 582 #ifdef DEV_NETMAP 583 int nm_rqidx, nm_tqidx; 584 #endif 585 const char *pcie_ts; 586 587 sc = device_get_softc(dev); 588 sc->dev = dev; 589 590 pci_enable_busmaster(dev); 591 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 592 uint32_t v; 593 594 pci_set_max_read_req(dev, 4096); 595 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 596 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 597 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 598 599 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5); 600 } 601 602 sc->traceq = -1; 603 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); 604 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", 605 device_get_nameunit(dev)); 606 607 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 608 device_get_nameunit(dev)); 609 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 610 sx_xlock(&t4_list_lock); 611 SLIST_INSERT_HEAD(&t4_list, sc, link); 612 sx_xunlock(&t4_list_lock); 613 614 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 615 TAILQ_INIT(&sc->sfl); 616 callout_init(&sc->sfl_callout, CALLOUT_MPSAFE); 617 618 mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF); 619 620 rc = map_bars_0_and_4(sc); 621 if (rc != 0) 622 goto done; /* error message displayed already */ 623 624 /* 625 * This is the real PF# to which we're attaching. Works from within PCI 626 * passthrough environments too, where pci_get_function() could return a 627 * different PF# depending on the passthrough configuration. We need to 628 * use the real PF# in all our communication with the firmware. 629 */ 630 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 631 sc->mbox = sc->pf; 632 633 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 634 sc->an_handler = an_not_handled; 635 for (i = 0; i < nitems(sc->cpl_handler); i++) 636 sc->cpl_handler[i] = cpl_not_handled; 637 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 638 sc->fw_msg_handler[i] = fw_msg_not_handled; 639 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 640 t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt); 641 t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt); 642 t4_init_sge_cpl_handlers(sc); 643 644 /* Prepare the adapter for operation */ 645 rc = -t4_prep_adapter(sc); 646 if (rc != 0) { 647 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 648 goto done; 649 } 650 651 /* 652 * Do this really early, with the memory windows set up even before the 653 * character device. The userland tool's register i/o and mem read 654 * will work even in "recovery mode". 655 */ 656 setup_memwin(sc); 657 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 658 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 659 device_get_nameunit(dev)); 660 if (sc->cdev == NULL) 661 device_printf(dev, "failed to create nexus char device.\n"); 662 else 663 sc->cdev->si_drv1 = sc; 664 665 /* Go no further if recovery mode has been requested. */ 666 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 667 device_printf(dev, "recovery mode.\n"); 668 goto done; 669 } 670 671 /* Prepare the firmware for operation */ 672 rc = prep_firmware(sc); 673 if (rc != 0) 674 goto done; /* error message displayed already */ 675 676 rc = get_params__post_init(sc); 677 if (rc != 0) 678 goto done; /* error message displayed already */ 679 680 rc = set_params__post_init(sc); 681 if (rc != 0) 682 goto done; /* error message displayed already */ 683 684 rc = map_bar_2(sc); 685 if (rc != 0) 686 goto done; /* error message displayed already */ 687 688 rc = t4_create_dma_tag(sc); 689 if (rc != 0) 690 goto done; /* error message displayed already */ 691 692 /* 693 * First pass over all the ports - allocate VIs and initialize some 694 * basic parameters like mac address, port type, etc. We also figure 695 * out whether a port is 10G or 1G and use that information when 696 * calculating how many interrupts to attempt to allocate. 697 */ 698 n10g = n1g = 0; 699 for_each_port(sc, i) { 700 struct port_info *pi; 701 702 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 703 sc->port[i] = pi; 704 705 /* These must be set before t4_port_init */ 706 pi->adapter = sc; 707 pi->port_id = i; 708 709 /* Allocate the vi and initialize parameters like mac addr */ 710 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 711 if (rc != 0) { 712 device_printf(dev, "unable to initialize port %d: %d\n", 713 i, rc); 714 free(pi, M_CXGBE); 715 sc->port[i] = NULL; 716 goto done; 717 } 718 719 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX); 720 pi->link_cfg.requested_fc |= t4_pause_settings; 721 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX); 722 pi->link_cfg.fc |= t4_pause_settings; 723 724 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 725 if (rc != 0) { 726 device_printf(dev, "port %d l1cfg failed: %d\n", i, rc); 727 free(pi, M_CXGBE); 728 sc->port[i] = NULL; 729 goto done; 730 } 731 732 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 733 device_get_nameunit(dev), i); 734 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 735 sc->chan_map[pi->tx_chan] = i; 736 737 if (is_10G_port(pi) || is_40G_port(pi)) { 738 n10g++; 739 pi->tmr_idx = t4_tmr_idx_10g; 740 pi->pktc_idx = t4_pktc_idx_10g; 741 } else { 742 n1g++; 743 pi->tmr_idx = t4_tmr_idx_1g; 744 pi->pktc_idx = t4_pktc_idx_1g; 745 } 746 747 pi->xact_addr_filt = -1; 748 pi->linkdnrc = -1; 749 750 pi->qsize_rxq = t4_qsize_rxq; 751 pi->qsize_txq = t4_qsize_txq; 752 753 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 754 if (pi->dev == NULL) { 755 device_printf(dev, 756 "failed to add device for port %d.\n", i); 757 rc = ENXIO; 758 goto done; 759 } 760 device_set_softc(pi->dev, pi); 761 } 762 763 /* 764 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 765 */ 766 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 767 if (rc != 0) 768 goto done; /* error message displayed already */ 769 770 sc->intr_type = iaq.intr_type; 771 sc->intr_count = iaq.nirq; 772 773 s = &sc->sge; 774 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 775 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 776 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 777 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 778 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 779 #ifdef TCP_OFFLOAD 780 if (is_offload(sc)) { 781 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 782 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 783 s->neq += s->nofldtxq + s->nofldrxq; 784 s->niq += s->nofldrxq; 785 786 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 787 M_CXGBE, M_ZERO | M_WAITOK); 788 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 789 M_CXGBE, M_ZERO | M_WAITOK); 790 } 791 #endif 792 #ifdef DEV_NETMAP 793 s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g; 794 s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g; 795 s->neq += s->nnmtxq + s->nnmrxq; 796 s->niq += s->nnmrxq; 797 798 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq), 799 M_CXGBE, M_ZERO | M_WAITOK); 800 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq), 801 M_CXGBE, M_ZERO | M_WAITOK); 802 #endif 803 804 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 805 M_ZERO | M_WAITOK); 806 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 807 M_ZERO | M_WAITOK); 808 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 809 M_ZERO | M_WAITOK); 810 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 811 M_ZERO | M_WAITOK); 812 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 813 M_ZERO | M_WAITOK); 814 815 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 816 M_ZERO | M_WAITOK); 817 818 t4_init_l2t(sc, M_WAITOK); 819 820 /* 821 * Second pass over the ports. This time we know the number of rx and 822 * tx queues that each port should get. 823 */ 824 rqidx = tqidx = 0; 825 #ifdef TCP_OFFLOAD 826 ofld_rqidx = ofld_tqidx = 0; 827 #endif 828 #ifdef DEV_NETMAP 829 nm_rqidx = nm_tqidx = 0; 830 #endif 831 for_each_port(sc, i) { 832 struct port_info *pi = sc->port[i]; 833 834 if (pi == NULL) 835 continue; 836 837 pi->first_rxq = rqidx; 838 pi->first_txq = tqidx; 839 if (is_10G_port(pi) || is_40G_port(pi)) { 840 pi->flags |= iaq.intr_flags_10g; 841 pi->nrxq = iaq.nrxq10g; 842 pi->ntxq = iaq.ntxq10g; 843 } else { 844 pi->flags |= iaq.intr_flags_1g; 845 pi->nrxq = iaq.nrxq1g; 846 pi->ntxq = iaq.ntxq1g; 847 } 848 849 if (pi->ntxq > 1) 850 pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0; 851 else 852 pi->rsrv_noflowq = 0; 853 854 rqidx += pi->nrxq; 855 tqidx += pi->ntxq; 856 #ifdef TCP_OFFLOAD 857 if (is_offload(sc)) { 858 pi->first_ofld_rxq = ofld_rqidx; 859 pi->first_ofld_txq = ofld_tqidx; 860 if (is_10G_port(pi) || is_40G_port(pi)) { 861 pi->nofldrxq = iaq.nofldrxq10g; 862 pi->nofldtxq = iaq.nofldtxq10g; 863 } else { 864 pi->nofldrxq = iaq.nofldrxq1g; 865 pi->nofldtxq = iaq.nofldtxq1g; 866 } 867 ofld_rqidx += pi->nofldrxq; 868 ofld_tqidx += pi->nofldtxq; 869 } 870 #endif 871 #ifdef DEV_NETMAP 872 pi->first_nm_rxq = nm_rqidx; 873 pi->first_nm_txq = nm_tqidx; 874 if (is_10G_port(pi) || is_40G_port(pi)) { 875 pi->nnmrxq = iaq.nnmrxq10g; 876 pi->nnmtxq = iaq.nnmtxq10g; 877 } else { 878 pi->nnmrxq = iaq.nnmrxq1g; 879 pi->nnmtxq = iaq.nnmtxq1g; 880 } 881 nm_rqidx += pi->nnmrxq; 882 nm_tqidx += pi->nnmtxq; 883 #endif 884 } 885 886 rc = setup_intr_handlers(sc); 887 if (rc != 0) { 888 device_printf(dev, 889 "failed to setup interrupt handlers: %d\n", rc); 890 goto done; 891 } 892 893 rc = bus_generic_attach(dev); 894 if (rc != 0) { 895 device_printf(dev, 896 "failed to attach all child ports: %d\n", rc); 897 goto done; 898 } 899 900 switch (sc->params.pci.speed) { 901 case 0x1: 902 pcie_ts = "2.5"; 903 break; 904 case 0x2: 905 pcie_ts = "5.0"; 906 break; 907 case 0x3: 908 pcie_ts = "8.0"; 909 break; 910 default: 911 pcie_ts = "??"; 912 break; 913 } 914 device_printf(dev, 915 "PCIe x%d (%s GTS/s) (%d), %d ports, %d %s interrupt%s, %d eq, %d iq\n", 916 sc->params.pci.width, pcie_ts, sc->params.pci.speed, 917 sc->params.nports, sc->intr_count, 918 sc->intr_type == INTR_MSIX ? "MSI-X" : 919 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 920 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 921 922 t4_set_desc(sc); 923 924 done: 925 if (rc != 0 && sc->cdev) { 926 /* cdev was created and so cxgbetool works; recover that way. */ 927 device_printf(dev, 928 "error during attach, adapter is now in recovery mode.\n"); 929 rc = 0; 930 } 931 932 if (rc != 0) 933 t4_detach(dev); 934 else 935 t4_sysctls(sc); 936 937 return (rc); 938 } 939 940 /* 941 * Idempotent 942 */ 943 static int 944 t4_detach(device_t dev) 945 { 946 struct adapter *sc; 947 struct port_info *pi; 948 int i, rc; 949 950 sc = device_get_softc(dev); 951 952 if (sc->flags & FULL_INIT_DONE) 953 t4_intr_disable(sc); 954 955 if (sc->cdev) { 956 destroy_dev(sc->cdev); 957 sc->cdev = NULL; 958 } 959 960 rc = bus_generic_detach(dev); 961 if (rc) { 962 device_printf(dev, 963 "failed to detach child devices: %d\n", rc); 964 return (rc); 965 } 966 967 for (i = 0; i < sc->intr_count; i++) 968 t4_free_irq(sc, &sc->irq[i]); 969 970 for (i = 0; i < MAX_NPORTS; i++) { 971 pi = sc->port[i]; 972 if (pi) { 973 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid); 974 if (pi->dev) 975 device_delete_child(dev, pi->dev); 976 977 mtx_destroy(&pi->pi_lock); 978 free(pi, M_CXGBE); 979 } 980 } 981 982 if (sc->flags & FULL_INIT_DONE) 983 adapter_full_uninit(sc); 984 985 if (sc->flags & FW_OK) 986 t4_fw_bye(sc, sc->mbox); 987 988 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 989 pci_release_msi(dev); 990 991 if (sc->regs_res) 992 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 993 sc->regs_res); 994 995 if (sc->udbs_res) 996 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 997 sc->udbs_res); 998 999 if (sc->msix_res) 1000 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 1001 sc->msix_res); 1002 1003 if (sc->l2t) 1004 t4_free_l2t(sc->l2t); 1005 1006 #ifdef TCP_OFFLOAD 1007 free(sc->sge.ofld_rxq, M_CXGBE); 1008 free(sc->sge.ofld_txq, M_CXGBE); 1009 #endif 1010 #ifdef DEV_NETMAP 1011 free(sc->sge.nm_rxq, M_CXGBE); 1012 free(sc->sge.nm_txq, M_CXGBE); 1013 #endif 1014 free(sc->irq, M_CXGBE); 1015 free(sc->sge.rxq, M_CXGBE); 1016 free(sc->sge.txq, M_CXGBE); 1017 free(sc->sge.ctrlq, M_CXGBE); 1018 free(sc->sge.iqmap, M_CXGBE); 1019 free(sc->sge.eqmap, M_CXGBE); 1020 free(sc->tids.ftid_tab, M_CXGBE); 1021 t4_destroy_dma_tag(sc); 1022 if (mtx_initialized(&sc->sc_lock)) { 1023 sx_xlock(&t4_list_lock); 1024 SLIST_REMOVE(&t4_list, sc, adapter, link); 1025 sx_xunlock(&t4_list_lock); 1026 mtx_destroy(&sc->sc_lock); 1027 } 1028 1029 if (mtx_initialized(&sc->tids.ftid_lock)) 1030 mtx_destroy(&sc->tids.ftid_lock); 1031 if (mtx_initialized(&sc->sfl_lock)) 1032 mtx_destroy(&sc->sfl_lock); 1033 if (mtx_initialized(&sc->ifp_lock)) 1034 mtx_destroy(&sc->ifp_lock); 1035 if (mtx_initialized(&sc->regwin_lock)) 1036 mtx_destroy(&sc->regwin_lock); 1037 1038 bzero(sc, sizeof(*sc)); 1039 1040 return (0); 1041 } 1042 1043 static int 1044 cxgbe_probe(device_t dev) 1045 { 1046 char buf[128]; 1047 struct port_info *pi = device_get_softc(dev); 1048 1049 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 1050 device_set_desc_copy(dev, buf); 1051 1052 return (BUS_PROBE_DEFAULT); 1053 } 1054 1055 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 1056 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 1057 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS) 1058 #define T4_CAP_ENABLE (T4_CAP) 1059 1060 static int 1061 cxgbe_attach(device_t dev) 1062 { 1063 struct port_info *pi = device_get_softc(dev); 1064 struct ifnet *ifp; 1065 char *s; 1066 int n, o; 1067 1068 /* Allocate an ifnet and set it up */ 1069 ifp = if_alloc(IFT_ETHER); 1070 if (ifp == NULL) { 1071 device_printf(dev, "Cannot allocate ifnet\n"); 1072 return (ENOMEM); 1073 } 1074 pi->ifp = ifp; 1075 ifp->if_softc = pi; 1076 1077 callout_init(&pi->tick, CALLOUT_MPSAFE); 1078 1079 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1080 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1081 1082 ifp->if_init = cxgbe_init; 1083 ifp->if_ioctl = cxgbe_ioctl; 1084 ifp->if_transmit = cxgbe_transmit; 1085 ifp->if_qflush = cxgbe_qflush; 1086 ifp->if_get_counter = cxgbe_get_counter; 1087 1088 ifp->if_capabilities = T4_CAP; 1089 #ifdef TCP_OFFLOAD 1090 if (is_offload(pi->adapter)) 1091 ifp->if_capabilities |= IFCAP_TOE; 1092 #endif 1093 ifp->if_capenable = T4_CAP_ENABLE; 1094 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1095 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1096 1097 ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1098 ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS; 1099 ifp->if_hw_tsomaxsegsize = 65536; 1100 1101 /* Initialize ifmedia for this port */ 1102 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 1103 cxgbe_media_status); 1104 build_medialist(pi, &pi->media); 1105 1106 pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 1107 EVENTHANDLER_PRI_ANY); 1108 1109 ether_ifattach(ifp, pi->hw_addr); 1110 1111 n = 128; 1112 s = malloc(n, M_CXGBE, M_WAITOK); 1113 o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq); 1114 MPASS(n > o); 1115 #ifdef TCP_OFFLOAD 1116 if (is_offload(pi->adapter)) { 1117 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)", 1118 pi->nofldtxq, pi->nofldrxq); 1119 MPASS(n > o); 1120 } 1121 #endif 1122 #ifdef DEV_NETMAP 1123 o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq, 1124 pi->nnmrxq); 1125 MPASS(n > o); 1126 #endif 1127 device_printf(dev, "%s\n", s); 1128 free(s, M_CXGBE); 1129 1130 #ifdef DEV_NETMAP 1131 /* nm_media handled here to keep implementation private to this file */ 1132 ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change, 1133 cxgbe_media_status); 1134 build_medialist(pi, &pi->nm_media); 1135 create_netmap_ifnet(pi); /* logs errors it something fails */ 1136 #endif 1137 cxgbe_sysctls(pi); 1138 1139 return (0); 1140 } 1141 1142 static int 1143 cxgbe_detach(device_t dev) 1144 { 1145 struct port_info *pi = device_get_softc(dev); 1146 struct adapter *sc = pi->adapter; 1147 struct ifnet *ifp = pi->ifp; 1148 1149 /* Tell if_ioctl and if_init that the port is going away */ 1150 ADAPTER_LOCK(sc); 1151 SET_DOOMED(pi); 1152 wakeup(&sc->flags); 1153 while (IS_BUSY(sc)) 1154 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 1155 SET_BUSY(sc); 1156 #ifdef INVARIANTS 1157 sc->last_op = "t4detach"; 1158 sc->last_op_thr = curthread; 1159 #endif 1160 ADAPTER_UNLOCK(sc); 1161 1162 if (pi->flags & HAS_TRACEQ) { 1163 sc->traceq = -1; /* cloner should not create ifnet */ 1164 t4_tracer_port_detach(sc); 1165 } 1166 1167 if (pi->vlan_c) 1168 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c); 1169 1170 PORT_LOCK(pi); 1171 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1172 callout_stop(&pi->tick); 1173 PORT_UNLOCK(pi); 1174 callout_drain(&pi->tick); 1175 1176 /* Let detach proceed even if these fail. */ 1177 cxgbe_uninit_synchronized(pi); 1178 port_full_uninit(pi); 1179 1180 ifmedia_removeall(&pi->media); 1181 ether_ifdetach(pi->ifp); 1182 if_free(pi->ifp); 1183 1184 #ifdef DEV_NETMAP 1185 /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */ 1186 destroy_netmap_ifnet(pi); 1187 #endif 1188 1189 ADAPTER_LOCK(sc); 1190 CLR_BUSY(sc); 1191 wakeup(&sc->flags); 1192 ADAPTER_UNLOCK(sc); 1193 1194 return (0); 1195 } 1196 1197 static void 1198 cxgbe_init(void *arg) 1199 { 1200 struct port_info *pi = arg; 1201 struct adapter *sc = pi->adapter; 1202 1203 if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0) 1204 return; 1205 cxgbe_init_synchronized(pi); 1206 end_synchronized_op(sc, 0); 1207 } 1208 1209 static int 1210 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1211 { 1212 int rc = 0, mtu, flags, can_sleep; 1213 struct port_info *pi = ifp->if_softc; 1214 struct adapter *sc = pi->adapter; 1215 struct ifreq *ifr = (struct ifreq *)data; 1216 uint32_t mask; 1217 1218 switch (cmd) { 1219 case SIOCSIFMTU: 1220 mtu = ifr->ifr_mtu; 1221 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1222 return (EINVAL); 1223 1224 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu"); 1225 if (rc) 1226 return (rc); 1227 ifp->if_mtu = mtu; 1228 if (pi->flags & PORT_INIT_DONE) { 1229 t4_update_fl_bufsize(ifp); 1230 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1231 rc = update_mac_settings(ifp, XGMAC_MTU); 1232 } 1233 end_synchronized_op(sc, 0); 1234 break; 1235 1236 case SIOCSIFFLAGS: 1237 can_sleep = 0; 1238 redo_sifflags: 1239 rc = begin_synchronized_op(sc, pi, 1240 can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg"); 1241 if (rc) 1242 return (rc); 1243 1244 if (ifp->if_flags & IFF_UP) { 1245 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1246 flags = pi->if_flags; 1247 if ((ifp->if_flags ^ flags) & 1248 (IFF_PROMISC | IFF_ALLMULTI)) { 1249 if (can_sleep == 1) { 1250 end_synchronized_op(sc, 0); 1251 can_sleep = 0; 1252 goto redo_sifflags; 1253 } 1254 rc = update_mac_settings(ifp, 1255 XGMAC_PROMISC | XGMAC_ALLMULTI); 1256 } 1257 } else { 1258 if (can_sleep == 0) { 1259 end_synchronized_op(sc, LOCK_HELD); 1260 can_sleep = 1; 1261 goto redo_sifflags; 1262 } 1263 rc = cxgbe_init_synchronized(pi); 1264 } 1265 pi->if_flags = ifp->if_flags; 1266 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1267 if (can_sleep == 0) { 1268 end_synchronized_op(sc, LOCK_HELD); 1269 can_sleep = 1; 1270 goto redo_sifflags; 1271 } 1272 rc = cxgbe_uninit_synchronized(pi); 1273 } 1274 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD); 1275 break; 1276 1277 case SIOCADDMULTI: 1278 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1279 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi"); 1280 if (rc) 1281 return (rc); 1282 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1283 rc = update_mac_settings(ifp, XGMAC_MCADDRS); 1284 end_synchronized_op(sc, LOCK_HELD); 1285 break; 1286 1287 case SIOCSIFCAP: 1288 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap"); 1289 if (rc) 1290 return (rc); 1291 1292 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1293 if (mask & IFCAP_TXCSUM) { 1294 ifp->if_capenable ^= IFCAP_TXCSUM; 1295 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1296 1297 if (IFCAP_TSO4 & ifp->if_capenable && 1298 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1299 ifp->if_capenable &= ~IFCAP_TSO4; 1300 if_printf(ifp, 1301 "tso4 disabled due to -txcsum.\n"); 1302 } 1303 } 1304 if (mask & IFCAP_TXCSUM_IPV6) { 1305 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1306 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1307 1308 if (IFCAP_TSO6 & ifp->if_capenable && 1309 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1310 ifp->if_capenable &= ~IFCAP_TSO6; 1311 if_printf(ifp, 1312 "tso6 disabled due to -txcsum6.\n"); 1313 } 1314 } 1315 if (mask & IFCAP_RXCSUM) 1316 ifp->if_capenable ^= IFCAP_RXCSUM; 1317 if (mask & IFCAP_RXCSUM_IPV6) 1318 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1319 1320 /* 1321 * Note that we leave CSUM_TSO alone (it is always set). The 1322 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1323 * sending a TSO request our way, so it's sufficient to toggle 1324 * IFCAP_TSOx only. 1325 */ 1326 if (mask & IFCAP_TSO4) { 1327 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1328 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1329 if_printf(ifp, "enable txcsum first.\n"); 1330 rc = EAGAIN; 1331 goto fail; 1332 } 1333 ifp->if_capenable ^= IFCAP_TSO4; 1334 } 1335 if (mask & IFCAP_TSO6) { 1336 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1337 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1338 if_printf(ifp, "enable txcsum6 first.\n"); 1339 rc = EAGAIN; 1340 goto fail; 1341 } 1342 ifp->if_capenable ^= IFCAP_TSO6; 1343 } 1344 if (mask & IFCAP_LRO) { 1345 #if defined(INET) || defined(INET6) 1346 int i; 1347 struct sge_rxq *rxq; 1348 1349 ifp->if_capenable ^= IFCAP_LRO; 1350 for_each_rxq(pi, i, rxq) { 1351 if (ifp->if_capenable & IFCAP_LRO) 1352 rxq->iq.flags |= IQ_LRO_ENABLED; 1353 else 1354 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1355 } 1356 #endif 1357 } 1358 #ifdef TCP_OFFLOAD 1359 if (mask & IFCAP_TOE) { 1360 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1361 1362 rc = toe_capability(pi, enable); 1363 if (rc != 0) 1364 goto fail; 1365 1366 ifp->if_capenable ^= mask; 1367 } 1368 #endif 1369 if (mask & IFCAP_VLAN_HWTAGGING) { 1370 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1371 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1372 rc = update_mac_settings(ifp, XGMAC_VLANEX); 1373 } 1374 if (mask & IFCAP_VLAN_MTU) { 1375 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1376 1377 /* Need to find out how to disable auto-mtu-inflation */ 1378 } 1379 if (mask & IFCAP_VLAN_HWTSO) 1380 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1381 if (mask & IFCAP_VLAN_HWCSUM) 1382 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1383 1384 #ifdef VLAN_CAPABILITIES 1385 VLAN_CAPABILITIES(ifp); 1386 #endif 1387 fail: 1388 end_synchronized_op(sc, 0); 1389 break; 1390 1391 case SIOCSIFMEDIA: 1392 case SIOCGIFMEDIA: 1393 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1394 break; 1395 1396 case SIOCGI2C: { 1397 struct ifi2creq i2c; 1398 1399 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); 1400 if (rc != 0) 1401 break; 1402 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) { 1403 rc = EPERM; 1404 break; 1405 } 1406 if (i2c.len > sizeof(i2c.data)) { 1407 rc = EINVAL; 1408 break; 1409 } 1410 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4i2c"); 1411 if (rc) 1412 return (rc); 1413 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr, 1414 i2c.offset, i2c.len, &i2c.data[0]); 1415 end_synchronized_op(sc, 0); 1416 if (rc == 0) 1417 rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); 1418 break; 1419 } 1420 1421 default: 1422 rc = ether_ioctl(ifp, cmd, data); 1423 } 1424 1425 return (rc); 1426 } 1427 1428 static int 1429 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1430 { 1431 struct port_info *pi = ifp->if_softc; 1432 struct adapter *sc = pi->adapter; 1433 struct sge_txq *txq; 1434 void *items[1]; 1435 int rc; 1436 1437 M_ASSERTPKTHDR(m); 1438 MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */ 1439 1440 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1441 m_freem(m); 1442 return (ENETDOWN); 1443 } 1444 1445 rc = parse_pkt(&m); 1446 if (__predict_false(rc != 0)) { 1447 MPASS(m == NULL); /* was freed already */ 1448 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */ 1449 return (rc); 1450 } 1451 1452 /* Select a txq. */ 1453 txq = &sc->sge.txq[pi->first_txq]; 1454 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 1455 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) + 1456 pi->rsrv_noflowq); 1457 1458 items[0] = m; 1459 rc = mp_ring_enqueue(txq->r, items, 1, 4096); 1460 if (__predict_false(rc != 0)) 1461 m_freem(m); 1462 1463 return (rc); 1464 } 1465 1466 static void 1467 cxgbe_qflush(struct ifnet *ifp) 1468 { 1469 struct port_info *pi = ifp->if_softc; 1470 struct sge_txq *txq; 1471 int i; 1472 1473 /* queues do not exist if !PORT_INIT_DONE. */ 1474 if (pi->flags & PORT_INIT_DONE) { 1475 for_each_txq(pi, i, txq) { 1476 TXQ_LOCK(txq); 1477 txq->eq.flags &= ~EQ_ENABLED; 1478 TXQ_UNLOCK(txq); 1479 while (!mp_ring_is_idle(txq->r)) { 1480 mp_ring_check_drainage(txq->r, 0); 1481 pause("qflush", 1); 1482 } 1483 } 1484 } 1485 if_qflush(ifp); 1486 } 1487 1488 static uint64_t 1489 cxgbe_get_counter(struct ifnet *ifp, ift_counter c) 1490 { 1491 struct port_info *pi = ifp->if_softc; 1492 struct adapter *sc = pi->adapter; 1493 struct port_stats *s = &pi->stats; 1494 1495 cxgbe_refresh_stats(sc, pi); 1496 1497 switch (c) { 1498 case IFCOUNTER_IPACKETS: 1499 return (s->rx_frames - s->rx_pause); 1500 1501 case IFCOUNTER_IERRORS: 1502 return (s->rx_jabber + s->rx_runt + s->rx_too_long + 1503 s->rx_fcs_err + s->rx_len_err); 1504 1505 case IFCOUNTER_OPACKETS: 1506 return (s->tx_frames - s->tx_pause); 1507 1508 case IFCOUNTER_OERRORS: 1509 return (s->tx_error_frames); 1510 1511 case IFCOUNTER_IBYTES: 1512 return (s->rx_octets - s->rx_pause * 64); 1513 1514 case IFCOUNTER_OBYTES: 1515 return (s->tx_octets - s->tx_pause * 64); 1516 1517 case IFCOUNTER_IMCASTS: 1518 return (s->rx_mcast_frames - s->rx_pause); 1519 1520 case IFCOUNTER_OMCASTS: 1521 return (s->tx_mcast_frames - s->tx_pause); 1522 1523 case IFCOUNTER_IQDROPS: 1524 return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 1525 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 1526 s->rx_trunc3 + pi->tnl_cong_drops); 1527 1528 case IFCOUNTER_OQDROPS: { 1529 uint64_t drops; 1530 1531 drops = s->tx_drop; 1532 if (pi->flags & PORT_INIT_DONE) { 1533 int i; 1534 struct sge_txq *txq; 1535 1536 for_each_txq(pi, i, txq) 1537 drops += counter_u64_fetch(txq->r->drops); 1538 } 1539 1540 return (drops); 1541 1542 } 1543 1544 default: 1545 return (if_get_counter_default(ifp, c)); 1546 } 1547 } 1548 1549 static int 1550 cxgbe_media_change(struct ifnet *ifp) 1551 { 1552 struct port_info *pi = ifp->if_softc; 1553 1554 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1555 1556 return (EOPNOTSUPP); 1557 } 1558 1559 static void 1560 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1561 { 1562 struct port_info *pi = ifp->if_softc; 1563 struct ifmedia *media = NULL; 1564 struct ifmedia_entry *cur; 1565 int speed = pi->link_cfg.speed; 1566 #ifdef INVARIANTS 1567 int data = (pi->port_type << 8) | pi->mod_type; 1568 #endif 1569 1570 if (ifp == pi->ifp) 1571 media = &pi->media; 1572 #ifdef DEV_NETMAP 1573 else if (ifp == pi->nm_ifp) 1574 media = &pi->nm_media; 1575 #endif 1576 MPASS(media != NULL); 1577 1578 cur = media->ifm_cur; 1579 MPASS(cur->ifm_data == data); 1580 1581 ifmr->ifm_status = IFM_AVALID; 1582 if (!pi->link_cfg.link_ok) 1583 return; 1584 1585 ifmr->ifm_status |= IFM_ACTIVE; 1586 1587 /* active and current will differ iff current media is autoselect. */ 1588 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1589 return; 1590 1591 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1592 if (speed == SPEED_10000) 1593 ifmr->ifm_active |= IFM_10G_T; 1594 else if (speed == SPEED_1000) 1595 ifmr->ifm_active |= IFM_1000_T; 1596 else if (speed == SPEED_100) 1597 ifmr->ifm_active |= IFM_100_TX; 1598 else if (speed == SPEED_10) 1599 ifmr->ifm_active |= IFM_10_T; 1600 else 1601 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1602 speed)); 1603 } 1604 1605 void 1606 t4_fatal_err(struct adapter *sc) 1607 { 1608 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1609 t4_intr_disable(sc); 1610 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1611 device_get_nameunit(sc->dev)); 1612 } 1613 1614 static int 1615 map_bars_0_and_4(struct adapter *sc) 1616 { 1617 sc->regs_rid = PCIR_BAR(0); 1618 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1619 &sc->regs_rid, RF_ACTIVE); 1620 if (sc->regs_res == NULL) { 1621 device_printf(sc->dev, "cannot map registers.\n"); 1622 return (ENXIO); 1623 } 1624 sc->bt = rman_get_bustag(sc->regs_res); 1625 sc->bh = rman_get_bushandle(sc->regs_res); 1626 sc->mmio_len = rman_get_size(sc->regs_res); 1627 setbit(&sc->doorbells, DOORBELL_KDB); 1628 1629 sc->msix_rid = PCIR_BAR(4); 1630 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1631 &sc->msix_rid, RF_ACTIVE); 1632 if (sc->msix_res == NULL) { 1633 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1634 return (ENXIO); 1635 } 1636 1637 return (0); 1638 } 1639 1640 static int 1641 map_bar_2(struct adapter *sc) 1642 { 1643 1644 /* 1645 * T4: only iWARP driver uses the userspace doorbells. There is no need 1646 * to map it if RDMA is disabled. 1647 */ 1648 if (is_t4(sc) && sc->rdmacaps == 0) 1649 return (0); 1650 1651 sc->udbs_rid = PCIR_BAR(2); 1652 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1653 &sc->udbs_rid, RF_ACTIVE); 1654 if (sc->udbs_res == NULL) { 1655 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1656 return (ENXIO); 1657 } 1658 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1659 1660 if (is_t5(sc)) { 1661 setbit(&sc->doorbells, DOORBELL_UDB); 1662 #if defined(__i386__) || defined(__amd64__) 1663 if (t5_write_combine) { 1664 int rc; 1665 1666 /* 1667 * Enable write combining on BAR2. This is the 1668 * userspace doorbell BAR and is split into 128B 1669 * (UDBS_SEG_SIZE) doorbell regions, each associated 1670 * with an egress queue. The first 64B has the doorbell 1671 * and the second 64B can be used to submit a tx work 1672 * request with an implicit doorbell. 1673 */ 1674 1675 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1676 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1677 if (rc == 0) { 1678 clrbit(&sc->doorbells, DOORBELL_UDB); 1679 setbit(&sc->doorbells, DOORBELL_WCWR); 1680 setbit(&sc->doorbells, DOORBELL_UDBWC); 1681 } else { 1682 device_printf(sc->dev, 1683 "couldn't enable write combining: %d\n", 1684 rc); 1685 } 1686 1687 t4_write_reg(sc, A_SGE_STAT_CFG, 1688 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1689 } 1690 #endif 1691 } 1692 1693 return (0); 1694 } 1695 1696 static const struct memwin t4_memwin[] = { 1697 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1698 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1699 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1700 }; 1701 1702 static const struct memwin t5_memwin[] = { 1703 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1704 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1705 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1706 }; 1707 1708 static void 1709 setup_memwin(struct adapter *sc) 1710 { 1711 const struct memwin *mw; 1712 int i, n; 1713 uint32_t bar0; 1714 1715 if (is_t4(sc)) { 1716 /* 1717 * Read low 32b of bar0 indirectly via the hardware backdoor 1718 * mechanism. Works from within PCI passthrough environments 1719 * too, where rman_get_start() can return a different value. We 1720 * need to program the T4 memory window decoders with the actual 1721 * addresses that will be coming across the PCIe link. 1722 */ 1723 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1724 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1725 1726 mw = &t4_memwin[0]; 1727 n = nitems(t4_memwin); 1728 } else { 1729 /* T5 uses the relative offset inside the PCIe BAR */ 1730 bar0 = 0; 1731 1732 mw = &t5_memwin[0]; 1733 n = nitems(t5_memwin); 1734 } 1735 1736 for (i = 0; i < n; i++, mw++) { 1737 t4_write_reg(sc, 1738 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 1739 (mw->base + bar0) | V_BIR(0) | 1740 V_WINDOW(ilog2(mw->aperture) - 10)); 1741 } 1742 1743 /* flush */ 1744 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1745 } 1746 1747 /* 1748 * Verify that the memory range specified by the addr/len pair is valid and lies 1749 * entirely within a single region (EDCx or MCx). 1750 */ 1751 static int 1752 validate_mem_range(struct adapter *sc, uint32_t addr, int len) 1753 { 1754 uint32_t em, addr_len, maddr, mlen; 1755 1756 /* Memory can only be accessed in naturally aligned 4 byte units */ 1757 if (addr & 3 || len & 3 || len == 0) 1758 return (EINVAL); 1759 1760 /* Enabled memories */ 1761 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1762 if (em & F_EDRAM0_ENABLE) { 1763 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1764 maddr = G_EDRAM0_BASE(addr_len) << 20; 1765 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1766 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1767 addr + len <= maddr + mlen) 1768 return (0); 1769 } 1770 if (em & F_EDRAM1_ENABLE) { 1771 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1772 maddr = G_EDRAM1_BASE(addr_len) << 20; 1773 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1774 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1775 addr + len <= maddr + mlen) 1776 return (0); 1777 } 1778 if (em & F_EXT_MEM_ENABLE) { 1779 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1780 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1781 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1782 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1783 addr + len <= maddr + mlen) 1784 return (0); 1785 } 1786 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 1787 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1788 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1789 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1790 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1791 addr + len <= maddr + mlen) 1792 return (0); 1793 } 1794 1795 return (EFAULT); 1796 } 1797 1798 static int 1799 fwmtype_to_hwmtype(int mtype) 1800 { 1801 1802 switch (mtype) { 1803 case FW_MEMTYPE_EDC0: 1804 return (MEM_EDC0); 1805 case FW_MEMTYPE_EDC1: 1806 return (MEM_EDC1); 1807 case FW_MEMTYPE_EXTMEM: 1808 return (MEM_MC0); 1809 case FW_MEMTYPE_EXTMEM1: 1810 return (MEM_MC1); 1811 default: 1812 panic("%s: cannot translate fw mtype %d.", __func__, mtype); 1813 } 1814 } 1815 1816 /* 1817 * Verify that the memory range specified by the memtype/offset/len pair is 1818 * valid and lies entirely within the memtype specified. The global address of 1819 * the start of the range is returned in addr. 1820 */ 1821 static int 1822 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 1823 uint32_t *addr) 1824 { 1825 uint32_t em, addr_len, maddr, mlen; 1826 1827 /* Memory can only be accessed in naturally aligned 4 byte units */ 1828 if (off & 3 || len & 3 || len == 0) 1829 return (EINVAL); 1830 1831 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1832 switch (fwmtype_to_hwmtype(mtype)) { 1833 case MEM_EDC0: 1834 if (!(em & F_EDRAM0_ENABLE)) 1835 return (EINVAL); 1836 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1837 maddr = G_EDRAM0_BASE(addr_len) << 20; 1838 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1839 break; 1840 case MEM_EDC1: 1841 if (!(em & F_EDRAM1_ENABLE)) 1842 return (EINVAL); 1843 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1844 maddr = G_EDRAM1_BASE(addr_len) << 20; 1845 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1846 break; 1847 case MEM_MC: 1848 if (!(em & F_EXT_MEM_ENABLE)) 1849 return (EINVAL); 1850 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1851 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1852 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1853 break; 1854 case MEM_MC1: 1855 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 1856 return (EINVAL); 1857 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1858 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1859 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1860 break; 1861 default: 1862 return (EINVAL); 1863 } 1864 1865 if (mlen > 0 && off < mlen && off + len <= mlen) { 1866 *addr = maddr + off; /* global address */ 1867 return (0); 1868 } 1869 1870 return (EFAULT); 1871 } 1872 1873 static void 1874 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 1875 { 1876 const struct memwin *mw; 1877 1878 if (is_t4(sc)) { 1879 KASSERT(win >= 0 && win < nitems(t4_memwin), 1880 ("%s: incorrect memwin# (%d)", __func__, win)); 1881 mw = &t4_memwin[win]; 1882 } else { 1883 KASSERT(win >= 0 && win < nitems(t5_memwin), 1884 ("%s: incorrect memwin# (%d)", __func__, win)); 1885 mw = &t5_memwin[win]; 1886 } 1887 1888 if (base != NULL) 1889 *base = mw->base; 1890 if (aperture != NULL) 1891 *aperture = mw->aperture; 1892 } 1893 1894 /* 1895 * Positions the memory window such that it can be used to access the specified 1896 * address in the chip's address space. The return value is the offset of addr 1897 * from the start of the window. 1898 */ 1899 static uint32_t 1900 position_memwin(struct adapter *sc, int n, uint32_t addr) 1901 { 1902 uint32_t start, pf; 1903 uint32_t reg; 1904 1905 KASSERT(n >= 0 && n <= 3, 1906 ("%s: invalid window %d.", __func__, n)); 1907 KASSERT((addr & 3) == 0, 1908 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 1909 1910 if (is_t4(sc)) { 1911 pf = 0; 1912 start = addr & ~0xf; /* start must be 16B aligned */ 1913 } else { 1914 pf = V_PFNUM(sc->pf); 1915 start = addr & ~0x7f; /* start must be 128B aligned */ 1916 } 1917 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 1918 1919 t4_write_reg(sc, reg, start | pf); 1920 t4_read_reg(sc, reg); 1921 1922 return (addr - start); 1923 } 1924 1925 static int 1926 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1927 struct intrs_and_queues *iaq) 1928 { 1929 int rc, itype, navail, nrxq10g, nrxq1g, n; 1930 int nofldrxq10g = 0, nofldrxq1g = 0; 1931 int nnmrxq10g = 0, nnmrxq1g = 0; 1932 1933 bzero(iaq, sizeof(*iaq)); 1934 1935 iaq->ntxq10g = t4_ntxq10g; 1936 iaq->ntxq1g = t4_ntxq1g; 1937 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1938 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1939 iaq->rsrv_noflowq = t4_rsrv_noflowq; 1940 #ifdef TCP_OFFLOAD 1941 if (is_offload(sc)) { 1942 iaq->nofldtxq10g = t4_nofldtxq10g; 1943 iaq->nofldtxq1g = t4_nofldtxq1g; 1944 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1945 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1946 } 1947 #endif 1948 #ifdef DEV_NETMAP 1949 iaq->nnmtxq10g = t4_nnmtxq10g; 1950 iaq->nnmtxq1g = t4_nnmtxq1g; 1951 iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g; 1952 iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g; 1953 #endif 1954 1955 for (itype = INTR_MSIX; itype; itype >>= 1) { 1956 1957 if ((itype & t4_intr_types) == 0) 1958 continue; /* not allowed */ 1959 1960 if (itype == INTR_MSIX) 1961 navail = pci_msix_count(sc->dev); 1962 else if (itype == INTR_MSI) 1963 navail = pci_msi_count(sc->dev); 1964 else 1965 navail = 1; 1966 restart: 1967 if (navail == 0) 1968 continue; 1969 1970 iaq->intr_type = itype; 1971 iaq->intr_flags_10g = 0; 1972 iaq->intr_flags_1g = 0; 1973 1974 /* 1975 * Best option: an interrupt vector for errors, one for the 1976 * firmware event queue, and one for every rxq (NIC, TOE, and 1977 * netmap). 1978 */ 1979 iaq->nirq = T4_EXTRA_INTR; 1980 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g); 1981 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g); 1982 if (iaq->nirq <= navail && 1983 (itype != INTR_MSI || powerof2(iaq->nirq))) { 1984 iaq->intr_flags_10g = INTR_ALL; 1985 iaq->intr_flags_1g = INTR_ALL; 1986 goto allocate; 1987 } 1988 1989 /* 1990 * Second best option: a vector for errors, one for the firmware 1991 * event queue, and vectors for either all the NIC rx queues or 1992 * all the TOE rx queues. The queues that don't get vectors 1993 * will forward their interrupts to those that do. 1994 * 1995 * Note: netmap rx queues cannot be created early and so they 1996 * can't be setup to receive forwarded interrupts for others. 1997 */ 1998 iaq->nirq = T4_EXTRA_INTR; 1999 if (nrxq10g >= nofldrxq10g) { 2000 iaq->intr_flags_10g = INTR_RXQ; 2001 iaq->nirq += n10g * nrxq10g; 2002 #ifdef DEV_NETMAP 2003 iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g); 2004 #endif 2005 } else { 2006 iaq->intr_flags_10g = INTR_OFLD_RXQ; 2007 iaq->nirq += n10g * nofldrxq10g; 2008 #ifdef DEV_NETMAP 2009 iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g); 2010 #endif 2011 } 2012 if (nrxq1g >= nofldrxq1g) { 2013 iaq->intr_flags_1g = INTR_RXQ; 2014 iaq->nirq += n1g * nrxq1g; 2015 #ifdef DEV_NETMAP 2016 iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g); 2017 #endif 2018 } else { 2019 iaq->intr_flags_1g = INTR_OFLD_RXQ; 2020 iaq->nirq += n1g * nofldrxq1g; 2021 #ifdef DEV_NETMAP 2022 iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g); 2023 #endif 2024 } 2025 if (iaq->nirq <= navail && 2026 (itype != INTR_MSI || powerof2(iaq->nirq))) 2027 goto allocate; 2028 2029 /* 2030 * Next best option: an interrupt vector for errors, one for the 2031 * firmware event queue, and at least one per port. At this 2032 * point we know we'll have to downsize nrxq and/or nofldrxq 2033 * and/or nnmrxq to fit what's available to us. 2034 */ 2035 iaq->nirq = T4_EXTRA_INTR; 2036 iaq->nirq += n10g + n1g; 2037 if (iaq->nirq <= navail) { 2038 int leftover = navail - iaq->nirq; 2039 2040 if (n10g > 0) { 2041 int target = max(nrxq10g, nofldrxq10g); 2042 2043 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ? 2044 INTR_RXQ : INTR_OFLD_RXQ; 2045 2046 n = 1; 2047 while (n < target && leftover >= n10g) { 2048 leftover -= n10g; 2049 iaq->nirq += n10g; 2050 n++; 2051 } 2052 iaq->nrxq10g = min(n, nrxq10g); 2053 #ifdef TCP_OFFLOAD 2054 iaq->nofldrxq10g = min(n, nofldrxq10g); 2055 #endif 2056 #ifdef DEV_NETMAP 2057 iaq->nnmrxq10g = min(n, nnmrxq10g); 2058 #endif 2059 } 2060 2061 if (n1g > 0) { 2062 int target = max(nrxq1g, nofldrxq1g); 2063 2064 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ? 2065 INTR_RXQ : INTR_OFLD_RXQ; 2066 2067 n = 1; 2068 while (n < target && leftover >= n1g) { 2069 leftover -= n1g; 2070 iaq->nirq += n1g; 2071 n++; 2072 } 2073 iaq->nrxq1g = min(n, nrxq1g); 2074 #ifdef TCP_OFFLOAD 2075 iaq->nofldrxq1g = min(n, nofldrxq1g); 2076 #endif 2077 #ifdef DEV_NETMAP 2078 iaq->nnmrxq1g = min(n, nnmrxq1g); 2079 #endif 2080 } 2081 2082 if (itype != INTR_MSI || powerof2(iaq->nirq)) 2083 goto allocate; 2084 } 2085 2086 /* 2087 * Least desirable option: one interrupt vector for everything. 2088 */ 2089 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 2090 iaq->intr_flags_10g = iaq->intr_flags_1g = 0; 2091 #ifdef TCP_OFFLOAD 2092 if (is_offload(sc)) 2093 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 2094 #endif 2095 #ifdef DEV_NETMAP 2096 iaq->nnmrxq10g = iaq->nnmrxq1g = 1; 2097 #endif 2098 2099 allocate: 2100 navail = iaq->nirq; 2101 rc = 0; 2102 if (itype == INTR_MSIX) 2103 rc = pci_alloc_msix(sc->dev, &navail); 2104 else if (itype == INTR_MSI) 2105 rc = pci_alloc_msi(sc->dev, &navail); 2106 2107 if (rc == 0) { 2108 if (navail == iaq->nirq) 2109 return (0); 2110 2111 /* 2112 * Didn't get the number requested. Use whatever number 2113 * the kernel is willing to allocate (it's in navail). 2114 */ 2115 device_printf(sc->dev, "fewer vectors than requested, " 2116 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 2117 itype, iaq->nirq, navail); 2118 pci_release_msi(sc->dev); 2119 goto restart; 2120 } 2121 2122 device_printf(sc->dev, 2123 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 2124 itype, rc, iaq->nirq, navail); 2125 } 2126 2127 device_printf(sc->dev, 2128 "failed to find a usable interrupt type. " 2129 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 2130 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 2131 2132 return (ENXIO); 2133 } 2134 2135 #define FW_VERSION(chip) ( \ 2136 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ 2137 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ 2138 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ 2139 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) 2140 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) 2141 2142 struct fw_info { 2143 uint8_t chip; 2144 char *kld_name; 2145 char *fw_mod_name; 2146 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 2147 } fw_info[] = { 2148 { 2149 .chip = CHELSIO_T4, 2150 .kld_name = "t4fw_cfg", 2151 .fw_mod_name = "t4fw", 2152 .fw_hdr = { 2153 .chip = FW_HDR_CHIP_T4, 2154 .fw_ver = htobe32_const(FW_VERSION(T4)), 2155 .intfver_nic = FW_INTFVER(T4, NIC), 2156 .intfver_vnic = FW_INTFVER(T4, VNIC), 2157 .intfver_ofld = FW_INTFVER(T4, OFLD), 2158 .intfver_ri = FW_INTFVER(T4, RI), 2159 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 2160 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 2161 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 2162 .intfver_fcoe = FW_INTFVER(T4, FCOE), 2163 }, 2164 }, { 2165 .chip = CHELSIO_T5, 2166 .kld_name = "t5fw_cfg", 2167 .fw_mod_name = "t5fw", 2168 .fw_hdr = { 2169 .chip = FW_HDR_CHIP_T5, 2170 .fw_ver = htobe32_const(FW_VERSION(T5)), 2171 .intfver_nic = FW_INTFVER(T5, NIC), 2172 .intfver_vnic = FW_INTFVER(T5, VNIC), 2173 .intfver_ofld = FW_INTFVER(T5, OFLD), 2174 .intfver_ri = FW_INTFVER(T5, RI), 2175 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 2176 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 2177 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 2178 .intfver_fcoe = FW_INTFVER(T5, FCOE), 2179 }, 2180 } 2181 }; 2182 2183 static struct fw_info * 2184 find_fw_info(int chip) 2185 { 2186 int i; 2187 2188 for (i = 0; i < nitems(fw_info); i++) { 2189 if (fw_info[i].chip == chip) 2190 return (&fw_info[i]); 2191 } 2192 return (NULL); 2193 } 2194 2195 /* 2196 * Is the given firmware API compatible with the one the driver was compiled 2197 * with? 2198 */ 2199 static int 2200 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 2201 { 2202 2203 /* short circuit if it's the exact same firmware version */ 2204 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 2205 return (1); 2206 2207 /* 2208 * XXX: Is this too conservative? Perhaps I should limit this to the 2209 * features that are supported in the driver. 2210 */ 2211 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 2212 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 2213 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 2214 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 2215 return (1); 2216 #undef SAME_INTF 2217 2218 return (0); 2219 } 2220 2221 /* 2222 * The firmware in the KLD is usable, but should it be installed? This routine 2223 * explains itself in detail if it indicates the KLD firmware should be 2224 * installed. 2225 */ 2226 static int 2227 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 2228 { 2229 const char *reason; 2230 2231 if (!card_fw_usable) { 2232 reason = "incompatible or unusable"; 2233 goto install; 2234 } 2235 2236 if (k > c) { 2237 reason = "older than the version bundled with this driver"; 2238 goto install; 2239 } 2240 2241 if (t4_fw_install == 2 && k != c) { 2242 reason = "different than the version bundled with this driver"; 2243 goto install; 2244 } 2245 2246 return (0); 2247 2248 install: 2249 if (t4_fw_install == 0) { 2250 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2251 "but the driver is prohibited from installing a different " 2252 "firmware on the card.\n", 2253 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2254 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 2255 2256 return (0); 2257 } 2258 2259 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2260 "installing firmware %u.%u.%u.%u on card.\n", 2261 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2262 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 2263 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2264 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2265 2266 return (1); 2267 } 2268 /* 2269 * Establish contact with the firmware and determine if we are the master driver 2270 * or not, and whether we are responsible for chip initialization. 2271 */ 2272 static int 2273 prep_firmware(struct adapter *sc) 2274 { 2275 const struct firmware *fw = NULL, *default_cfg; 2276 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 2277 enum dev_state state; 2278 struct fw_info *fw_info; 2279 struct fw_hdr *card_fw; /* fw on the card */ 2280 const struct fw_hdr *kld_fw; /* fw in the KLD */ 2281 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 2282 against */ 2283 2284 /* Contact firmware. */ 2285 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 2286 if (rc < 0 || state == DEV_STATE_ERR) { 2287 rc = -rc; 2288 device_printf(sc->dev, 2289 "failed to connect to the firmware: %d, %d.\n", rc, state); 2290 return (rc); 2291 } 2292 pf = rc; 2293 if (pf == sc->mbox) 2294 sc->flags |= MASTER_PF; 2295 else if (state == DEV_STATE_UNINIT) { 2296 /* 2297 * We didn't get to be the master so we definitely won't be 2298 * configuring the chip. It's a bug if someone else hasn't 2299 * configured it already. 2300 */ 2301 device_printf(sc->dev, "couldn't be master(%d), " 2302 "device not already initialized either(%d).\n", rc, state); 2303 return (EDOOFUS); 2304 } 2305 2306 /* This is the firmware whose headers the driver was compiled against */ 2307 fw_info = find_fw_info(chip_id(sc)); 2308 if (fw_info == NULL) { 2309 device_printf(sc->dev, 2310 "unable to look up firmware information for chip %d.\n", 2311 chip_id(sc)); 2312 return (EINVAL); 2313 } 2314 drv_fw = &fw_info->fw_hdr; 2315 2316 /* 2317 * The firmware KLD contains many modules. The KLD name is also the 2318 * name of the module that contains the default config file. 2319 */ 2320 default_cfg = firmware_get(fw_info->kld_name); 2321 2322 /* Read the header of the firmware on the card */ 2323 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 2324 rc = -t4_read_flash(sc, FLASH_FW_START, 2325 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 2326 if (rc == 0) 2327 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 2328 else { 2329 device_printf(sc->dev, 2330 "Unable to read card's firmware header: %d\n", rc); 2331 card_fw_usable = 0; 2332 } 2333 2334 /* This is the firmware in the KLD */ 2335 fw = firmware_get(fw_info->fw_mod_name); 2336 if (fw != NULL) { 2337 kld_fw = (const void *)fw->data; 2338 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 2339 } else { 2340 kld_fw = NULL; 2341 kld_fw_usable = 0; 2342 } 2343 2344 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2345 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) { 2346 /* 2347 * Common case: the firmware on the card is an exact match and 2348 * the KLD is an exact match too, or the KLD is 2349 * absent/incompatible. Note that t4_fw_install = 2 is ignored 2350 * here -- use cxgbetool loadfw if you want to reinstall the 2351 * same firmware as the one on the card. 2352 */ 2353 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2354 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2355 be32toh(card_fw->fw_ver))) { 2356 2357 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 2358 if (rc != 0) { 2359 device_printf(sc->dev, 2360 "failed to install firmware: %d\n", rc); 2361 goto done; 2362 } 2363 2364 /* Installed successfully, update the cached header too. */ 2365 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2366 card_fw_usable = 1; 2367 need_fw_reset = 0; /* already reset as part of load_fw */ 2368 } 2369 2370 if (!card_fw_usable) { 2371 uint32_t d, c, k; 2372 2373 d = ntohl(drv_fw->fw_ver); 2374 c = ntohl(card_fw->fw_ver); 2375 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2376 2377 device_printf(sc->dev, "Cannot find a usable firmware: " 2378 "fw_install %d, chip state %d, " 2379 "driver compiled with %d.%d.%d.%d, " 2380 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2381 t4_fw_install, state, 2382 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2383 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2384 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2385 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2386 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2387 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2388 rc = EINVAL; 2389 goto done; 2390 } 2391 2392 /* We're using whatever's on the card and it's known to be good. */ 2393 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2394 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2395 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2396 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2397 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2398 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2399 t4_get_tp_version(sc, &sc->params.tp_vers); 2400 2401 /* Reset device */ 2402 if (need_fw_reset && 2403 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2404 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2405 if (rc != ETIMEDOUT && rc != EIO) 2406 t4_fw_bye(sc, sc->mbox); 2407 goto done; 2408 } 2409 sc->flags |= FW_OK; 2410 2411 rc = get_params__pre_init(sc); 2412 if (rc != 0) 2413 goto done; /* error message displayed already */ 2414 2415 /* Partition adapter resources as specified in the config file. */ 2416 if (state == DEV_STATE_UNINIT) { 2417 2418 KASSERT(sc->flags & MASTER_PF, 2419 ("%s: trying to change chip settings when not master.", 2420 __func__)); 2421 2422 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2423 if (rc != 0) 2424 goto done; /* error message displayed already */ 2425 2426 t4_tweak_chip_settings(sc); 2427 2428 /* get basic stuff going */ 2429 rc = -t4_fw_initialize(sc, sc->mbox); 2430 if (rc != 0) { 2431 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2432 goto done; 2433 } 2434 } else { 2435 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2436 sc->cfcsum = 0; 2437 } 2438 2439 done: 2440 free(card_fw, M_CXGBE); 2441 if (fw != NULL) 2442 firmware_put(fw, FIRMWARE_UNLOAD); 2443 if (default_cfg != NULL) 2444 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2445 2446 return (rc); 2447 } 2448 2449 #define FW_PARAM_DEV(param) \ 2450 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2451 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2452 #define FW_PARAM_PFVF(param) \ 2453 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2454 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2455 2456 /* 2457 * Partition chip resources for use between various PFs, VFs, etc. 2458 */ 2459 static int 2460 partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2461 const char *name_prefix) 2462 { 2463 const struct firmware *cfg = NULL; 2464 int rc = 0; 2465 struct fw_caps_config_cmd caps; 2466 uint32_t mtype, moff, finicsum, cfcsum; 2467 2468 /* 2469 * Figure out what configuration file to use. Pick the default config 2470 * file for the card if the user hasn't specified one explicitly. 2471 */ 2472 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2473 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2474 /* Card specific overrides go here. */ 2475 if (pci_get_device(sc->dev) == 0x440a) 2476 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2477 if (is_fpga(sc)) 2478 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2479 } 2480 2481 /* 2482 * We need to load another module if the profile is anything except 2483 * "default" or "flash". 2484 */ 2485 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2486 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2487 char s[32]; 2488 2489 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2490 cfg = firmware_get(s); 2491 if (cfg == NULL) { 2492 if (default_cfg != NULL) { 2493 device_printf(sc->dev, 2494 "unable to load module \"%s\" for " 2495 "configuration profile \"%s\", will use " 2496 "the default config file instead.\n", 2497 s, sc->cfg_file); 2498 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2499 "%s", DEFAULT_CF); 2500 } else { 2501 device_printf(sc->dev, 2502 "unable to load module \"%s\" for " 2503 "configuration profile \"%s\", will use " 2504 "the config file on the card's flash " 2505 "instead.\n", s, sc->cfg_file); 2506 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2507 "%s", FLASH_CF); 2508 } 2509 } 2510 } 2511 2512 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2513 default_cfg == NULL) { 2514 device_printf(sc->dev, 2515 "default config file not available, will use the config " 2516 "file on the card's flash instead.\n"); 2517 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2518 } 2519 2520 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2521 u_int cflen, i, n; 2522 const uint32_t *cfdata; 2523 uint32_t param, val, addr, off, mw_base, mw_aperture; 2524 2525 KASSERT(cfg != NULL || default_cfg != NULL, 2526 ("%s: no config to upload", __func__)); 2527 2528 /* 2529 * Ask the firmware where it wants us to upload the config file. 2530 */ 2531 param = FW_PARAM_DEV(CF); 2532 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2533 if (rc != 0) { 2534 /* No support for config file? Shouldn't happen. */ 2535 device_printf(sc->dev, 2536 "failed to query config file location: %d.\n", rc); 2537 goto done; 2538 } 2539 mtype = G_FW_PARAMS_PARAM_Y(val); 2540 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2541 2542 /* 2543 * XXX: sheer laziness. We deliberately added 4 bytes of 2544 * useless stuffing/comments at the end of the config file so 2545 * it's ok to simply throw away the last remaining bytes when 2546 * the config file is not an exact multiple of 4. This also 2547 * helps with the validate_mt_off_len check. 2548 */ 2549 if (cfg != NULL) { 2550 cflen = cfg->datasize & ~3; 2551 cfdata = cfg->data; 2552 } else { 2553 cflen = default_cfg->datasize & ~3; 2554 cfdata = default_cfg->data; 2555 } 2556 2557 if (cflen > FLASH_CFG_MAX_SIZE) { 2558 device_printf(sc->dev, 2559 "config file too long (%d, max allowed is %d). " 2560 "Will try to use the config on the card, if any.\n", 2561 cflen, FLASH_CFG_MAX_SIZE); 2562 goto use_config_on_flash; 2563 } 2564 2565 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2566 if (rc != 0) { 2567 device_printf(sc->dev, 2568 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2569 "Will try to use the config on the card, if any.\n", 2570 __func__, mtype, moff, cflen, rc); 2571 goto use_config_on_flash; 2572 } 2573 2574 memwin_info(sc, 2, &mw_base, &mw_aperture); 2575 while (cflen) { 2576 off = position_memwin(sc, 2, addr); 2577 n = min(cflen, mw_aperture - off); 2578 for (i = 0; i < n; i += 4) 2579 t4_write_reg(sc, mw_base + off + i, *cfdata++); 2580 cflen -= n; 2581 addr += n; 2582 } 2583 } else { 2584 use_config_on_flash: 2585 mtype = FW_MEMTYPE_FLASH; 2586 moff = t4_flash_cfg_addr(sc); 2587 } 2588 2589 bzero(&caps, sizeof(caps)); 2590 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2591 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2592 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2593 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2594 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2595 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2596 if (rc != 0) { 2597 device_printf(sc->dev, 2598 "failed to pre-process config file: %d " 2599 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2600 goto done; 2601 } 2602 2603 finicsum = be32toh(caps.finicsum); 2604 cfcsum = be32toh(caps.cfcsum); 2605 if (finicsum != cfcsum) { 2606 device_printf(sc->dev, 2607 "WARNING: config file checksum mismatch: %08x %08x\n", 2608 finicsum, cfcsum); 2609 } 2610 sc->cfcsum = cfcsum; 2611 2612 #define LIMIT_CAPS(x) do { \ 2613 caps.x &= htobe16(t4_##x##_allowed); \ 2614 } while (0) 2615 2616 /* 2617 * Let the firmware know what features will (not) be used so it can tune 2618 * things accordingly. 2619 */ 2620 LIMIT_CAPS(linkcaps); 2621 LIMIT_CAPS(niccaps); 2622 LIMIT_CAPS(toecaps); 2623 LIMIT_CAPS(rdmacaps); 2624 LIMIT_CAPS(iscsicaps); 2625 LIMIT_CAPS(fcoecaps); 2626 #undef LIMIT_CAPS 2627 2628 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2629 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2630 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2631 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2632 if (rc != 0) { 2633 device_printf(sc->dev, 2634 "failed to process config file: %d.\n", rc); 2635 } 2636 done: 2637 if (cfg != NULL) 2638 firmware_put(cfg, FIRMWARE_UNLOAD); 2639 return (rc); 2640 } 2641 2642 /* 2643 * Retrieve parameters that are needed (or nice to have) very early. 2644 */ 2645 static int 2646 get_params__pre_init(struct adapter *sc) 2647 { 2648 int rc; 2649 uint32_t param[2], val[2]; 2650 struct fw_devlog_cmd cmd; 2651 struct devlog_params *dlog = &sc->params.devlog; 2652 2653 param[0] = FW_PARAM_DEV(PORTVEC); 2654 param[1] = FW_PARAM_DEV(CCLK); 2655 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2656 if (rc != 0) { 2657 device_printf(sc->dev, 2658 "failed to query parameters (pre_init): %d.\n", rc); 2659 return (rc); 2660 } 2661 2662 sc->params.portvec = val[0]; 2663 sc->params.nports = bitcount32(val[0]); 2664 sc->params.vpd.cclk = val[1]; 2665 2666 /* Read device log parameters. */ 2667 bzero(&cmd, sizeof(cmd)); 2668 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2669 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2670 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2671 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2672 if (rc != 0) { 2673 device_printf(sc->dev, 2674 "failed to get devlog parameters: %d.\n", rc); 2675 bzero(dlog, sizeof (*dlog)); 2676 rc = 0; /* devlog isn't critical for device operation */ 2677 } else { 2678 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2679 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2680 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2681 dlog->size = be32toh(cmd.memsize_devlog); 2682 } 2683 2684 return (rc); 2685 } 2686 2687 /* 2688 * Retrieve various parameters that are of interest to the driver. The device 2689 * has been initialized by the firmware at this point. 2690 */ 2691 static int 2692 get_params__post_init(struct adapter *sc) 2693 { 2694 int rc; 2695 uint32_t param[7], val[7]; 2696 struct fw_caps_config_cmd caps; 2697 2698 param[0] = FW_PARAM_PFVF(IQFLINT_START); 2699 param[1] = FW_PARAM_PFVF(EQ_START); 2700 param[2] = FW_PARAM_PFVF(FILTER_START); 2701 param[3] = FW_PARAM_PFVF(FILTER_END); 2702 param[4] = FW_PARAM_PFVF(L2T_START); 2703 param[5] = FW_PARAM_PFVF(L2T_END); 2704 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2705 if (rc != 0) { 2706 device_printf(sc->dev, 2707 "failed to query parameters (post_init): %d.\n", rc); 2708 return (rc); 2709 } 2710 2711 sc->sge.iq_start = val[0]; 2712 sc->sge.eq_start = val[1]; 2713 sc->tids.ftid_base = val[2]; 2714 sc->tids.nftids = val[3] - val[2] + 1; 2715 sc->params.ftid_min = val[2]; 2716 sc->params.ftid_max = val[3]; 2717 sc->vres.l2t.start = val[4]; 2718 sc->vres.l2t.size = val[5] - val[4] + 1; 2719 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2720 ("%s: L2 table size (%u) larger than expected (%u)", 2721 __func__, sc->vres.l2t.size, L2T_SIZE)); 2722 2723 /* get capabilites */ 2724 bzero(&caps, sizeof(caps)); 2725 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2726 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2727 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2728 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2729 if (rc != 0) { 2730 device_printf(sc->dev, 2731 "failed to get card capabilities: %d.\n", rc); 2732 return (rc); 2733 } 2734 2735 #define READ_CAPS(x) do { \ 2736 sc->x = htobe16(caps.x); \ 2737 } while (0) 2738 READ_CAPS(linkcaps); 2739 READ_CAPS(niccaps); 2740 READ_CAPS(toecaps); 2741 READ_CAPS(rdmacaps); 2742 READ_CAPS(iscsicaps); 2743 READ_CAPS(fcoecaps); 2744 2745 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { 2746 param[0] = FW_PARAM_PFVF(ETHOFLD_START); 2747 param[1] = FW_PARAM_PFVF(ETHOFLD_END); 2748 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2749 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); 2750 if (rc != 0) { 2751 device_printf(sc->dev, 2752 "failed to query NIC parameters: %d.\n", rc); 2753 return (rc); 2754 } 2755 sc->tids.etid_base = val[0]; 2756 sc->params.etid_min = val[0]; 2757 sc->tids.netids = val[1] - val[0] + 1; 2758 sc->params.netids = sc->tids.netids; 2759 sc->params.eo_wr_cred = val[2]; 2760 sc->params.ethoffload = 1; 2761 } 2762 2763 if (sc->toecaps) { 2764 /* query offload-related parameters */ 2765 param[0] = FW_PARAM_DEV(NTID); 2766 param[1] = FW_PARAM_PFVF(SERVER_START); 2767 param[2] = FW_PARAM_PFVF(SERVER_END); 2768 param[3] = FW_PARAM_PFVF(TDDP_START); 2769 param[4] = FW_PARAM_PFVF(TDDP_END); 2770 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2771 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2772 if (rc != 0) { 2773 device_printf(sc->dev, 2774 "failed to query TOE parameters: %d.\n", rc); 2775 return (rc); 2776 } 2777 sc->tids.ntids = val[0]; 2778 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 2779 sc->tids.stid_base = val[1]; 2780 sc->tids.nstids = val[2] - val[1] + 1; 2781 sc->vres.ddp.start = val[3]; 2782 sc->vres.ddp.size = val[4] - val[3] + 1; 2783 sc->params.ofldq_wr_cred = val[5]; 2784 sc->params.offload = 1; 2785 } 2786 if (sc->rdmacaps) { 2787 param[0] = FW_PARAM_PFVF(STAG_START); 2788 param[1] = FW_PARAM_PFVF(STAG_END); 2789 param[2] = FW_PARAM_PFVF(RQ_START); 2790 param[3] = FW_PARAM_PFVF(RQ_END); 2791 param[4] = FW_PARAM_PFVF(PBL_START); 2792 param[5] = FW_PARAM_PFVF(PBL_END); 2793 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2794 if (rc != 0) { 2795 device_printf(sc->dev, 2796 "failed to query RDMA parameters(1): %d.\n", rc); 2797 return (rc); 2798 } 2799 sc->vres.stag.start = val[0]; 2800 sc->vres.stag.size = val[1] - val[0] + 1; 2801 sc->vres.rq.start = val[2]; 2802 sc->vres.rq.size = val[3] - val[2] + 1; 2803 sc->vres.pbl.start = val[4]; 2804 sc->vres.pbl.size = val[5] - val[4] + 1; 2805 2806 param[0] = FW_PARAM_PFVF(SQRQ_START); 2807 param[1] = FW_PARAM_PFVF(SQRQ_END); 2808 param[2] = FW_PARAM_PFVF(CQ_START); 2809 param[3] = FW_PARAM_PFVF(CQ_END); 2810 param[4] = FW_PARAM_PFVF(OCQ_START); 2811 param[5] = FW_PARAM_PFVF(OCQ_END); 2812 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2813 if (rc != 0) { 2814 device_printf(sc->dev, 2815 "failed to query RDMA parameters(2): %d.\n", rc); 2816 return (rc); 2817 } 2818 sc->vres.qp.start = val[0]; 2819 sc->vres.qp.size = val[1] - val[0] + 1; 2820 sc->vres.cq.start = val[2]; 2821 sc->vres.cq.size = val[3] - val[2] + 1; 2822 sc->vres.ocq.start = val[4]; 2823 sc->vres.ocq.size = val[5] - val[4] + 1; 2824 } 2825 if (sc->iscsicaps) { 2826 param[0] = FW_PARAM_PFVF(ISCSI_START); 2827 param[1] = FW_PARAM_PFVF(ISCSI_END); 2828 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2829 if (rc != 0) { 2830 device_printf(sc->dev, 2831 "failed to query iSCSI parameters: %d.\n", rc); 2832 return (rc); 2833 } 2834 sc->vres.iscsi.start = val[0]; 2835 sc->vres.iscsi.size = val[1] - val[0] + 1; 2836 } 2837 2838 /* 2839 * We've got the params we wanted to query via the firmware. Now grab 2840 * some others directly from the chip. 2841 */ 2842 rc = t4_read_chip_settings(sc); 2843 2844 return (rc); 2845 } 2846 2847 static int 2848 set_params__post_init(struct adapter *sc) 2849 { 2850 uint32_t param, val; 2851 2852 /* ask for encapsulated CPLs */ 2853 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 2854 val = 1; 2855 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2856 2857 return (0); 2858 } 2859 2860 #undef FW_PARAM_PFVF 2861 #undef FW_PARAM_DEV 2862 2863 static void 2864 t4_set_desc(struct adapter *sc) 2865 { 2866 char buf[128]; 2867 struct adapter_params *p = &sc->params; 2868 2869 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, " 2870 "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "", 2871 chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec); 2872 2873 device_set_desc_copy(sc->dev, buf); 2874 } 2875 2876 static void 2877 build_medialist(struct port_info *pi, struct ifmedia *media) 2878 { 2879 int data, m; 2880 2881 PORT_LOCK(pi); 2882 2883 ifmedia_removeall(media); 2884 2885 m = IFM_ETHER | IFM_FDX; 2886 data = (pi->port_type << 8) | pi->mod_type; 2887 2888 switch(pi->port_type) { 2889 case FW_PORT_TYPE_BT_XFI: 2890 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2891 break; 2892 2893 case FW_PORT_TYPE_BT_XAUI: 2894 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2895 /* fall through */ 2896 2897 case FW_PORT_TYPE_BT_SGMII: 2898 ifmedia_add(media, m | IFM_1000_T, data, NULL); 2899 ifmedia_add(media, m | IFM_100_TX, data, NULL); 2900 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL); 2901 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2902 break; 2903 2904 case FW_PORT_TYPE_CX4: 2905 ifmedia_add(media, m | IFM_10G_CX4, data, NULL); 2906 ifmedia_set(media, m | IFM_10G_CX4); 2907 break; 2908 2909 case FW_PORT_TYPE_QSFP_10G: 2910 case FW_PORT_TYPE_SFP: 2911 case FW_PORT_TYPE_FIBER_XFI: 2912 case FW_PORT_TYPE_FIBER_XAUI: 2913 switch (pi->mod_type) { 2914 2915 case FW_PORT_MOD_TYPE_LR: 2916 ifmedia_add(media, m | IFM_10G_LR, data, NULL); 2917 ifmedia_set(media, m | IFM_10G_LR); 2918 break; 2919 2920 case FW_PORT_MOD_TYPE_SR: 2921 ifmedia_add(media, m | IFM_10G_SR, data, NULL); 2922 ifmedia_set(media, m | IFM_10G_SR); 2923 break; 2924 2925 case FW_PORT_MOD_TYPE_LRM: 2926 ifmedia_add(media, m | IFM_10G_LRM, data, NULL); 2927 ifmedia_set(media, m | IFM_10G_LRM); 2928 break; 2929 2930 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2931 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2932 ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL); 2933 ifmedia_set(media, m | IFM_10G_TWINAX); 2934 break; 2935 2936 case FW_PORT_MOD_TYPE_NONE: 2937 m &= ~IFM_FDX; 2938 ifmedia_add(media, m | IFM_NONE, data, NULL); 2939 ifmedia_set(media, m | IFM_NONE); 2940 break; 2941 2942 case FW_PORT_MOD_TYPE_NA: 2943 case FW_PORT_MOD_TYPE_ER: 2944 default: 2945 device_printf(pi->dev, 2946 "unknown port_type (%d), mod_type (%d)\n", 2947 pi->port_type, pi->mod_type); 2948 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2949 ifmedia_set(media, m | IFM_UNKNOWN); 2950 break; 2951 } 2952 break; 2953 2954 case FW_PORT_TYPE_QSFP: 2955 switch (pi->mod_type) { 2956 2957 case FW_PORT_MOD_TYPE_LR: 2958 ifmedia_add(media, m | IFM_40G_LR4, data, NULL); 2959 ifmedia_set(media, m | IFM_40G_LR4); 2960 break; 2961 2962 case FW_PORT_MOD_TYPE_SR: 2963 ifmedia_add(media, m | IFM_40G_SR4, data, NULL); 2964 ifmedia_set(media, m | IFM_40G_SR4); 2965 break; 2966 2967 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2968 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2969 ifmedia_add(media, m | IFM_40G_CR4, data, NULL); 2970 ifmedia_set(media, m | IFM_40G_CR4); 2971 break; 2972 2973 case FW_PORT_MOD_TYPE_NONE: 2974 m &= ~IFM_FDX; 2975 ifmedia_add(media, m | IFM_NONE, data, NULL); 2976 ifmedia_set(media, m | IFM_NONE); 2977 break; 2978 2979 default: 2980 device_printf(pi->dev, 2981 "unknown port_type (%d), mod_type (%d)\n", 2982 pi->port_type, pi->mod_type); 2983 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2984 ifmedia_set(media, m | IFM_UNKNOWN); 2985 break; 2986 } 2987 break; 2988 2989 default: 2990 device_printf(pi->dev, 2991 "unknown port_type (%d), mod_type (%d)\n", pi->port_type, 2992 pi->mod_type); 2993 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2994 ifmedia_set(media, m | IFM_UNKNOWN); 2995 break; 2996 } 2997 2998 PORT_UNLOCK(pi); 2999 } 3000 3001 #define FW_MAC_EXACT_CHUNK 7 3002 3003 /* 3004 * Program the port's XGMAC based on parameters in ifnet. The caller also 3005 * indicates which parameters should be programmed (the rest are left alone). 3006 */ 3007 int 3008 update_mac_settings(struct ifnet *ifp, int flags) 3009 { 3010 int rc = 0; 3011 struct port_info *pi = ifp->if_softc; 3012 struct adapter *sc = pi->adapter; 3013 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 3014 uint16_t viid = 0xffff; 3015 int16_t *xact_addr_filt = NULL; 3016 3017 ASSERT_SYNCHRONIZED_OP(sc); 3018 KASSERT(flags, ("%s: not told what to update.", __func__)); 3019 3020 if (ifp == pi->ifp) { 3021 viid = pi->viid; 3022 xact_addr_filt = &pi->xact_addr_filt; 3023 } 3024 #ifdef DEV_NETMAP 3025 else if (ifp == pi->nm_ifp) { 3026 viid = pi->nm_viid; 3027 xact_addr_filt = &pi->nm_xact_addr_filt; 3028 } 3029 #endif 3030 if (flags & XGMAC_MTU) 3031 mtu = ifp->if_mtu; 3032 3033 if (flags & XGMAC_PROMISC) 3034 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 3035 3036 if (flags & XGMAC_ALLMULTI) 3037 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 3038 3039 if (flags & XGMAC_VLANEX) 3040 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 3041 3042 if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) { 3043 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti, 3044 1, vlanex, false); 3045 if (rc) { 3046 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, 3047 rc); 3048 return (rc); 3049 } 3050 } 3051 3052 if (flags & XGMAC_UCADDR) { 3053 uint8_t ucaddr[ETHER_ADDR_LEN]; 3054 3055 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 3056 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr, 3057 true, true); 3058 if (rc < 0) { 3059 rc = -rc; 3060 if_printf(ifp, "change_mac failed: %d\n", rc); 3061 return (rc); 3062 } else { 3063 *xact_addr_filt = rc; 3064 rc = 0; 3065 } 3066 } 3067 3068 if (flags & XGMAC_MCADDRS) { 3069 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 3070 int del = 1; 3071 uint64_t hash = 0; 3072 struct ifmultiaddr *ifma; 3073 int i = 0, j; 3074 3075 if_maddr_rlock(ifp); 3076 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3077 if (ifma->ifma_addr->sa_family != AF_LINK) 3078 continue; 3079 mcaddr[i] = 3080 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 3081 MPASS(ETHER_IS_MULTICAST(mcaddr[i])); 3082 i++; 3083 3084 if (i == FW_MAC_EXACT_CHUNK) { 3085 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, 3086 i, mcaddr, NULL, &hash, 0); 3087 if (rc < 0) { 3088 rc = -rc; 3089 for (j = 0; j < i; j++) { 3090 if_printf(ifp, 3091 "failed to add mc address" 3092 " %02x:%02x:%02x:" 3093 "%02x:%02x:%02x rc=%d\n", 3094 mcaddr[j][0], mcaddr[j][1], 3095 mcaddr[j][2], mcaddr[j][3], 3096 mcaddr[j][4], mcaddr[j][5], 3097 rc); 3098 } 3099 goto mcfail; 3100 } 3101 del = 0; 3102 i = 0; 3103 } 3104 } 3105 if (i > 0) { 3106 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i, 3107 mcaddr, NULL, &hash, 0); 3108 if (rc < 0) { 3109 rc = -rc; 3110 for (j = 0; j < i; j++) { 3111 if_printf(ifp, 3112 "failed to add mc address" 3113 " %02x:%02x:%02x:" 3114 "%02x:%02x:%02x rc=%d\n", 3115 mcaddr[j][0], mcaddr[j][1], 3116 mcaddr[j][2], mcaddr[j][3], 3117 mcaddr[j][4], mcaddr[j][5], 3118 rc); 3119 } 3120 goto mcfail; 3121 } 3122 } 3123 3124 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0); 3125 if (rc != 0) 3126 if_printf(ifp, "failed to set mc address hash: %d", rc); 3127 mcfail: 3128 if_maddr_runlock(ifp); 3129 } 3130 3131 return (rc); 3132 } 3133 3134 int 3135 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags, 3136 char *wmesg) 3137 { 3138 int rc, pri; 3139 3140 #ifdef WITNESS 3141 /* the caller thinks it's ok to sleep, but is it really? */ 3142 if (flags & SLEEP_OK) 3143 pause("t4slptst", 1); 3144 #endif 3145 3146 if (INTR_OK) 3147 pri = PCATCH; 3148 else 3149 pri = 0; 3150 3151 ADAPTER_LOCK(sc); 3152 for (;;) { 3153 3154 if (pi && IS_DOOMED(pi)) { 3155 rc = ENXIO; 3156 goto done; 3157 } 3158 3159 if (!IS_BUSY(sc)) { 3160 rc = 0; 3161 break; 3162 } 3163 3164 if (!(flags & SLEEP_OK)) { 3165 rc = EBUSY; 3166 goto done; 3167 } 3168 3169 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 3170 rc = EINTR; 3171 goto done; 3172 } 3173 } 3174 3175 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 3176 SET_BUSY(sc); 3177 #ifdef INVARIANTS 3178 sc->last_op = wmesg; 3179 sc->last_op_thr = curthread; 3180 #endif 3181 3182 done: 3183 if (!(flags & HOLD_LOCK) || rc) 3184 ADAPTER_UNLOCK(sc); 3185 3186 return (rc); 3187 } 3188 3189 void 3190 end_synchronized_op(struct adapter *sc, int flags) 3191 { 3192 3193 if (flags & LOCK_HELD) 3194 ADAPTER_LOCK_ASSERT_OWNED(sc); 3195 else 3196 ADAPTER_LOCK(sc); 3197 3198 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 3199 CLR_BUSY(sc); 3200 wakeup(&sc->flags); 3201 ADAPTER_UNLOCK(sc); 3202 } 3203 3204 static int 3205 cxgbe_init_synchronized(struct port_info *pi) 3206 { 3207 struct adapter *sc = pi->adapter; 3208 struct ifnet *ifp = pi->ifp; 3209 int rc = 0, i; 3210 struct sge_txq *txq; 3211 3212 ASSERT_SYNCHRONIZED_OP(sc); 3213 3214 if (isset(&sc->open_device_map, pi->port_id)) { 3215 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 3216 ("mismatch between open_device_map and if_drv_flags")); 3217 return (0); /* already running */ 3218 } 3219 3220 if (!(sc->flags & FULL_INIT_DONE) && 3221 ((rc = adapter_full_init(sc)) != 0)) 3222 return (rc); /* error message displayed already */ 3223 3224 if (!(pi->flags & PORT_INIT_DONE) && 3225 ((rc = port_full_init(pi)) != 0)) 3226 return (rc); /* error message displayed already */ 3227 3228 rc = update_mac_settings(ifp, XGMAC_ALL); 3229 if (rc) 3230 goto done; /* error message displayed already */ 3231 3232 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 3233 if (rc != 0) { 3234 if_printf(ifp, "enable_vi failed: %d\n", rc); 3235 goto done; 3236 } 3237 3238 /* 3239 * Can't fail from this point onwards. Review cxgbe_uninit_synchronized 3240 * if this changes. 3241 */ 3242 3243 for_each_txq(pi, i, txq) { 3244 TXQ_LOCK(txq); 3245 txq->eq.flags |= EQ_ENABLED; 3246 TXQ_UNLOCK(txq); 3247 } 3248 3249 /* 3250 * The first iq of the first port to come up is used for tracing. 3251 */ 3252 if (sc->traceq < 0) { 3253 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id; 3254 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : 3255 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | 3256 V_QUEUENUMBER(sc->traceq)); 3257 pi->flags |= HAS_TRACEQ; 3258 } 3259 3260 /* all ok */ 3261 setbit(&sc->open_device_map, pi->port_id); 3262 PORT_LOCK(pi); 3263 ifp->if_drv_flags |= IFF_DRV_RUNNING; 3264 PORT_UNLOCK(pi); 3265 3266 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 3267 done: 3268 if (rc != 0) 3269 cxgbe_uninit_synchronized(pi); 3270 3271 return (rc); 3272 } 3273 3274 /* 3275 * Idempotent. 3276 */ 3277 static int 3278 cxgbe_uninit_synchronized(struct port_info *pi) 3279 { 3280 struct adapter *sc = pi->adapter; 3281 struct ifnet *ifp = pi->ifp; 3282 int rc, i; 3283 struct sge_txq *txq; 3284 3285 ASSERT_SYNCHRONIZED_OP(sc); 3286 3287 /* 3288 * Disable the VI so that all its data in either direction is discarded 3289 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 3290 * tick) intact as the TP can deliver negative advice or data that it's 3291 * holding in its RAM (for an offloaded connection) even after the VI is 3292 * disabled. 3293 */ 3294 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 3295 if (rc) { 3296 if_printf(ifp, "disable_vi failed: %d\n", rc); 3297 return (rc); 3298 } 3299 3300 for_each_txq(pi, i, txq) { 3301 TXQ_LOCK(txq); 3302 txq->eq.flags &= ~EQ_ENABLED; 3303 TXQ_UNLOCK(txq); 3304 } 3305 3306 clrbit(&sc->open_device_map, pi->port_id); 3307 PORT_LOCK(pi); 3308 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3309 PORT_UNLOCK(pi); 3310 3311 pi->link_cfg.link_ok = 0; 3312 pi->link_cfg.speed = 0; 3313 pi->linkdnrc = -1; 3314 t4_os_link_changed(sc, pi->port_id, 0, -1); 3315 3316 return (0); 3317 } 3318 3319 /* 3320 * It is ok for this function to fail midway and return right away. t4_detach 3321 * will walk the entire sc->irq list and clean up whatever is valid. 3322 */ 3323 static int 3324 setup_intr_handlers(struct adapter *sc) 3325 { 3326 int rc, rid, p, q; 3327 char s[8]; 3328 struct irq *irq; 3329 struct port_info *pi; 3330 struct sge_rxq *rxq; 3331 #ifdef TCP_OFFLOAD 3332 struct sge_ofld_rxq *ofld_rxq; 3333 #endif 3334 #ifdef DEV_NETMAP 3335 struct sge_nm_rxq *nm_rxq; 3336 #endif 3337 3338 /* 3339 * Setup interrupts. 3340 */ 3341 irq = &sc->irq[0]; 3342 rid = sc->intr_type == INTR_INTX ? 0 : 1; 3343 if (sc->intr_count == 1) 3344 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all")); 3345 3346 /* Multiple interrupts. */ 3347 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 3348 ("%s: too few intr.", __func__)); 3349 3350 /* The first one is always error intr */ 3351 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 3352 if (rc != 0) 3353 return (rc); 3354 irq++; 3355 rid++; 3356 3357 /* The second one is always the firmware event queue */ 3358 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt"); 3359 if (rc != 0) 3360 return (rc); 3361 irq++; 3362 rid++; 3363 3364 for_each_port(sc, p) { 3365 pi = sc->port[p]; 3366 3367 if (pi->flags & INTR_RXQ) { 3368 for_each_rxq(pi, q, rxq) { 3369 snprintf(s, sizeof(s), "%d.%d", p, q); 3370 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq, 3371 s); 3372 if (rc != 0) 3373 return (rc); 3374 irq++; 3375 rid++; 3376 } 3377 } 3378 #ifdef TCP_OFFLOAD 3379 if (pi->flags & INTR_OFLD_RXQ) { 3380 for_each_ofld_rxq(pi, q, ofld_rxq) { 3381 snprintf(s, sizeof(s), "%d,%d", p, q); 3382 rc = t4_alloc_irq(sc, irq, rid, t4_intr, 3383 ofld_rxq, s); 3384 if (rc != 0) 3385 return (rc); 3386 irq++; 3387 rid++; 3388 } 3389 } 3390 #endif 3391 #ifdef DEV_NETMAP 3392 if (pi->flags & INTR_NM_RXQ) { 3393 for_each_nm_rxq(pi, q, nm_rxq) { 3394 snprintf(s, sizeof(s), "%d-%d", p, q); 3395 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr, 3396 nm_rxq, s); 3397 if (rc != 0) 3398 return (rc); 3399 irq++; 3400 rid++; 3401 } 3402 } 3403 #endif 3404 } 3405 MPASS(irq == &sc->irq[sc->intr_count]); 3406 3407 return (0); 3408 } 3409 3410 int 3411 adapter_full_init(struct adapter *sc) 3412 { 3413 int rc, i; 3414 3415 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3416 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 3417 ("%s: FULL_INIT_DONE already", __func__)); 3418 3419 /* 3420 * queues that belong to the adapter (not any particular port). 3421 */ 3422 rc = t4_setup_adapter_queues(sc); 3423 if (rc != 0) 3424 goto done; 3425 3426 for (i = 0; i < nitems(sc->tq); i++) { 3427 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 3428 taskqueue_thread_enqueue, &sc->tq[i]); 3429 if (sc->tq[i] == NULL) { 3430 device_printf(sc->dev, 3431 "failed to allocate task queue %d\n", i); 3432 rc = ENOMEM; 3433 goto done; 3434 } 3435 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3436 device_get_nameunit(sc->dev), i); 3437 } 3438 3439 t4_intr_enable(sc); 3440 sc->flags |= FULL_INIT_DONE; 3441 done: 3442 if (rc != 0) 3443 adapter_full_uninit(sc); 3444 3445 return (rc); 3446 } 3447 3448 int 3449 adapter_full_uninit(struct adapter *sc) 3450 { 3451 int i; 3452 3453 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3454 3455 t4_teardown_adapter_queues(sc); 3456 3457 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3458 taskqueue_free(sc->tq[i]); 3459 sc->tq[i] = NULL; 3460 } 3461 3462 sc->flags &= ~FULL_INIT_DONE; 3463 3464 return (0); 3465 } 3466 3467 int 3468 port_full_init(struct port_info *pi) 3469 { 3470 struct adapter *sc = pi->adapter; 3471 struct ifnet *ifp = pi->ifp; 3472 uint16_t *rss; 3473 struct sge_rxq *rxq; 3474 int rc, i, j; 3475 3476 ASSERT_SYNCHRONIZED_OP(sc); 3477 KASSERT((pi->flags & PORT_INIT_DONE) == 0, 3478 ("%s: PORT_INIT_DONE already", __func__)); 3479 3480 sysctl_ctx_init(&pi->ctx); 3481 pi->flags |= PORT_SYSCTL_CTX; 3482 3483 /* 3484 * Allocate tx/rx/fl queues for this port. 3485 */ 3486 rc = t4_setup_port_queues(pi); 3487 if (rc != 0) 3488 goto done; /* error message displayed already */ 3489 3490 /* 3491 * Setup RSS for this port. Save a copy of the RSS table for later use. 3492 */ 3493 rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); 3494 for (i = 0; i < pi->rss_size;) { 3495 for_each_rxq(pi, j, rxq) { 3496 rss[i++] = rxq->iq.abs_id; 3497 if (i == pi->rss_size) 3498 break; 3499 } 3500 } 3501 3502 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss, 3503 pi->rss_size); 3504 if (rc != 0) { 3505 if_printf(ifp, "rss_config failed: %d\n", rc); 3506 goto done; 3507 } 3508 3509 pi->rss = rss; 3510 pi->flags |= PORT_INIT_DONE; 3511 done: 3512 if (rc != 0) 3513 port_full_uninit(pi); 3514 3515 return (rc); 3516 } 3517 3518 /* 3519 * Idempotent. 3520 */ 3521 int 3522 port_full_uninit(struct port_info *pi) 3523 { 3524 struct adapter *sc = pi->adapter; 3525 int i; 3526 struct sge_rxq *rxq; 3527 struct sge_txq *txq; 3528 #ifdef TCP_OFFLOAD 3529 struct sge_ofld_rxq *ofld_rxq; 3530 struct sge_wrq *ofld_txq; 3531 #endif 3532 3533 if (pi->flags & PORT_INIT_DONE) { 3534 3535 /* Need to quiesce queues. */ 3536 3537 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]); 3538 3539 for_each_txq(pi, i, txq) { 3540 quiesce_txq(sc, txq); 3541 } 3542 3543 #ifdef TCP_OFFLOAD 3544 for_each_ofld_txq(pi, i, ofld_txq) { 3545 quiesce_wrq(sc, ofld_txq); 3546 } 3547 #endif 3548 3549 for_each_rxq(pi, i, rxq) { 3550 quiesce_iq(sc, &rxq->iq); 3551 quiesce_fl(sc, &rxq->fl); 3552 } 3553 3554 #ifdef TCP_OFFLOAD 3555 for_each_ofld_rxq(pi, i, ofld_rxq) { 3556 quiesce_iq(sc, &ofld_rxq->iq); 3557 quiesce_fl(sc, &ofld_rxq->fl); 3558 } 3559 #endif 3560 free(pi->rss, M_CXGBE); 3561 } 3562 3563 t4_teardown_port_queues(pi); 3564 pi->flags &= ~PORT_INIT_DONE; 3565 3566 return (0); 3567 } 3568 3569 static void 3570 quiesce_txq(struct adapter *sc, struct sge_txq *txq) 3571 { 3572 struct sge_eq *eq = &txq->eq; 3573 struct sge_qstat *spg = (void *)&eq->desc[eq->sidx]; 3574 3575 (void) sc; /* unused */ 3576 3577 #ifdef INVARIANTS 3578 TXQ_LOCK(txq); 3579 MPASS((eq->flags & EQ_ENABLED) == 0); 3580 TXQ_UNLOCK(txq); 3581 #endif 3582 3583 /* Wait for the mp_ring to empty. */ 3584 while (!mp_ring_is_idle(txq->r)) { 3585 mp_ring_check_drainage(txq->r, 0); 3586 pause("rquiesce", 1); 3587 } 3588 3589 /* Then wait for the hardware to finish. */ 3590 while (spg->cidx != htobe16(eq->pidx)) 3591 pause("equiesce", 1); 3592 3593 /* Finally, wait for the driver to reclaim all descriptors. */ 3594 while (eq->cidx != eq->pidx) 3595 pause("dquiesce", 1); 3596 } 3597 3598 static void 3599 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq) 3600 { 3601 3602 /* XXXTX */ 3603 } 3604 3605 static void 3606 quiesce_iq(struct adapter *sc, struct sge_iq *iq) 3607 { 3608 (void) sc; /* unused */ 3609 3610 /* Synchronize with the interrupt handler */ 3611 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 3612 pause("iqfree", 1); 3613 } 3614 3615 static void 3616 quiesce_fl(struct adapter *sc, struct sge_fl *fl) 3617 { 3618 mtx_lock(&sc->sfl_lock); 3619 FL_LOCK(fl); 3620 fl->flags |= FL_DOOMED; 3621 FL_UNLOCK(fl); 3622 mtx_unlock(&sc->sfl_lock); 3623 3624 callout_drain(&sc->sfl_callout); 3625 KASSERT((fl->flags & FL_STARVING) == 0, 3626 ("%s: still starving", __func__)); 3627 } 3628 3629 static int 3630 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 3631 driver_intr_t *handler, void *arg, char *name) 3632 { 3633 int rc; 3634 3635 irq->rid = rid; 3636 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 3637 RF_SHAREABLE | RF_ACTIVE); 3638 if (irq->res == NULL) { 3639 device_printf(sc->dev, 3640 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 3641 return (ENOMEM); 3642 } 3643 3644 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 3645 NULL, handler, arg, &irq->tag); 3646 if (rc != 0) { 3647 device_printf(sc->dev, 3648 "failed to setup interrupt for rid %d, name %s: %d\n", 3649 rid, name, rc); 3650 } else if (name) 3651 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 3652 3653 return (rc); 3654 } 3655 3656 static int 3657 t4_free_irq(struct adapter *sc, struct irq *irq) 3658 { 3659 if (irq->tag) 3660 bus_teardown_intr(sc->dev, irq->res, irq->tag); 3661 if (irq->res) 3662 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 3663 3664 bzero(irq, sizeof(*irq)); 3665 3666 return (0); 3667 } 3668 3669 static void 3670 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 3671 unsigned int end) 3672 { 3673 uint32_t *p = (uint32_t *)(buf + start); 3674 3675 for ( ; start <= end; start += sizeof(uint32_t)) 3676 *p++ = t4_read_reg(sc, start); 3677 } 3678 3679 static void 3680 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 3681 { 3682 int i, n; 3683 const unsigned int *reg_ranges; 3684 static const unsigned int t4_reg_ranges[] = { 3685 0x1008, 0x1108, 3686 0x1180, 0x11b4, 3687 0x11fc, 0x123c, 3688 0x1300, 0x173c, 3689 0x1800, 0x18fc, 3690 0x3000, 0x30d8, 3691 0x30e0, 0x5924, 3692 0x5960, 0x59d4, 3693 0x5a00, 0x5af8, 3694 0x6000, 0x6098, 3695 0x6100, 0x6150, 3696 0x6200, 0x6208, 3697 0x6240, 0x6248, 3698 0x6280, 0x6338, 3699 0x6370, 0x638c, 3700 0x6400, 0x643c, 3701 0x6500, 0x6524, 3702 0x6a00, 0x6a38, 3703 0x6a60, 0x6a78, 3704 0x6b00, 0x6b84, 3705 0x6bf0, 0x6c84, 3706 0x6cf0, 0x6d84, 3707 0x6df0, 0x6e84, 3708 0x6ef0, 0x6f84, 3709 0x6ff0, 0x7084, 3710 0x70f0, 0x7184, 3711 0x71f0, 0x7284, 3712 0x72f0, 0x7384, 3713 0x73f0, 0x7450, 3714 0x7500, 0x7530, 3715 0x7600, 0x761c, 3716 0x7680, 0x76cc, 3717 0x7700, 0x7798, 3718 0x77c0, 0x77fc, 3719 0x7900, 0x79fc, 3720 0x7b00, 0x7c38, 3721 0x7d00, 0x7efc, 3722 0x8dc0, 0x8e1c, 3723 0x8e30, 0x8e78, 3724 0x8ea0, 0x8f6c, 3725 0x8fc0, 0x9074, 3726 0x90fc, 0x90fc, 3727 0x9400, 0x9458, 3728 0x9600, 0x96bc, 3729 0x9800, 0x9808, 3730 0x9820, 0x983c, 3731 0x9850, 0x9864, 3732 0x9c00, 0x9c6c, 3733 0x9c80, 0x9cec, 3734 0x9d00, 0x9d6c, 3735 0x9d80, 0x9dec, 3736 0x9e00, 0x9e6c, 3737 0x9e80, 0x9eec, 3738 0x9f00, 0x9f6c, 3739 0x9f80, 0x9fec, 3740 0xd004, 0xd03c, 3741 0xdfc0, 0xdfe0, 3742 0xe000, 0xea7c, 3743 0xf000, 0x11110, 3744 0x11118, 0x11190, 3745 0x19040, 0x1906c, 3746 0x19078, 0x19080, 3747 0x1908c, 0x19124, 3748 0x19150, 0x191b0, 3749 0x191d0, 0x191e8, 3750 0x19238, 0x1924c, 3751 0x193f8, 0x19474, 3752 0x19490, 0x194f8, 3753 0x19800, 0x19f30, 3754 0x1a000, 0x1a06c, 3755 0x1a0b0, 0x1a120, 3756 0x1a128, 0x1a138, 3757 0x1a190, 0x1a1c4, 3758 0x1a1fc, 0x1a1fc, 3759 0x1e040, 0x1e04c, 3760 0x1e284, 0x1e28c, 3761 0x1e2c0, 0x1e2c0, 3762 0x1e2e0, 0x1e2e0, 3763 0x1e300, 0x1e384, 3764 0x1e3c0, 0x1e3c8, 3765 0x1e440, 0x1e44c, 3766 0x1e684, 0x1e68c, 3767 0x1e6c0, 0x1e6c0, 3768 0x1e6e0, 0x1e6e0, 3769 0x1e700, 0x1e784, 3770 0x1e7c0, 0x1e7c8, 3771 0x1e840, 0x1e84c, 3772 0x1ea84, 0x1ea8c, 3773 0x1eac0, 0x1eac0, 3774 0x1eae0, 0x1eae0, 3775 0x1eb00, 0x1eb84, 3776 0x1ebc0, 0x1ebc8, 3777 0x1ec40, 0x1ec4c, 3778 0x1ee84, 0x1ee8c, 3779 0x1eec0, 0x1eec0, 3780 0x1eee0, 0x1eee0, 3781 0x1ef00, 0x1ef84, 3782 0x1efc0, 0x1efc8, 3783 0x1f040, 0x1f04c, 3784 0x1f284, 0x1f28c, 3785 0x1f2c0, 0x1f2c0, 3786 0x1f2e0, 0x1f2e0, 3787 0x1f300, 0x1f384, 3788 0x1f3c0, 0x1f3c8, 3789 0x1f440, 0x1f44c, 3790 0x1f684, 0x1f68c, 3791 0x1f6c0, 0x1f6c0, 3792 0x1f6e0, 0x1f6e0, 3793 0x1f700, 0x1f784, 3794 0x1f7c0, 0x1f7c8, 3795 0x1f840, 0x1f84c, 3796 0x1fa84, 0x1fa8c, 3797 0x1fac0, 0x1fac0, 3798 0x1fae0, 0x1fae0, 3799 0x1fb00, 0x1fb84, 3800 0x1fbc0, 0x1fbc8, 3801 0x1fc40, 0x1fc4c, 3802 0x1fe84, 0x1fe8c, 3803 0x1fec0, 0x1fec0, 3804 0x1fee0, 0x1fee0, 3805 0x1ff00, 0x1ff84, 3806 0x1ffc0, 0x1ffc8, 3807 0x20000, 0x2002c, 3808 0x20100, 0x2013c, 3809 0x20190, 0x201c8, 3810 0x20200, 0x20318, 3811 0x20400, 0x20528, 3812 0x20540, 0x20614, 3813 0x21000, 0x21040, 3814 0x2104c, 0x21060, 3815 0x210c0, 0x210ec, 3816 0x21200, 0x21268, 3817 0x21270, 0x21284, 3818 0x212fc, 0x21388, 3819 0x21400, 0x21404, 3820 0x21500, 0x21518, 3821 0x2152c, 0x2153c, 3822 0x21550, 0x21554, 3823 0x21600, 0x21600, 3824 0x21608, 0x21628, 3825 0x21630, 0x2163c, 3826 0x21700, 0x2171c, 3827 0x21780, 0x2178c, 3828 0x21800, 0x21c38, 3829 0x21c80, 0x21d7c, 3830 0x21e00, 0x21e04, 3831 0x22000, 0x2202c, 3832 0x22100, 0x2213c, 3833 0x22190, 0x221c8, 3834 0x22200, 0x22318, 3835 0x22400, 0x22528, 3836 0x22540, 0x22614, 3837 0x23000, 0x23040, 3838 0x2304c, 0x23060, 3839 0x230c0, 0x230ec, 3840 0x23200, 0x23268, 3841 0x23270, 0x23284, 3842 0x232fc, 0x23388, 3843 0x23400, 0x23404, 3844 0x23500, 0x23518, 3845 0x2352c, 0x2353c, 3846 0x23550, 0x23554, 3847 0x23600, 0x23600, 3848 0x23608, 0x23628, 3849 0x23630, 0x2363c, 3850 0x23700, 0x2371c, 3851 0x23780, 0x2378c, 3852 0x23800, 0x23c38, 3853 0x23c80, 0x23d7c, 3854 0x23e00, 0x23e04, 3855 0x24000, 0x2402c, 3856 0x24100, 0x2413c, 3857 0x24190, 0x241c8, 3858 0x24200, 0x24318, 3859 0x24400, 0x24528, 3860 0x24540, 0x24614, 3861 0x25000, 0x25040, 3862 0x2504c, 0x25060, 3863 0x250c0, 0x250ec, 3864 0x25200, 0x25268, 3865 0x25270, 0x25284, 3866 0x252fc, 0x25388, 3867 0x25400, 0x25404, 3868 0x25500, 0x25518, 3869 0x2552c, 0x2553c, 3870 0x25550, 0x25554, 3871 0x25600, 0x25600, 3872 0x25608, 0x25628, 3873 0x25630, 0x2563c, 3874 0x25700, 0x2571c, 3875 0x25780, 0x2578c, 3876 0x25800, 0x25c38, 3877 0x25c80, 0x25d7c, 3878 0x25e00, 0x25e04, 3879 0x26000, 0x2602c, 3880 0x26100, 0x2613c, 3881 0x26190, 0x261c8, 3882 0x26200, 0x26318, 3883 0x26400, 0x26528, 3884 0x26540, 0x26614, 3885 0x27000, 0x27040, 3886 0x2704c, 0x27060, 3887 0x270c0, 0x270ec, 3888 0x27200, 0x27268, 3889 0x27270, 0x27284, 3890 0x272fc, 0x27388, 3891 0x27400, 0x27404, 3892 0x27500, 0x27518, 3893 0x2752c, 0x2753c, 3894 0x27550, 0x27554, 3895 0x27600, 0x27600, 3896 0x27608, 0x27628, 3897 0x27630, 0x2763c, 3898 0x27700, 0x2771c, 3899 0x27780, 0x2778c, 3900 0x27800, 0x27c38, 3901 0x27c80, 0x27d7c, 3902 0x27e00, 0x27e04 3903 }; 3904 static const unsigned int t5_reg_ranges[] = { 3905 0x1008, 0x1148, 3906 0x1180, 0x11b4, 3907 0x11fc, 0x123c, 3908 0x1280, 0x173c, 3909 0x1800, 0x18fc, 3910 0x3000, 0x3028, 3911 0x3060, 0x30d8, 3912 0x30e0, 0x30fc, 3913 0x3140, 0x357c, 3914 0x35a8, 0x35cc, 3915 0x35ec, 0x35ec, 3916 0x3600, 0x5624, 3917 0x56cc, 0x575c, 3918 0x580c, 0x5814, 3919 0x5890, 0x58bc, 3920 0x5940, 0x59dc, 3921 0x59fc, 0x5a18, 3922 0x5a60, 0x5a9c, 3923 0x5b94, 0x5bfc, 3924 0x6000, 0x6040, 3925 0x6058, 0x614c, 3926 0x7700, 0x7798, 3927 0x77c0, 0x78fc, 3928 0x7b00, 0x7c54, 3929 0x7d00, 0x7efc, 3930 0x8dc0, 0x8de0, 3931 0x8df8, 0x8e84, 3932 0x8ea0, 0x8f84, 3933 0x8fc0, 0x90f8, 3934 0x9400, 0x9470, 3935 0x9600, 0x96f4, 3936 0x9800, 0x9808, 3937 0x9820, 0x983c, 3938 0x9850, 0x9864, 3939 0x9c00, 0x9c6c, 3940 0x9c80, 0x9cec, 3941 0x9d00, 0x9d6c, 3942 0x9d80, 0x9dec, 3943 0x9e00, 0x9e6c, 3944 0x9e80, 0x9eec, 3945 0x9f00, 0x9f6c, 3946 0x9f80, 0xa020, 3947 0xd004, 0xd03c, 3948 0xdfc0, 0xdfe0, 3949 0xe000, 0x11088, 3950 0x1109c, 0x11110, 3951 0x11118, 0x1117c, 3952 0x11190, 0x11204, 3953 0x19040, 0x1906c, 3954 0x19078, 0x19080, 3955 0x1908c, 0x19124, 3956 0x19150, 0x191b0, 3957 0x191d0, 0x191e8, 3958 0x19238, 0x19290, 3959 0x193f8, 0x19474, 3960 0x19490, 0x194cc, 3961 0x194f0, 0x194f8, 3962 0x19c00, 0x19c60, 3963 0x19c94, 0x19e10, 3964 0x19e50, 0x19f34, 3965 0x19f40, 0x19f50, 3966 0x19f90, 0x19fe4, 3967 0x1a000, 0x1a06c, 3968 0x1a0b0, 0x1a120, 3969 0x1a128, 0x1a138, 3970 0x1a190, 0x1a1c4, 3971 0x1a1fc, 0x1a1fc, 3972 0x1e008, 0x1e00c, 3973 0x1e040, 0x1e04c, 3974 0x1e284, 0x1e290, 3975 0x1e2c0, 0x1e2c0, 3976 0x1e2e0, 0x1e2e0, 3977 0x1e300, 0x1e384, 3978 0x1e3c0, 0x1e3c8, 3979 0x1e408, 0x1e40c, 3980 0x1e440, 0x1e44c, 3981 0x1e684, 0x1e690, 3982 0x1e6c0, 0x1e6c0, 3983 0x1e6e0, 0x1e6e0, 3984 0x1e700, 0x1e784, 3985 0x1e7c0, 0x1e7c8, 3986 0x1e808, 0x1e80c, 3987 0x1e840, 0x1e84c, 3988 0x1ea84, 0x1ea90, 3989 0x1eac0, 0x1eac0, 3990 0x1eae0, 0x1eae0, 3991 0x1eb00, 0x1eb84, 3992 0x1ebc0, 0x1ebc8, 3993 0x1ec08, 0x1ec0c, 3994 0x1ec40, 0x1ec4c, 3995 0x1ee84, 0x1ee90, 3996 0x1eec0, 0x1eec0, 3997 0x1eee0, 0x1eee0, 3998 0x1ef00, 0x1ef84, 3999 0x1efc0, 0x1efc8, 4000 0x1f008, 0x1f00c, 4001 0x1f040, 0x1f04c, 4002 0x1f284, 0x1f290, 4003 0x1f2c0, 0x1f2c0, 4004 0x1f2e0, 0x1f2e0, 4005 0x1f300, 0x1f384, 4006 0x1f3c0, 0x1f3c8, 4007 0x1f408, 0x1f40c, 4008 0x1f440, 0x1f44c, 4009 0x1f684, 0x1f690, 4010 0x1f6c0, 0x1f6c0, 4011 0x1f6e0, 0x1f6e0, 4012 0x1f700, 0x1f784, 4013 0x1f7c0, 0x1f7c8, 4014 0x1f808, 0x1f80c, 4015 0x1f840, 0x1f84c, 4016 0x1fa84, 0x1fa90, 4017 0x1fac0, 0x1fac0, 4018 0x1fae0, 0x1fae0, 4019 0x1fb00, 0x1fb84, 4020 0x1fbc0, 0x1fbc8, 4021 0x1fc08, 0x1fc0c, 4022 0x1fc40, 0x1fc4c, 4023 0x1fe84, 0x1fe90, 4024 0x1fec0, 0x1fec0, 4025 0x1fee0, 0x1fee0, 4026 0x1ff00, 0x1ff84, 4027 0x1ffc0, 0x1ffc8, 4028 0x30000, 0x30030, 4029 0x30100, 0x30144, 4030 0x30190, 0x301d0, 4031 0x30200, 0x30318, 4032 0x30400, 0x3052c, 4033 0x30540, 0x3061c, 4034 0x30800, 0x30834, 4035 0x308c0, 0x30908, 4036 0x30910, 0x309ac, 4037 0x30a00, 0x30a2c, 4038 0x30a44, 0x30a50, 4039 0x30a74, 0x30c24, 4040 0x30d00, 0x30d00, 4041 0x30d08, 0x30d14, 4042 0x30d1c, 0x30d20, 4043 0x30d3c, 0x30d50, 4044 0x31200, 0x3120c, 4045 0x31220, 0x31220, 4046 0x31240, 0x31240, 4047 0x31600, 0x3160c, 4048 0x31a00, 0x31a1c, 4049 0x31e00, 0x31e20, 4050 0x31e38, 0x31e3c, 4051 0x31e80, 0x31e80, 4052 0x31e88, 0x31ea8, 4053 0x31eb0, 0x31eb4, 4054 0x31ec8, 0x31ed4, 4055 0x31fb8, 0x32004, 4056 0x32200, 0x32200, 4057 0x32208, 0x32240, 4058 0x32248, 0x32280, 4059 0x32288, 0x322c0, 4060 0x322c8, 0x322fc, 4061 0x32600, 0x32630, 4062 0x32a00, 0x32abc, 4063 0x32b00, 0x32b70, 4064 0x33000, 0x33048, 4065 0x33060, 0x3309c, 4066 0x330f0, 0x33148, 4067 0x33160, 0x3319c, 4068 0x331f0, 0x332e4, 4069 0x332f8, 0x333e4, 4070 0x333f8, 0x33448, 4071 0x33460, 0x3349c, 4072 0x334f0, 0x33548, 4073 0x33560, 0x3359c, 4074 0x335f0, 0x336e4, 4075 0x336f8, 0x337e4, 4076 0x337f8, 0x337fc, 4077 0x33814, 0x33814, 4078 0x3382c, 0x3382c, 4079 0x33880, 0x3388c, 4080 0x338e8, 0x338ec, 4081 0x33900, 0x33948, 4082 0x33960, 0x3399c, 4083 0x339f0, 0x33ae4, 4084 0x33af8, 0x33b10, 4085 0x33b28, 0x33b28, 4086 0x33b3c, 0x33b50, 4087 0x33bf0, 0x33c10, 4088 0x33c28, 0x33c28, 4089 0x33c3c, 0x33c50, 4090 0x33cf0, 0x33cfc, 4091 0x34000, 0x34030, 4092 0x34100, 0x34144, 4093 0x34190, 0x341d0, 4094 0x34200, 0x34318, 4095 0x34400, 0x3452c, 4096 0x34540, 0x3461c, 4097 0x34800, 0x34834, 4098 0x348c0, 0x34908, 4099 0x34910, 0x349ac, 4100 0x34a00, 0x34a2c, 4101 0x34a44, 0x34a50, 4102 0x34a74, 0x34c24, 4103 0x34d00, 0x34d00, 4104 0x34d08, 0x34d14, 4105 0x34d1c, 0x34d20, 4106 0x34d3c, 0x34d50, 4107 0x35200, 0x3520c, 4108 0x35220, 0x35220, 4109 0x35240, 0x35240, 4110 0x35600, 0x3560c, 4111 0x35a00, 0x35a1c, 4112 0x35e00, 0x35e20, 4113 0x35e38, 0x35e3c, 4114 0x35e80, 0x35e80, 4115 0x35e88, 0x35ea8, 4116 0x35eb0, 0x35eb4, 4117 0x35ec8, 0x35ed4, 4118 0x35fb8, 0x36004, 4119 0x36200, 0x36200, 4120 0x36208, 0x36240, 4121 0x36248, 0x36280, 4122 0x36288, 0x362c0, 4123 0x362c8, 0x362fc, 4124 0x36600, 0x36630, 4125 0x36a00, 0x36abc, 4126 0x36b00, 0x36b70, 4127 0x37000, 0x37048, 4128 0x37060, 0x3709c, 4129 0x370f0, 0x37148, 4130 0x37160, 0x3719c, 4131 0x371f0, 0x372e4, 4132 0x372f8, 0x373e4, 4133 0x373f8, 0x37448, 4134 0x37460, 0x3749c, 4135 0x374f0, 0x37548, 4136 0x37560, 0x3759c, 4137 0x375f0, 0x376e4, 4138 0x376f8, 0x377e4, 4139 0x377f8, 0x377fc, 4140 0x37814, 0x37814, 4141 0x3782c, 0x3782c, 4142 0x37880, 0x3788c, 4143 0x378e8, 0x378ec, 4144 0x37900, 0x37948, 4145 0x37960, 0x3799c, 4146 0x379f0, 0x37ae4, 4147 0x37af8, 0x37b10, 4148 0x37b28, 0x37b28, 4149 0x37b3c, 0x37b50, 4150 0x37bf0, 0x37c10, 4151 0x37c28, 0x37c28, 4152 0x37c3c, 0x37c50, 4153 0x37cf0, 0x37cfc, 4154 0x38000, 0x38030, 4155 0x38100, 0x38144, 4156 0x38190, 0x381d0, 4157 0x38200, 0x38318, 4158 0x38400, 0x3852c, 4159 0x38540, 0x3861c, 4160 0x38800, 0x38834, 4161 0x388c0, 0x38908, 4162 0x38910, 0x389ac, 4163 0x38a00, 0x38a2c, 4164 0x38a44, 0x38a50, 4165 0x38a74, 0x38c24, 4166 0x38d00, 0x38d00, 4167 0x38d08, 0x38d14, 4168 0x38d1c, 0x38d20, 4169 0x38d3c, 0x38d50, 4170 0x39200, 0x3920c, 4171 0x39220, 0x39220, 4172 0x39240, 0x39240, 4173 0x39600, 0x3960c, 4174 0x39a00, 0x39a1c, 4175 0x39e00, 0x39e20, 4176 0x39e38, 0x39e3c, 4177 0x39e80, 0x39e80, 4178 0x39e88, 0x39ea8, 4179 0x39eb0, 0x39eb4, 4180 0x39ec8, 0x39ed4, 4181 0x39fb8, 0x3a004, 4182 0x3a200, 0x3a200, 4183 0x3a208, 0x3a240, 4184 0x3a248, 0x3a280, 4185 0x3a288, 0x3a2c0, 4186 0x3a2c8, 0x3a2fc, 4187 0x3a600, 0x3a630, 4188 0x3aa00, 0x3aabc, 4189 0x3ab00, 0x3ab70, 4190 0x3b000, 0x3b048, 4191 0x3b060, 0x3b09c, 4192 0x3b0f0, 0x3b148, 4193 0x3b160, 0x3b19c, 4194 0x3b1f0, 0x3b2e4, 4195 0x3b2f8, 0x3b3e4, 4196 0x3b3f8, 0x3b448, 4197 0x3b460, 0x3b49c, 4198 0x3b4f0, 0x3b548, 4199 0x3b560, 0x3b59c, 4200 0x3b5f0, 0x3b6e4, 4201 0x3b6f8, 0x3b7e4, 4202 0x3b7f8, 0x3b7fc, 4203 0x3b814, 0x3b814, 4204 0x3b82c, 0x3b82c, 4205 0x3b880, 0x3b88c, 4206 0x3b8e8, 0x3b8ec, 4207 0x3b900, 0x3b948, 4208 0x3b960, 0x3b99c, 4209 0x3b9f0, 0x3bae4, 4210 0x3baf8, 0x3bb10, 4211 0x3bb28, 0x3bb28, 4212 0x3bb3c, 0x3bb50, 4213 0x3bbf0, 0x3bc10, 4214 0x3bc28, 0x3bc28, 4215 0x3bc3c, 0x3bc50, 4216 0x3bcf0, 0x3bcfc, 4217 0x3c000, 0x3c030, 4218 0x3c100, 0x3c144, 4219 0x3c190, 0x3c1d0, 4220 0x3c200, 0x3c318, 4221 0x3c400, 0x3c52c, 4222 0x3c540, 0x3c61c, 4223 0x3c800, 0x3c834, 4224 0x3c8c0, 0x3c908, 4225 0x3c910, 0x3c9ac, 4226 0x3ca00, 0x3ca2c, 4227 0x3ca44, 0x3ca50, 4228 0x3ca74, 0x3cc24, 4229 0x3cd00, 0x3cd00, 4230 0x3cd08, 0x3cd14, 4231 0x3cd1c, 0x3cd20, 4232 0x3cd3c, 0x3cd50, 4233 0x3d200, 0x3d20c, 4234 0x3d220, 0x3d220, 4235 0x3d240, 0x3d240, 4236 0x3d600, 0x3d60c, 4237 0x3da00, 0x3da1c, 4238 0x3de00, 0x3de20, 4239 0x3de38, 0x3de3c, 4240 0x3de80, 0x3de80, 4241 0x3de88, 0x3dea8, 4242 0x3deb0, 0x3deb4, 4243 0x3dec8, 0x3ded4, 4244 0x3dfb8, 0x3e004, 4245 0x3e200, 0x3e200, 4246 0x3e208, 0x3e240, 4247 0x3e248, 0x3e280, 4248 0x3e288, 0x3e2c0, 4249 0x3e2c8, 0x3e2fc, 4250 0x3e600, 0x3e630, 4251 0x3ea00, 0x3eabc, 4252 0x3eb00, 0x3eb70, 4253 0x3f000, 0x3f048, 4254 0x3f060, 0x3f09c, 4255 0x3f0f0, 0x3f148, 4256 0x3f160, 0x3f19c, 4257 0x3f1f0, 0x3f2e4, 4258 0x3f2f8, 0x3f3e4, 4259 0x3f3f8, 0x3f448, 4260 0x3f460, 0x3f49c, 4261 0x3f4f0, 0x3f548, 4262 0x3f560, 0x3f59c, 4263 0x3f5f0, 0x3f6e4, 4264 0x3f6f8, 0x3f7e4, 4265 0x3f7f8, 0x3f7fc, 4266 0x3f814, 0x3f814, 4267 0x3f82c, 0x3f82c, 4268 0x3f880, 0x3f88c, 4269 0x3f8e8, 0x3f8ec, 4270 0x3f900, 0x3f948, 4271 0x3f960, 0x3f99c, 4272 0x3f9f0, 0x3fae4, 4273 0x3faf8, 0x3fb10, 4274 0x3fb28, 0x3fb28, 4275 0x3fb3c, 0x3fb50, 4276 0x3fbf0, 0x3fc10, 4277 0x3fc28, 0x3fc28, 4278 0x3fc3c, 0x3fc50, 4279 0x3fcf0, 0x3fcfc, 4280 0x40000, 0x4000c, 4281 0x40040, 0x40068, 4282 0x4007c, 0x40144, 4283 0x40180, 0x4018c, 4284 0x40200, 0x40298, 4285 0x402ac, 0x4033c, 4286 0x403f8, 0x403fc, 4287 0x41304, 0x413c4, 4288 0x41400, 0x4141c, 4289 0x41480, 0x414d0, 4290 0x44000, 0x44078, 4291 0x440c0, 0x44278, 4292 0x442c0, 0x44478, 4293 0x444c0, 0x44678, 4294 0x446c0, 0x44878, 4295 0x448c0, 0x449fc, 4296 0x45000, 0x45068, 4297 0x45080, 0x45084, 4298 0x450a0, 0x450b0, 4299 0x45200, 0x45268, 4300 0x45280, 0x45284, 4301 0x452a0, 0x452b0, 4302 0x460c0, 0x460e4, 4303 0x47000, 0x4708c, 4304 0x47200, 0x47250, 4305 0x47400, 0x47420, 4306 0x47600, 0x47618, 4307 0x47800, 0x47814, 4308 0x48000, 0x4800c, 4309 0x48040, 0x48068, 4310 0x4807c, 0x48144, 4311 0x48180, 0x4818c, 4312 0x48200, 0x48298, 4313 0x482ac, 0x4833c, 4314 0x483f8, 0x483fc, 4315 0x49304, 0x493c4, 4316 0x49400, 0x4941c, 4317 0x49480, 0x494d0, 4318 0x4c000, 0x4c078, 4319 0x4c0c0, 0x4c278, 4320 0x4c2c0, 0x4c478, 4321 0x4c4c0, 0x4c678, 4322 0x4c6c0, 0x4c878, 4323 0x4c8c0, 0x4c9fc, 4324 0x4d000, 0x4d068, 4325 0x4d080, 0x4d084, 4326 0x4d0a0, 0x4d0b0, 4327 0x4d200, 0x4d268, 4328 0x4d280, 0x4d284, 4329 0x4d2a0, 0x4d2b0, 4330 0x4e0c0, 0x4e0e4, 4331 0x4f000, 0x4f08c, 4332 0x4f200, 0x4f250, 4333 0x4f400, 0x4f420, 4334 0x4f600, 0x4f618, 4335 0x4f800, 0x4f814, 4336 0x50000, 0x500cc, 4337 0x50400, 0x50400, 4338 0x50800, 0x508cc, 4339 0x50c00, 0x50c00, 4340 0x51000, 0x5101c, 4341 0x51300, 0x51308, 4342 }; 4343 4344 if (is_t4(sc)) { 4345 reg_ranges = &t4_reg_ranges[0]; 4346 n = nitems(t4_reg_ranges); 4347 } else { 4348 reg_ranges = &t5_reg_ranges[0]; 4349 n = nitems(t5_reg_ranges); 4350 } 4351 4352 regs->version = chip_id(sc) | chip_rev(sc) << 10; 4353 for (i = 0; i < n; i += 2) 4354 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 4355 } 4356 4357 static void 4358 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi) 4359 { 4360 int i; 4361 u_int v, tnl_cong_drops; 4362 struct timeval tv; 4363 const struct timeval interval = {0, 250000}; /* 250ms */ 4364 4365 getmicrotime(&tv); 4366 timevalsub(&tv, &interval); 4367 if (timevalcmp(&tv, &pi->last_refreshed, <)) 4368 return; 4369 4370 tnl_cong_drops = 0; 4371 t4_get_port_stats(sc, pi->tx_chan, &pi->stats); 4372 for (i = 0; i < NCHAN; i++) { 4373 if (pi->rx_chan_map & (1 << i)) { 4374 mtx_lock(&sc->regwin_lock); 4375 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 4376 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 4377 mtx_unlock(&sc->regwin_lock); 4378 tnl_cong_drops += v; 4379 } 4380 } 4381 pi->tnl_cong_drops = tnl_cong_drops; 4382 getmicrotime(&pi->last_refreshed); 4383 } 4384 4385 static void 4386 cxgbe_tick(void *arg) 4387 { 4388 struct port_info *pi = arg; 4389 struct adapter *sc = pi->adapter; 4390 struct ifnet *ifp = pi->ifp; 4391 4392 PORT_LOCK(pi); 4393 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 4394 PORT_UNLOCK(pi); 4395 return; /* without scheduling another callout */ 4396 } 4397 4398 cxgbe_refresh_stats(sc, pi); 4399 4400 callout_schedule(&pi->tick, hz); 4401 PORT_UNLOCK(pi); 4402 } 4403 4404 static void 4405 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 4406 { 4407 struct ifnet *vlan; 4408 4409 if (arg != ifp || ifp->if_type != IFT_ETHER) 4410 return; 4411 4412 vlan = VLAN_DEVAT(ifp, vid); 4413 VLAN_SETCOOKIE(vlan, ifp); 4414 } 4415 4416 static int 4417 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 4418 { 4419 4420 #ifdef INVARIANTS 4421 panic("%s: opcode 0x%02x on iq %p with payload %p", 4422 __func__, rss->opcode, iq, m); 4423 #else 4424 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 4425 __func__, rss->opcode, iq, m); 4426 m_freem(m); 4427 #endif 4428 return (EDOOFUS); 4429 } 4430 4431 int 4432 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 4433 { 4434 uintptr_t *loc, new; 4435 4436 if (opcode >= nitems(sc->cpl_handler)) 4437 return (EINVAL); 4438 4439 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 4440 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 4441 atomic_store_rel_ptr(loc, new); 4442 4443 return (0); 4444 } 4445 4446 static int 4447 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 4448 { 4449 4450 #ifdef INVARIANTS 4451 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 4452 #else 4453 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 4454 __func__, iq, ctrl); 4455 #endif 4456 return (EDOOFUS); 4457 } 4458 4459 int 4460 t4_register_an_handler(struct adapter *sc, an_handler_t h) 4461 { 4462 uintptr_t *loc, new; 4463 4464 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 4465 loc = (uintptr_t *) &sc->an_handler; 4466 atomic_store_rel_ptr(loc, new); 4467 4468 return (0); 4469 } 4470 4471 static int 4472 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 4473 { 4474 const struct cpl_fw6_msg *cpl = 4475 __containerof(rpl, struct cpl_fw6_msg, data[0]); 4476 4477 #ifdef INVARIANTS 4478 panic("%s: fw_msg type %d", __func__, cpl->type); 4479 #else 4480 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 4481 #endif 4482 return (EDOOFUS); 4483 } 4484 4485 int 4486 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 4487 { 4488 uintptr_t *loc, new; 4489 4490 if (type >= nitems(sc->fw_msg_handler)) 4491 return (EINVAL); 4492 4493 /* 4494 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 4495 * handler dispatch table. Reject any attempt to install a handler for 4496 * this subtype. 4497 */ 4498 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 4499 return (EINVAL); 4500 4501 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 4502 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 4503 atomic_store_rel_ptr(loc, new); 4504 4505 return (0); 4506 } 4507 4508 static int 4509 t4_sysctls(struct adapter *sc) 4510 { 4511 struct sysctl_ctx_list *ctx; 4512 struct sysctl_oid *oid; 4513 struct sysctl_oid_list *children, *c0; 4514 static char *caps[] = { 4515 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 4516 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL" /* caps[1] niccaps */ 4517 "\6HASHFILTER\7ETHOFLD", 4518 "\20\1TOE", /* caps[2] toecaps */ 4519 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 4520 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 4521 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 4522 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 4523 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 4524 "\4PO_INITIAOR\5PO_TARGET" 4525 }; 4526 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 4527 4528 ctx = device_get_sysctl_ctx(sc->dev); 4529 4530 /* 4531 * dev.t4nex.X. 4532 */ 4533 oid = device_get_sysctl_tree(sc->dev); 4534 c0 = children = SYSCTL_CHILDREN(oid); 4535 4536 sc->sc_do_rxcopy = 1; 4537 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, 4538 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); 4539 4540 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 4541 sc->params.nports, "# of ports"); 4542 4543 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 4544 NULL, chip_rev(sc), "chip hardware revision"); 4545 4546 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 4547 CTLFLAG_RD, sc->fw_version, 0, "firmware version"); 4548 4549 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 4550 CTLFLAG_RD, sc->cfg_file, 0, "configuration file"); 4551 4552 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 4553 sc->cfcsum, "config file checksum"); 4554 4555 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 4556 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 4557 sysctl_bitfield, "A", "available doorbells"); 4558 4559 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 4560 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 4561 sysctl_bitfield, "A", "available link capabilities"); 4562 4563 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 4564 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 4565 sysctl_bitfield, "A", "available NIC capabilities"); 4566 4567 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 4568 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 4569 sysctl_bitfield, "A", "available TCP offload capabilities"); 4570 4571 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 4572 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 4573 sysctl_bitfield, "A", "available RDMA capabilities"); 4574 4575 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 4576 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 4577 sysctl_bitfield, "A", "available iSCSI capabilities"); 4578 4579 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 4580 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 4581 sysctl_bitfield, "A", "available FCoE capabilities"); 4582 4583 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 4584 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 4585 4586 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 4587 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 4588 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 4589 "interrupt holdoff timer values (us)"); 4590 4591 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 4592 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 4593 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 4594 "interrupt holdoff packet counter values"); 4595 4596 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, 4597 NULL, sc->tids.nftids, "number of filters"); 4598 4599 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | 4600 CTLFLAG_RD, sc, 0, sysctl_temperature, "I", 4601 "chip temperature (in Celsius)"); 4602 4603 t4_sge_sysctls(sc, ctx, children); 4604 4605 sc->lro_timeout = 100; 4606 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, 4607 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); 4608 4609 #ifdef SBUF_DRAIN 4610 /* 4611 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 4612 */ 4613 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 4614 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 4615 "logs and miscellaneous information"); 4616 children = SYSCTL_CHILDREN(oid); 4617 4618 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 4619 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4620 sysctl_cctrl, "A", "congestion control"); 4621 4622 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 4623 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4624 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 4625 4626 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 4627 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 4628 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 4629 4630 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 4631 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 4632 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 4633 4634 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 4635 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 4636 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 4637 4638 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 4639 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 4640 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 4641 4642 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 4643 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 4644 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 4645 4646 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 4647 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4648 sysctl_cim_la, "A", "CIM logic analyzer"); 4649 4650 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", 4651 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4652 sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); 4653 4654 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 4655 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 4656 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 4657 4658 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 4659 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 4660 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 4661 4662 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 4663 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 4664 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 4665 4666 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 4667 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 4668 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 4669 4670 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 4671 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 4672 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 4673 4674 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 4675 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 4676 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 4677 4678 if (is_t5(sc)) { 4679 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 4680 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 4681 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 4682 4683 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 4684 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 4685 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 4686 } 4687 4688 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", 4689 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4690 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); 4691 4692 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 4693 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4694 sysctl_cim_qcfg, "A", "CIM queue configuration"); 4695 4696 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 4697 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4698 sysctl_cpl_stats, "A", "CPL statistics"); 4699 4700 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 4701 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4702 sysctl_ddp_stats, "A", "non-TCP DDP statistics"); 4703 4704 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 4705 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4706 sysctl_devlog, "A", "firmware's device log"); 4707 4708 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 4709 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4710 sysctl_fcoe_stats, "A", "FCoE statistics"); 4711 4712 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 4713 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4714 sysctl_hw_sched, "A", "hardware scheduler "); 4715 4716 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 4717 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4718 sysctl_l2t, "A", "hardware L2 table"); 4719 4720 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 4721 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4722 sysctl_lb_stats, "A", "loopback statistics"); 4723 4724 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 4725 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4726 sysctl_meminfo, "A", "memory regions"); 4727 4728 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", 4729 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4730 sysctl_mps_tcam, "A", "MPS TCAM entries"); 4731 4732 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 4733 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4734 sysctl_path_mtus, "A", "path MTUs"); 4735 4736 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 4737 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4738 sysctl_pm_stats, "A", "PM statistics"); 4739 4740 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 4741 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4742 sysctl_rdma_stats, "A", "RDMA statistics"); 4743 4744 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 4745 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4746 sysctl_tcp_stats, "A", "TCP statistics"); 4747 4748 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 4749 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4750 sysctl_tids, "A", "TID information"); 4751 4752 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 4753 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4754 sysctl_tp_err_stats, "A", "TP error statistics"); 4755 4756 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", 4757 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4758 sysctl_tp_la, "A", "TP logic analyzer"); 4759 4760 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 4761 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4762 sysctl_tx_rate, "A", "Tx rate"); 4763 4764 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", 4765 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4766 sysctl_ulprx_la, "A", "ULPRX logic analyzer"); 4767 4768 if (is_t5(sc)) { 4769 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 4770 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4771 sysctl_wcwr_stats, "A", "write combined work requests"); 4772 } 4773 #endif 4774 4775 #ifdef TCP_OFFLOAD 4776 if (is_offload(sc)) { 4777 /* 4778 * dev.t4nex.X.toe. 4779 */ 4780 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 4781 NULL, "TOE parameters"); 4782 children = SYSCTL_CHILDREN(oid); 4783 4784 sc->tt.sndbuf = 256 * 1024; 4785 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 4786 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 4787 4788 sc->tt.ddp = 0; 4789 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 4790 &sc->tt.ddp, 0, "DDP allowed"); 4791 4792 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 4793 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 4794 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 4795 4796 sc->tt.ddp_thres = 4797 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 4798 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 4799 &sc->tt.ddp_thres, 0, "DDP threshold"); 4800 4801 sc->tt.rx_coalesce = 1; 4802 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", 4803 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); 4804 4805 sc->tt.tx_align = 1; 4806 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align", 4807 CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload"); 4808 } 4809 #endif 4810 4811 4812 return (0); 4813 } 4814 4815 static int 4816 cxgbe_sysctls(struct port_info *pi) 4817 { 4818 struct sysctl_ctx_list *ctx; 4819 struct sysctl_oid *oid; 4820 struct sysctl_oid_list *children; 4821 struct adapter *sc = pi->adapter; 4822 4823 ctx = device_get_sysctl_ctx(pi->dev); 4824 4825 /* 4826 * dev.cxgbe.X. 4827 */ 4828 oid = device_get_sysctl_tree(pi->dev); 4829 children = SYSCTL_CHILDREN(oid); 4830 4831 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | 4832 CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); 4833 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { 4834 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 4835 CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I", 4836 "PHY temperature (in Celsius)"); 4837 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", 4838 CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I", 4839 "PHY firmware version"); 4840 } 4841 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 4842 &pi->nrxq, 0, "# of rx queues"); 4843 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 4844 &pi->ntxq, 0, "# of tx queues"); 4845 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 4846 &pi->first_rxq, 0, "index of first rx queue"); 4847 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 4848 &pi->first_txq, 0, "index of first tx queue"); 4849 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT | 4850 CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU", 4851 "Reserve queue 0 for non-flowid packets"); 4852 4853 #ifdef TCP_OFFLOAD 4854 if (is_offload(sc)) { 4855 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 4856 &pi->nofldrxq, 0, 4857 "# of rx queues for offloaded TCP connections"); 4858 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 4859 &pi->nofldtxq, 0, 4860 "# of tx queues for offloaded TCP connections"); 4861 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 4862 CTLFLAG_RD, &pi->first_ofld_rxq, 0, 4863 "index of first TOE rx queue"); 4864 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 4865 CTLFLAG_RD, &pi->first_ofld_txq, 0, 4866 "index of first TOE tx queue"); 4867 } 4868 #endif 4869 #ifdef DEV_NETMAP 4870 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD, 4871 &pi->nnmrxq, 0, "# of rx queues for netmap"); 4872 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD, 4873 &pi->nnmtxq, 0, "# of tx queues for netmap"); 4874 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq", 4875 CTLFLAG_RD, &pi->first_nm_rxq, 0, 4876 "index of first netmap rx queue"); 4877 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq", 4878 CTLFLAG_RD, &pi->first_nm_txq, 0, 4879 "index of first netmap tx queue"); 4880 #endif 4881 4882 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 4883 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 4884 "holdoff timer index"); 4885 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 4886 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 4887 "holdoff packet counter index"); 4888 4889 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 4890 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 4891 "rx queue size"); 4892 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 4893 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 4894 "tx queue size"); 4895 4896 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings", 4897 CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings, 4898 "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)"); 4899 4900 /* 4901 * dev.cxgbe.X.stats. 4902 */ 4903 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 4904 NULL, "port statistics"); 4905 children = SYSCTL_CHILDREN(oid); 4906 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD, 4907 &pi->tx_parse_error, 0, 4908 "# of tx packets with invalid length or # of segments"); 4909 4910 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 4911 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 4912 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ 4913 sysctl_handle_t4_reg64, "QU", desc) 4914 4915 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 4916 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 4917 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 4918 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 4919 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 4920 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 4921 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 4922 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 4923 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 4924 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 4925 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 4926 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 4927 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 4928 "# of tx frames in this range", 4929 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 4930 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 4931 "# of tx frames in this range", 4932 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 4933 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 4934 "# of tx frames in this range", 4935 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 4936 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 4937 "# of tx frames in this range", 4938 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 4939 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 4940 "# of tx frames in this range", 4941 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 4942 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 4943 "# of tx frames in this range", 4944 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 4945 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 4946 "# of tx frames in this range", 4947 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 4948 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 4949 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 4950 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 4951 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 4952 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 4953 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 4954 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 4955 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 4956 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 4957 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 4958 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 4959 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 4960 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 4961 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 4962 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 4963 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 4964 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 4965 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 4966 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 4967 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 4968 4969 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 4970 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 4971 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 4972 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 4973 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 4974 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 4975 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 4976 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 4977 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 4978 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 4979 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 4980 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 4981 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 4982 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 4983 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 4984 "# of frames received with bad FCS", 4985 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 4986 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 4987 "# of frames received with length error", 4988 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 4989 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 4990 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 4991 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 4992 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 4993 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 4994 "# of rx frames in this range", 4995 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 4996 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 4997 "# of rx frames in this range", 4998 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 4999 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 5000 "# of rx frames in this range", 5001 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 5002 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 5003 "# of rx frames in this range", 5004 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 5005 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 5006 "# of rx frames in this range", 5007 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 5008 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 5009 "# of rx frames in this range", 5010 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 5011 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 5012 "# of rx frames in this range", 5013 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 5014 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 5015 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 5016 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 5017 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 5018 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 5019 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 5020 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 5021 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 5022 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 5023 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 5024 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 5025 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 5026 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 5027 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 5028 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 5029 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 5030 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 5031 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 5032 5033 #undef SYSCTL_ADD_T4_REG64 5034 5035 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 5036 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 5037 &pi->stats.name, desc) 5038 5039 /* We get these from port_stats and they may be stale by upto 1s */ 5040 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 5041 "# drops due to buffer-group 0 overflows"); 5042 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 5043 "# drops due to buffer-group 1 overflows"); 5044 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 5045 "# drops due to buffer-group 2 overflows"); 5046 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 5047 "# drops due to buffer-group 3 overflows"); 5048 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 5049 "# of buffer-group 0 truncated packets"); 5050 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 5051 "# of buffer-group 1 truncated packets"); 5052 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 5053 "# of buffer-group 2 truncated packets"); 5054 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 5055 "# of buffer-group 3 truncated packets"); 5056 5057 #undef SYSCTL_ADD_T4_PORTSTAT 5058 5059 return (0); 5060 } 5061 5062 static int 5063 sysctl_int_array(SYSCTL_HANDLER_ARGS) 5064 { 5065 int rc, *i; 5066 struct sbuf sb; 5067 5068 sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND); 5069 for (i = arg1; arg2; arg2 -= sizeof(int), i++) 5070 sbuf_printf(&sb, "%d ", *i); 5071 sbuf_trim(&sb); 5072 sbuf_finish(&sb); 5073 rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 5074 sbuf_delete(&sb); 5075 return (rc); 5076 } 5077 5078 static int 5079 sysctl_bitfield(SYSCTL_HANDLER_ARGS) 5080 { 5081 int rc; 5082 struct sbuf *sb; 5083 5084 rc = sysctl_wire_old_buffer(req, 0); 5085 if (rc != 0) 5086 return(rc); 5087 5088 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5089 if (sb == NULL) 5090 return (ENOMEM); 5091 5092 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 5093 rc = sbuf_finish(sb); 5094 sbuf_delete(sb); 5095 5096 return (rc); 5097 } 5098 5099 static int 5100 sysctl_btphy(SYSCTL_HANDLER_ARGS) 5101 { 5102 struct port_info *pi = arg1; 5103 int op = arg2; 5104 struct adapter *sc = pi->adapter; 5105 u_int v; 5106 int rc; 5107 5108 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt"); 5109 if (rc) 5110 return (rc); 5111 /* XXX: magic numbers */ 5112 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, 5113 &v); 5114 end_synchronized_op(sc, 0); 5115 if (rc) 5116 return (rc); 5117 if (op == 0) 5118 v /= 256; 5119 5120 rc = sysctl_handle_int(oidp, &v, 0, req); 5121 return (rc); 5122 } 5123 5124 static int 5125 sysctl_noflowq(SYSCTL_HANDLER_ARGS) 5126 { 5127 struct port_info *pi = arg1; 5128 int rc, val; 5129 5130 val = pi->rsrv_noflowq; 5131 rc = sysctl_handle_int(oidp, &val, 0, req); 5132 if (rc != 0 || req->newptr == NULL) 5133 return (rc); 5134 5135 if ((val >= 1) && (pi->ntxq > 1)) 5136 pi->rsrv_noflowq = 1; 5137 else 5138 pi->rsrv_noflowq = 0; 5139 5140 return (rc); 5141 } 5142 5143 static int 5144 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 5145 { 5146 struct port_info *pi = arg1; 5147 struct adapter *sc = pi->adapter; 5148 int idx, rc, i; 5149 struct sge_rxq *rxq; 5150 #ifdef TCP_OFFLOAD 5151 struct sge_ofld_rxq *ofld_rxq; 5152 #endif 5153 uint8_t v; 5154 5155 idx = pi->tmr_idx; 5156 5157 rc = sysctl_handle_int(oidp, &idx, 0, req); 5158 if (rc != 0 || req->newptr == NULL) 5159 return (rc); 5160 5161 if (idx < 0 || idx >= SGE_NTIMERS) 5162 return (EINVAL); 5163 5164 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5165 "t4tmr"); 5166 if (rc) 5167 return (rc); 5168 5169 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 5170 for_each_rxq(pi, i, rxq) { 5171 #ifdef atomic_store_rel_8 5172 atomic_store_rel_8(&rxq->iq.intr_params, v); 5173 #else 5174 rxq->iq.intr_params = v; 5175 #endif 5176 } 5177 #ifdef TCP_OFFLOAD 5178 for_each_ofld_rxq(pi, i, ofld_rxq) { 5179 #ifdef atomic_store_rel_8 5180 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); 5181 #else 5182 ofld_rxq->iq.intr_params = v; 5183 #endif 5184 } 5185 #endif 5186 pi->tmr_idx = idx; 5187 5188 end_synchronized_op(sc, LOCK_HELD); 5189 return (0); 5190 } 5191 5192 static int 5193 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 5194 { 5195 struct port_info *pi = arg1; 5196 struct adapter *sc = pi->adapter; 5197 int idx, rc; 5198 5199 idx = pi->pktc_idx; 5200 5201 rc = sysctl_handle_int(oidp, &idx, 0, req); 5202 if (rc != 0 || req->newptr == NULL) 5203 return (rc); 5204 5205 if (idx < -1 || idx >= SGE_NCOUNTERS) 5206 return (EINVAL); 5207 5208 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5209 "t4pktc"); 5210 if (rc) 5211 return (rc); 5212 5213 if (pi->flags & PORT_INIT_DONE) 5214 rc = EBUSY; /* cannot be changed once the queues are created */ 5215 else 5216 pi->pktc_idx = idx; 5217 5218 end_synchronized_op(sc, LOCK_HELD); 5219 return (rc); 5220 } 5221 5222 static int 5223 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 5224 { 5225 struct port_info *pi = arg1; 5226 struct adapter *sc = pi->adapter; 5227 int qsize, rc; 5228 5229 qsize = pi->qsize_rxq; 5230 5231 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5232 if (rc != 0 || req->newptr == NULL) 5233 return (rc); 5234 5235 if (qsize < 128 || (qsize & 7)) 5236 return (EINVAL); 5237 5238 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5239 "t4rxqs"); 5240 if (rc) 5241 return (rc); 5242 5243 if (pi->flags & PORT_INIT_DONE) 5244 rc = EBUSY; /* cannot be changed once the queues are created */ 5245 else 5246 pi->qsize_rxq = qsize; 5247 5248 end_synchronized_op(sc, LOCK_HELD); 5249 return (rc); 5250 } 5251 5252 static int 5253 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 5254 { 5255 struct port_info *pi = arg1; 5256 struct adapter *sc = pi->adapter; 5257 int qsize, rc; 5258 5259 qsize = pi->qsize_txq; 5260 5261 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5262 if (rc != 0 || req->newptr == NULL) 5263 return (rc); 5264 5265 if (qsize < 128 || qsize > 65536) 5266 return (EINVAL); 5267 5268 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5269 "t4txqs"); 5270 if (rc) 5271 return (rc); 5272 5273 if (pi->flags & PORT_INIT_DONE) 5274 rc = EBUSY; /* cannot be changed once the queues are created */ 5275 else 5276 pi->qsize_txq = qsize; 5277 5278 end_synchronized_op(sc, LOCK_HELD); 5279 return (rc); 5280 } 5281 5282 static int 5283 sysctl_pause_settings(SYSCTL_HANDLER_ARGS) 5284 { 5285 struct port_info *pi = arg1; 5286 struct adapter *sc = pi->adapter; 5287 struct link_config *lc = &pi->link_cfg; 5288 int rc; 5289 5290 if (req->newptr == NULL) { 5291 struct sbuf *sb; 5292 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX"; 5293 5294 rc = sysctl_wire_old_buffer(req, 0); 5295 if (rc != 0) 5296 return(rc); 5297 5298 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5299 if (sb == NULL) 5300 return (ENOMEM); 5301 5302 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits); 5303 rc = sbuf_finish(sb); 5304 sbuf_delete(sb); 5305 } else { 5306 char s[2]; 5307 int n; 5308 5309 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX)); 5310 s[1] = 0; 5311 5312 rc = sysctl_handle_string(oidp, s, sizeof(s), req); 5313 if (rc != 0) 5314 return(rc); 5315 5316 if (s[1] != 0) 5317 return (EINVAL); 5318 if (s[0] < '0' || s[0] > '9') 5319 return (EINVAL); /* not a number */ 5320 n = s[0] - '0'; 5321 if (n & ~(PAUSE_TX | PAUSE_RX)) 5322 return (EINVAL); /* some other bit is set too */ 5323 5324 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE"); 5325 if (rc) 5326 return (rc); 5327 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) { 5328 int link_ok = lc->link_ok; 5329 5330 lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX); 5331 lc->requested_fc |= n; 5332 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc); 5333 lc->link_ok = link_ok; /* restore */ 5334 } 5335 end_synchronized_op(sc, 0); 5336 } 5337 5338 return (rc); 5339 } 5340 5341 static int 5342 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 5343 { 5344 struct adapter *sc = arg1; 5345 int reg = arg2; 5346 uint64_t val; 5347 5348 val = t4_read_reg64(sc, reg); 5349 5350 return (sysctl_handle_64(oidp, &val, 0, req)); 5351 } 5352 5353 static int 5354 sysctl_temperature(SYSCTL_HANDLER_ARGS) 5355 { 5356 struct adapter *sc = arg1; 5357 int rc, t; 5358 uint32_t param, val; 5359 5360 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); 5361 if (rc) 5362 return (rc); 5363 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 5364 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 5365 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); 5366 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 5367 end_synchronized_op(sc, 0); 5368 if (rc) 5369 return (rc); 5370 5371 /* unknown is returned as 0 but we display -1 in that case */ 5372 t = val == 0 ? -1 : val; 5373 5374 rc = sysctl_handle_int(oidp, &t, 0, req); 5375 return (rc); 5376 } 5377 5378 #ifdef SBUF_DRAIN 5379 static int 5380 sysctl_cctrl(SYSCTL_HANDLER_ARGS) 5381 { 5382 struct adapter *sc = arg1; 5383 struct sbuf *sb; 5384 int rc, i; 5385 uint16_t incr[NMTUS][NCCTRL_WIN]; 5386 static const char *dec_fac[] = { 5387 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 5388 "0.9375" 5389 }; 5390 5391 rc = sysctl_wire_old_buffer(req, 0); 5392 if (rc != 0) 5393 return (rc); 5394 5395 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5396 if (sb == NULL) 5397 return (ENOMEM); 5398 5399 t4_read_cong_tbl(sc, incr); 5400 5401 for (i = 0; i < NCCTRL_WIN; ++i) { 5402 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 5403 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 5404 incr[5][i], incr[6][i], incr[7][i]); 5405 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 5406 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 5407 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 5408 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 5409 } 5410 5411 rc = sbuf_finish(sb); 5412 sbuf_delete(sb); 5413 5414 return (rc); 5415 } 5416 5417 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 5418 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 5419 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 5420 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 5421 }; 5422 5423 static int 5424 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 5425 { 5426 struct adapter *sc = arg1; 5427 struct sbuf *sb; 5428 int rc, i, n, qid = arg2; 5429 uint32_t *buf, *p; 5430 char *qtype; 5431 u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5; 5432 5433 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 5434 ("%s: bad qid %d\n", __func__, qid)); 5435 5436 if (qid < CIM_NUM_IBQ) { 5437 /* inbound queue */ 5438 qtype = "IBQ"; 5439 n = 4 * CIM_IBQ_SIZE; 5440 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5441 rc = t4_read_cim_ibq(sc, qid, buf, n); 5442 } else { 5443 /* outbound queue */ 5444 qtype = "OBQ"; 5445 qid -= CIM_NUM_IBQ; 5446 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 5447 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5448 rc = t4_read_cim_obq(sc, qid, buf, n); 5449 } 5450 5451 if (rc < 0) { 5452 rc = -rc; 5453 goto done; 5454 } 5455 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 5456 5457 rc = sysctl_wire_old_buffer(req, 0); 5458 if (rc != 0) 5459 goto done; 5460 5461 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5462 if (sb == NULL) { 5463 rc = ENOMEM; 5464 goto done; 5465 } 5466 5467 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 5468 for (i = 0, p = buf; i < n; i += 16, p += 4) 5469 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 5470 p[2], p[3]); 5471 5472 rc = sbuf_finish(sb); 5473 sbuf_delete(sb); 5474 done: 5475 free(buf, M_CXGBE); 5476 return (rc); 5477 } 5478 5479 static int 5480 sysctl_cim_la(SYSCTL_HANDLER_ARGS) 5481 { 5482 struct adapter *sc = arg1; 5483 u_int cfg; 5484 struct sbuf *sb; 5485 uint32_t *buf, *p; 5486 int rc; 5487 5488 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 5489 if (rc != 0) 5490 return (rc); 5491 5492 rc = sysctl_wire_old_buffer(req, 0); 5493 if (rc != 0) 5494 return (rc); 5495 5496 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5497 if (sb == NULL) 5498 return (ENOMEM); 5499 5500 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 5501 M_ZERO | M_WAITOK); 5502 5503 rc = -t4_cim_read_la(sc, buf, NULL); 5504 if (rc != 0) 5505 goto done; 5506 5507 sbuf_printf(sb, "Status Data PC%s", 5508 cfg & F_UPDBGLACAPTPCONLY ? "" : 5509 " LS0Stat LS0Addr LS0Data"); 5510 5511 KASSERT((sc->params.cim_la_size & 7) == 0, 5512 ("%s: p will walk off the end of buf", __func__)); 5513 5514 for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) { 5515 if (cfg & F_UPDBGLACAPTPCONLY) { 5516 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 5517 p[6], p[7]); 5518 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 5519 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 5520 p[4] & 0xff, p[5] >> 8); 5521 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 5522 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5523 p[1] & 0xf, p[2] >> 4); 5524 } else { 5525 sbuf_printf(sb, 5526 "\n %02x %x%07x %x%07x %08x %08x " 5527 "%08x%08x%08x%08x", 5528 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5529 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 5530 p[6], p[7]); 5531 } 5532 } 5533 5534 rc = sbuf_finish(sb); 5535 sbuf_delete(sb); 5536 done: 5537 free(buf, M_CXGBE); 5538 return (rc); 5539 } 5540 5541 static int 5542 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) 5543 { 5544 struct adapter *sc = arg1; 5545 u_int i; 5546 struct sbuf *sb; 5547 uint32_t *buf, *p; 5548 int rc; 5549 5550 rc = sysctl_wire_old_buffer(req, 0); 5551 if (rc != 0) 5552 return (rc); 5553 5554 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5555 if (sb == NULL) 5556 return (ENOMEM); 5557 5558 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, 5559 M_ZERO | M_WAITOK); 5560 5561 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); 5562 p = buf; 5563 5564 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5565 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], 5566 p[1], p[0]); 5567 } 5568 5569 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); 5570 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5571 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", 5572 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, 5573 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, 5574 (p[1] >> 2) | ((p[2] & 3) << 30), 5575 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, 5576 p[0] & 1); 5577 } 5578 5579 rc = sbuf_finish(sb); 5580 sbuf_delete(sb); 5581 free(buf, M_CXGBE); 5582 return (rc); 5583 } 5584 5585 static int 5586 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) 5587 { 5588 struct adapter *sc = arg1; 5589 u_int i; 5590 struct sbuf *sb; 5591 uint32_t *buf, *p; 5592 int rc; 5593 5594 rc = sysctl_wire_old_buffer(req, 0); 5595 if (rc != 0) 5596 return (rc); 5597 5598 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5599 if (sb == NULL) 5600 return (ENOMEM); 5601 5602 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, 5603 M_ZERO | M_WAITOK); 5604 5605 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); 5606 p = buf; 5607 5608 sbuf_printf(sb, "Cntl ID DataBE Addr Data"); 5609 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5610 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", 5611 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, 5612 p[4], p[3], p[2], p[1], p[0]); 5613 } 5614 5615 sbuf_printf(sb, "\n\nCntl ID Data"); 5616 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5617 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", 5618 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); 5619 } 5620 5621 rc = sbuf_finish(sb); 5622 sbuf_delete(sb); 5623 free(buf, M_CXGBE); 5624 return (rc); 5625 } 5626 5627 static int 5628 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 5629 { 5630 struct adapter *sc = arg1; 5631 struct sbuf *sb; 5632 int rc, i; 5633 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5634 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5635 uint16_t thres[CIM_NUM_IBQ]; 5636 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 5637 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 5638 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 5639 5640 if (is_t4(sc)) { 5641 cim_num_obq = CIM_NUM_OBQ; 5642 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 5643 obq_rdaddr = A_UP_OBQ_0_REALADDR; 5644 } else { 5645 cim_num_obq = CIM_NUM_OBQ_T5; 5646 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 5647 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 5648 } 5649 nq = CIM_NUM_IBQ + cim_num_obq; 5650 5651 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 5652 if (rc == 0) 5653 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 5654 if (rc != 0) 5655 return (rc); 5656 5657 t4_read_cimq_cfg(sc, base, size, thres); 5658 5659 rc = sysctl_wire_old_buffer(req, 0); 5660 if (rc != 0) 5661 return (rc); 5662 5663 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5664 if (sb == NULL) 5665 return (ENOMEM); 5666 5667 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 5668 5669 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 5670 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 5671 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 5672 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5673 G_QUEREMFLITS(p[2]) * 16); 5674 for ( ; i < nq; i++, p += 4, wr += 2) 5675 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 5676 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 5677 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5678 G_QUEREMFLITS(p[2]) * 16); 5679 5680 rc = sbuf_finish(sb); 5681 sbuf_delete(sb); 5682 5683 return (rc); 5684 } 5685 5686 static int 5687 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 5688 { 5689 struct adapter *sc = arg1; 5690 struct sbuf *sb; 5691 int rc; 5692 struct tp_cpl_stats stats; 5693 5694 rc = sysctl_wire_old_buffer(req, 0); 5695 if (rc != 0) 5696 return (rc); 5697 5698 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5699 if (sb == NULL) 5700 return (ENOMEM); 5701 5702 t4_tp_get_cpl_stats(sc, &stats); 5703 5704 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 5705 "channel 3\n"); 5706 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 5707 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 5708 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 5709 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 5710 5711 rc = sbuf_finish(sb); 5712 sbuf_delete(sb); 5713 5714 return (rc); 5715 } 5716 5717 static int 5718 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 5719 { 5720 struct adapter *sc = arg1; 5721 struct sbuf *sb; 5722 int rc; 5723 struct tp_usm_stats stats; 5724 5725 rc = sysctl_wire_old_buffer(req, 0); 5726 if (rc != 0) 5727 return(rc); 5728 5729 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5730 if (sb == NULL) 5731 return (ENOMEM); 5732 5733 t4_get_usm_stats(sc, &stats); 5734 5735 sbuf_printf(sb, "Frames: %u\n", stats.frames); 5736 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 5737 sbuf_printf(sb, "Drops: %u", stats.drops); 5738 5739 rc = sbuf_finish(sb); 5740 sbuf_delete(sb); 5741 5742 return (rc); 5743 } 5744 5745 const char *devlog_level_strings[] = { 5746 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 5747 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 5748 [FW_DEVLOG_LEVEL_ERR] = "ERR", 5749 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 5750 [FW_DEVLOG_LEVEL_INFO] = "INFO", 5751 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 5752 }; 5753 5754 const char *devlog_facility_strings[] = { 5755 [FW_DEVLOG_FACILITY_CORE] = "CORE", 5756 [FW_DEVLOG_FACILITY_CF] = "CF", 5757 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 5758 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 5759 [FW_DEVLOG_FACILITY_RES] = "RES", 5760 [FW_DEVLOG_FACILITY_HW] = "HW", 5761 [FW_DEVLOG_FACILITY_FLR] = "FLR", 5762 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 5763 [FW_DEVLOG_FACILITY_PHY] = "PHY", 5764 [FW_DEVLOG_FACILITY_MAC] = "MAC", 5765 [FW_DEVLOG_FACILITY_PORT] = "PORT", 5766 [FW_DEVLOG_FACILITY_VI] = "VI", 5767 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 5768 [FW_DEVLOG_FACILITY_ACL] = "ACL", 5769 [FW_DEVLOG_FACILITY_TM] = "TM", 5770 [FW_DEVLOG_FACILITY_QFC] = "QFC", 5771 [FW_DEVLOG_FACILITY_DCB] = "DCB", 5772 [FW_DEVLOG_FACILITY_ETH] = "ETH", 5773 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 5774 [FW_DEVLOG_FACILITY_RI] = "RI", 5775 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 5776 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 5777 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 5778 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 5779 }; 5780 5781 static int 5782 sysctl_devlog(SYSCTL_HANDLER_ARGS) 5783 { 5784 struct adapter *sc = arg1; 5785 struct devlog_params *dparams = &sc->params.devlog; 5786 struct fw_devlog_e *buf, *e; 5787 int i, j, rc, nentries, first = 0, m; 5788 struct sbuf *sb; 5789 uint64_t ftstamp = UINT64_MAX; 5790 5791 if (dparams->start == 0) { 5792 dparams->memtype = FW_MEMTYPE_EDC0; 5793 dparams->start = 0x84000; 5794 dparams->size = 32768; 5795 } 5796 5797 nentries = dparams->size / sizeof(struct fw_devlog_e); 5798 5799 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 5800 if (buf == NULL) 5801 return (ENOMEM); 5802 5803 m = fwmtype_to_hwmtype(dparams->memtype); 5804 rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf); 5805 if (rc != 0) 5806 goto done; 5807 5808 for (i = 0; i < nentries; i++) { 5809 e = &buf[i]; 5810 5811 if (e->timestamp == 0) 5812 break; /* end */ 5813 5814 e->timestamp = be64toh(e->timestamp); 5815 e->seqno = be32toh(e->seqno); 5816 for (j = 0; j < 8; j++) 5817 e->params[j] = be32toh(e->params[j]); 5818 5819 if (e->timestamp < ftstamp) { 5820 ftstamp = e->timestamp; 5821 first = i; 5822 } 5823 } 5824 5825 if (buf[first].timestamp == 0) 5826 goto done; /* nothing in the log */ 5827 5828 rc = sysctl_wire_old_buffer(req, 0); 5829 if (rc != 0) 5830 goto done; 5831 5832 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5833 if (sb == NULL) { 5834 rc = ENOMEM; 5835 goto done; 5836 } 5837 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 5838 "Seq#", "Tstamp", "Level", "Facility", "Message"); 5839 5840 i = first; 5841 do { 5842 e = &buf[i]; 5843 if (e->timestamp == 0) 5844 break; /* end */ 5845 5846 sbuf_printf(sb, "%10d %15ju %8s %8s ", 5847 e->seqno, e->timestamp, 5848 (e->level < nitems(devlog_level_strings) ? 5849 devlog_level_strings[e->level] : "UNKNOWN"), 5850 (e->facility < nitems(devlog_facility_strings) ? 5851 devlog_facility_strings[e->facility] : "UNKNOWN")); 5852 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 5853 e->params[2], e->params[3], e->params[4], 5854 e->params[5], e->params[6], e->params[7]); 5855 5856 if (++i == nentries) 5857 i = 0; 5858 } while (i != first); 5859 5860 rc = sbuf_finish(sb); 5861 sbuf_delete(sb); 5862 done: 5863 free(buf, M_CXGBE); 5864 return (rc); 5865 } 5866 5867 static int 5868 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 5869 { 5870 struct adapter *sc = arg1; 5871 struct sbuf *sb; 5872 int rc; 5873 struct tp_fcoe_stats stats[4]; 5874 5875 rc = sysctl_wire_old_buffer(req, 0); 5876 if (rc != 0) 5877 return (rc); 5878 5879 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5880 if (sb == NULL) 5881 return (ENOMEM); 5882 5883 t4_get_fcoe_stats(sc, 0, &stats[0]); 5884 t4_get_fcoe_stats(sc, 1, &stats[1]); 5885 t4_get_fcoe_stats(sc, 2, &stats[2]); 5886 t4_get_fcoe_stats(sc, 3, &stats[3]); 5887 5888 sbuf_printf(sb, " channel 0 channel 1 " 5889 "channel 2 channel 3\n"); 5890 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 5891 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 5892 stats[3].octetsDDP); 5893 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 5894 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 5895 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 5896 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 5897 stats[3].framesDrop); 5898 5899 rc = sbuf_finish(sb); 5900 sbuf_delete(sb); 5901 5902 return (rc); 5903 } 5904 5905 static int 5906 sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 5907 { 5908 struct adapter *sc = arg1; 5909 struct sbuf *sb; 5910 int rc, i; 5911 unsigned int map, kbps, ipg, mode; 5912 unsigned int pace_tab[NTX_SCHED]; 5913 5914 rc = sysctl_wire_old_buffer(req, 0); 5915 if (rc != 0) 5916 return (rc); 5917 5918 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5919 if (sb == NULL) 5920 return (ENOMEM); 5921 5922 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 5923 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 5924 t4_read_pace_tbl(sc, pace_tab); 5925 5926 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 5927 "Class IPG (0.1 ns) Flow IPG (us)"); 5928 5929 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 5930 t4_get_tx_sched(sc, i, &kbps, &ipg); 5931 sbuf_printf(sb, "\n %u %-5s %u ", i, 5932 (mode & (1 << i)) ? "flow" : "class", map & 3); 5933 if (kbps) 5934 sbuf_printf(sb, "%9u ", kbps); 5935 else 5936 sbuf_printf(sb, " disabled "); 5937 5938 if (ipg) 5939 sbuf_printf(sb, "%13u ", ipg); 5940 else 5941 sbuf_printf(sb, " disabled "); 5942 5943 if (pace_tab[i]) 5944 sbuf_printf(sb, "%10u", pace_tab[i]); 5945 else 5946 sbuf_printf(sb, " disabled"); 5947 } 5948 5949 rc = sbuf_finish(sb); 5950 sbuf_delete(sb); 5951 5952 return (rc); 5953 } 5954 5955 static int 5956 sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 5957 { 5958 struct adapter *sc = arg1; 5959 struct sbuf *sb; 5960 int rc, i, j; 5961 uint64_t *p0, *p1; 5962 struct lb_port_stats s[2]; 5963 static const char *stat_name[] = { 5964 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 5965 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 5966 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 5967 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 5968 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 5969 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 5970 "BG2FramesTrunc:", "BG3FramesTrunc:" 5971 }; 5972 5973 rc = sysctl_wire_old_buffer(req, 0); 5974 if (rc != 0) 5975 return (rc); 5976 5977 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5978 if (sb == NULL) 5979 return (ENOMEM); 5980 5981 memset(s, 0, sizeof(s)); 5982 5983 for (i = 0; i < 4; i += 2) { 5984 t4_get_lb_stats(sc, i, &s[0]); 5985 t4_get_lb_stats(sc, i + 1, &s[1]); 5986 5987 p0 = &s[0].octets; 5988 p1 = &s[1].octets; 5989 sbuf_printf(sb, "%s Loopback %u" 5990 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 5991 5992 for (j = 0; j < nitems(stat_name); j++) 5993 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 5994 *p0++, *p1++); 5995 } 5996 5997 rc = sbuf_finish(sb); 5998 sbuf_delete(sb); 5999 6000 return (rc); 6001 } 6002 6003 static int 6004 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) 6005 { 6006 int rc = 0; 6007 struct port_info *pi = arg1; 6008 struct sbuf *sb; 6009 static const char *linkdnreasons[] = { 6010 "non-specific", "remote fault", "autoneg failed", "reserved3", 6011 "PHY overheated", "unknown", "rx los", "reserved7" 6012 }; 6013 6014 rc = sysctl_wire_old_buffer(req, 0); 6015 if (rc != 0) 6016 return(rc); 6017 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); 6018 if (sb == NULL) 6019 return (ENOMEM); 6020 6021 if (pi->linkdnrc < 0) 6022 sbuf_printf(sb, "n/a"); 6023 else if (pi->linkdnrc < nitems(linkdnreasons)) 6024 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]); 6025 else 6026 sbuf_printf(sb, "%d", pi->linkdnrc); 6027 6028 rc = sbuf_finish(sb); 6029 sbuf_delete(sb); 6030 6031 return (rc); 6032 } 6033 6034 struct mem_desc { 6035 unsigned int base; 6036 unsigned int limit; 6037 unsigned int idx; 6038 }; 6039 6040 static int 6041 mem_desc_cmp(const void *a, const void *b) 6042 { 6043 return ((const struct mem_desc *)a)->base - 6044 ((const struct mem_desc *)b)->base; 6045 } 6046 6047 static void 6048 mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 6049 unsigned int to) 6050 { 6051 unsigned int size; 6052 6053 size = to - from + 1; 6054 if (size == 0) 6055 return; 6056 6057 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 6058 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 6059 } 6060 6061 static int 6062 sysctl_meminfo(SYSCTL_HANDLER_ARGS) 6063 { 6064 struct adapter *sc = arg1; 6065 struct sbuf *sb; 6066 int rc, i, n; 6067 uint32_t lo, hi, used, alloc; 6068 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 6069 static const char *region[] = { 6070 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 6071 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 6072 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 6073 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 6074 "RQUDP region:", "PBL region:", "TXPBL region:", 6075 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 6076 "On-chip queues:" 6077 }; 6078 struct mem_desc avail[4]; 6079 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 6080 struct mem_desc *md = mem; 6081 6082 rc = sysctl_wire_old_buffer(req, 0); 6083 if (rc != 0) 6084 return (rc); 6085 6086 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6087 if (sb == NULL) 6088 return (ENOMEM); 6089 6090 for (i = 0; i < nitems(mem); i++) { 6091 mem[i].limit = 0; 6092 mem[i].idx = i; 6093 } 6094 6095 /* Find and sort the populated memory ranges */ 6096 i = 0; 6097 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 6098 if (lo & F_EDRAM0_ENABLE) { 6099 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 6100 avail[i].base = G_EDRAM0_BASE(hi) << 20; 6101 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 6102 avail[i].idx = 0; 6103 i++; 6104 } 6105 if (lo & F_EDRAM1_ENABLE) { 6106 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 6107 avail[i].base = G_EDRAM1_BASE(hi) << 20; 6108 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 6109 avail[i].idx = 1; 6110 i++; 6111 } 6112 if (lo & F_EXT_MEM_ENABLE) { 6113 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 6114 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 6115 avail[i].limit = avail[i].base + 6116 (G_EXT_MEM_SIZE(hi) << 20); 6117 avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */ 6118 i++; 6119 } 6120 if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) { 6121 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 6122 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 6123 avail[i].limit = avail[i].base + 6124 (G_EXT_MEM1_SIZE(hi) << 20); 6125 avail[i].idx = 4; 6126 i++; 6127 } 6128 if (!i) /* no memory available */ 6129 return 0; 6130 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 6131 6132 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 6133 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 6134 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 6135 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 6136 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 6137 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 6138 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 6139 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 6140 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 6141 6142 /* the next few have explicit upper bounds */ 6143 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 6144 md->limit = md->base - 1 + 6145 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 6146 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 6147 md++; 6148 6149 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 6150 md->limit = md->base - 1 + 6151 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 6152 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 6153 md++; 6154 6155 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6156 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 6157 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 6158 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 6159 } else { 6160 md->base = 0; 6161 md->idx = nitems(region); /* hide it */ 6162 } 6163 md++; 6164 6165 #define ulp_region(reg) \ 6166 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 6167 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 6168 6169 ulp_region(RX_ISCSI); 6170 ulp_region(RX_TDDP); 6171 ulp_region(TX_TPT); 6172 ulp_region(RX_STAG); 6173 ulp_region(RX_RQ); 6174 ulp_region(RX_RQUDP); 6175 ulp_region(RX_PBL); 6176 ulp_region(TX_PBL); 6177 #undef ulp_region 6178 6179 md->base = 0; 6180 md->idx = nitems(region); 6181 if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) { 6182 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); 6183 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc, 6184 A_SGE_DBVFIFO_SIZE))) << 2) - 1; 6185 } 6186 md++; 6187 6188 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 6189 md->limit = md->base + sc->tids.ntids - 1; 6190 md++; 6191 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 6192 md->limit = md->base + sc->tids.ntids - 1; 6193 md++; 6194 6195 md->base = sc->vres.ocq.start; 6196 if (sc->vres.ocq.size) 6197 md->limit = md->base + sc->vres.ocq.size - 1; 6198 else 6199 md->idx = nitems(region); /* hide it */ 6200 md++; 6201 6202 /* add any address-space holes, there can be up to 3 */ 6203 for (n = 0; n < i - 1; n++) 6204 if (avail[n].limit < avail[n + 1].base) 6205 (md++)->base = avail[n].limit; 6206 if (avail[n].limit) 6207 (md++)->base = avail[n].limit; 6208 6209 n = md - mem; 6210 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 6211 6212 for (lo = 0; lo < i; lo++) 6213 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 6214 avail[lo].limit - 1); 6215 6216 sbuf_printf(sb, "\n"); 6217 for (i = 0; i < n; i++) { 6218 if (mem[i].idx >= nitems(region)) 6219 continue; /* skip holes */ 6220 if (!mem[i].limit) 6221 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 6222 mem_region_show(sb, region[mem[i].idx], mem[i].base, 6223 mem[i].limit); 6224 } 6225 6226 sbuf_printf(sb, "\n"); 6227 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 6228 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 6229 mem_region_show(sb, "uP RAM:", lo, hi); 6230 6231 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 6232 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 6233 mem_region_show(sb, "uP Extmem2:", lo, hi); 6234 6235 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 6236 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 6237 G_PMRXMAXPAGE(lo), 6238 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 6239 (lo & F_PMRXNUMCHN) ? 2 : 1); 6240 6241 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 6242 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 6243 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 6244 G_PMTXMAXPAGE(lo), 6245 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 6246 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 6247 sbuf_printf(sb, "%u p-structs\n", 6248 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 6249 6250 for (i = 0; i < 4; i++) { 6251 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 6252 if (is_t4(sc)) { 6253 used = G_USED(lo); 6254 alloc = G_ALLOC(lo); 6255 } else { 6256 used = G_T5_USED(lo); 6257 alloc = G_T5_ALLOC(lo); 6258 } 6259 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 6260 i, used, alloc); 6261 } 6262 for (i = 0; i < 4; i++) { 6263 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 6264 if (is_t4(sc)) { 6265 used = G_USED(lo); 6266 alloc = G_ALLOC(lo); 6267 } else { 6268 used = G_T5_USED(lo); 6269 alloc = G_T5_ALLOC(lo); 6270 } 6271 sbuf_printf(sb, 6272 "\nLoopback %d using %u pages out of %u allocated", 6273 i, used, alloc); 6274 } 6275 6276 rc = sbuf_finish(sb); 6277 sbuf_delete(sb); 6278 6279 return (rc); 6280 } 6281 6282 static inline void 6283 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) 6284 { 6285 *mask = x | y; 6286 y = htobe64(y); 6287 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); 6288 } 6289 6290 static int 6291 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) 6292 { 6293 struct adapter *sc = arg1; 6294 struct sbuf *sb; 6295 int rc, i, n; 6296 6297 rc = sysctl_wire_old_buffer(req, 0); 6298 if (rc != 0) 6299 return (rc); 6300 6301 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6302 if (sb == NULL) 6303 return (ENOMEM); 6304 6305 sbuf_printf(sb, 6306 "Idx Ethernet address Mask Vld Ports PF" 6307 " VF Replication P0 P1 P2 P3 ML"); 6308 n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES : 6309 NUM_MPS_T5_CLS_SRAM_L_INSTANCES; 6310 for (i = 0; i < n; i++) { 6311 uint64_t tcamx, tcamy, mask; 6312 uint32_t cls_lo, cls_hi; 6313 uint8_t addr[ETHER_ADDR_LEN]; 6314 6315 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); 6316 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); 6317 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 6318 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 6319 6320 if (tcamx & tcamy) 6321 continue; 6322 6323 tcamxy2valmask(tcamx, tcamy, addr, &mask); 6324 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" 6325 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], 6326 addr[3], addr[4], addr[5], (uintmax_t)mask, 6327 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', 6328 G_PORTMAP(cls_hi), G_PF(cls_lo), 6329 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); 6330 6331 if (cls_lo & F_REPLICATE) { 6332 struct fw_ldst_cmd ldst_cmd; 6333 6334 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 6335 ldst_cmd.op_to_addrspace = 6336 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 6337 F_FW_CMD_REQUEST | F_FW_CMD_READ | 6338 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 6339 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 6340 ldst_cmd.u.mps.fid_ctl = 6341 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 6342 V_FW_LDST_CMD_CTL(i)); 6343 6344 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 6345 "t4mps"); 6346 if (rc) 6347 break; 6348 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 6349 sizeof(ldst_cmd), &ldst_cmd); 6350 end_synchronized_op(sc, 0); 6351 6352 if (rc != 0) { 6353 sbuf_printf(sb, 6354 " ------------ error %3u ------------", rc); 6355 rc = 0; 6356 } else { 6357 sbuf_printf(sb, " %08x %08x %08x %08x", 6358 be32toh(ldst_cmd.u.mps.rplc127_96), 6359 be32toh(ldst_cmd.u.mps.rplc95_64), 6360 be32toh(ldst_cmd.u.mps.rplc63_32), 6361 be32toh(ldst_cmd.u.mps.rplc31_0)); 6362 } 6363 } else 6364 sbuf_printf(sb, "%36s", ""); 6365 6366 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), 6367 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), 6368 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); 6369 } 6370 6371 if (rc) 6372 (void) sbuf_finish(sb); 6373 else 6374 rc = sbuf_finish(sb); 6375 sbuf_delete(sb); 6376 6377 return (rc); 6378 } 6379 6380 static int 6381 sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 6382 { 6383 struct adapter *sc = arg1; 6384 struct sbuf *sb; 6385 int rc; 6386 uint16_t mtus[NMTUS]; 6387 6388 rc = sysctl_wire_old_buffer(req, 0); 6389 if (rc != 0) 6390 return (rc); 6391 6392 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6393 if (sb == NULL) 6394 return (ENOMEM); 6395 6396 t4_read_mtu_tbl(sc, mtus, NULL); 6397 6398 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 6399 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 6400 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 6401 mtus[14], mtus[15]); 6402 6403 rc = sbuf_finish(sb); 6404 sbuf_delete(sb); 6405 6406 return (rc); 6407 } 6408 6409 static int 6410 sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 6411 { 6412 struct adapter *sc = arg1; 6413 struct sbuf *sb; 6414 int rc, i; 6415 uint32_t cnt[PM_NSTATS]; 6416 uint64_t cyc[PM_NSTATS]; 6417 static const char *rx_stats[] = { 6418 "Read:", "Write bypass:", "Write mem:", "Flush:" 6419 }; 6420 static const char *tx_stats[] = { 6421 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:" 6422 }; 6423 6424 rc = sysctl_wire_old_buffer(req, 0); 6425 if (rc != 0) 6426 return (rc); 6427 6428 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6429 if (sb == NULL) 6430 return (ENOMEM); 6431 6432 t4_pmtx_get_stats(sc, cnt, cyc); 6433 sbuf_printf(sb, " Tx pcmds Tx bytes"); 6434 for (i = 0; i < ARRAY_SIZE(tx_stats); i++) 6435 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i], 6436 cyc[i]); 6437 6438 t4_pmrx_get_stats(sc, cnt, cyc); 6439 sbuf_printf(sb, "\n Rx pcmds Rx bytes"); 6440 for (i = 0; i < ARRAY_SIZE(rx_stats); i++) 6441 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i], 6442 cyc[i]); 6443 6444 rc = sbuf_finish(sb); 6445 sbuf_delete(sb); 6446 6447 return (rc); 6448 } 6449 6450 static int 6451 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 6452 { 6453 struct adapter *sc = arg1; 6454 struct sbuf *sb; 6455 int rc; 6456 struct tp_rdma_stats stats; 6457 6458 rc = sysctl_wire_old_buffer(req, 0); 6459 if (rc != 0) 6460 return (rc); 6461 6462 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6463 if (sb == NULL) 6464 return (ENOMEM); 6465 6466 t4_tp_get_rdma_stats(sc, &stats); 6467 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 6468 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 6469 6470 rc = sbuf_finish(sb); 6471 sbuf_delete(sb); 6472 6473 return (rc); 6474 } 6475 6476 static int 6477 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 6478 { 6479 struct adapter *sc = arg1; 6480 struct sbuf *sb; 6481 int rc; 6482 struct tp_tcp_stats v4, v6; 6483 6484 rc = sysctl_wire_old_buffer(req, 0); 6485 if (rc != 0) 6486 return (rc); 6487 6488 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6489 if (sb == NULL) 6490 return (ENOMEM); 6491 6492 t4_tp_get_tcp_stats(sc, &v4, &v6); 6493 sbuf_printf(sb, 6494 " IP IPv6\n"); 6495 sbuf_printf(sb, "OutRsts: %20u %20u\n", 6496 v4.tcpOutRsts, v6.tcpOutRsts); 6497 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 6498 v4.tcpInSegs, v6.tcpInSegs); 6499 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 6500 v4.tcpOutSegs, v6.tcpOutSegs); 6501 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 6502 v4.tcpRetransSegs, v6.tcpRetransSegs); 6503 6504 rc = sbuf_finish(sb); 6505 sbuf_delete(sb); 6506 6507 return (rc); 6508 } 6509 6510 static int 6511 sysctl_tids(SYSCTL_HANDLER_ARGS) 6512 { 6513 struct adapter *sc = arg1; 6514 struct sbuf *sb; 6515 int rc; 6516 struct tid_info *t = &sc->tids; 6517 6518 rc = sysctl_wire_old_buffer(req, 0); 6519 if (rc != 0) 6520 return (rc); 6521 6522 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6523 if (sb == NULL) 6524 return (ENOMEM); 6525 6526 if (t->natids) { 6527 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 6528 t->atids_in_use); 6529 } 6530 6531 if (t->ntids) { 6532 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6533 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 6534 6535 if (b) { 6536 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 6537 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6538 t->ntids - 1); 6539 } else { 6540 sbuf_printf(sb, "TID range: %u-%u", 6541 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6542 t->ntids - 1); 6543 } 6544 } else 6545 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 6546 sbuf_printf(sb, ", in use: %u\n", 6547 atomic_load_acq_int(&t->tids_in_use)); 6548 } 6549 6550 if (t->nstids) { 6551 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 6552 t->stid_base + t->nstids - 1, t->stids_in_use); 6553 } 6554 6555 if (t->nftids) { 6556 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 6557 t->ftid_base + t->nftids - 1); 6558 } 6559 6560 if (t->netids) { 6561 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base, 6562 t->etid_base + t->netids - 1); 6563 } 6564 6565 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 6566 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 6567 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 6568 6569 rc = sbuf_finish(sb); 6570 sbuf_delete(sb); 6571 6572 return (rc); 6573 } 6574 6575 static int 6576 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 6577 { 6578 struct adapter *sc = arg1; 6579 struct sbuf *sb; 6580 int rc; 6581 struct tp_err_stats stats; 6582 6583 rc = sysctl_wire_old_buffer(req, 0); 6584 if (rc != 0) 6585 return (rc); 6586 6587 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6588 if (sb == NULL) 6589 return (ENOMEM); 6590 6591 t4_tp_get_err_stats(sc, &stats); 6592 6593 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6594 "channel 3\n"); 6595 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 6596 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 6597 stats.macInErrs[3]); 6598 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 6599 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 6600 stats.hdrInErrs[3]); 6601 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 6602 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 6603 stats.tcpInErrs[3]); 6604 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 6605 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 6606 stats.tcp6InErrs[3]); 6607 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 6608 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 6609 stats.tnlCongDrops[3]); 6610 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 6611 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 6612 stats.tnlTxDrops[3]); 6613 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 6614 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 6615 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 6616 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 6617 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 6618 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 6619 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 6620 stats.ofldNoNeigh, stats.ofldCongDefer); 6621 6622 rc = sbuf_finish(sb); 6623 sbuf_delete(sb); 6624 6625 return (rc); 6626 } 6627 6628 struct field_desc { 6629 const char *name; 6630 u_int start; 6631 u_int width; 6632 }; 6633 6634 static void 6635 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) 6636 { 6637 char buf[32]; 6638 int line_size = 0; 6639 6640 while (f->name) { 6641 uint64_t mask = (1ULL << f->width) - 1; 6642 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, 6643 ((uintmax_t)v >> f->start) & mask); 6644 6645 if (line_size + len >= 79) { 6646 line_size = 8; 6647 sbuf_printf(sb, "\n "); 6648 } 6649 sbuf_printf(sb, "%s ", buf); 6650 line_size += len + 1; 6651 f++; 6652 } 6653 sbuf_printf(sb, "\n"); 6654 } 6655 6656 static struct field_desc tp_la0[] = { 6657 { "RcfOpCodeOut", 60, 4 }, 6658 { "State", 56, 4 }, 6659 { "WcfState", 52, 4 }, 6660 { "RcfOpcSrcOut", 50, 2 }, 6661 { "CRxError", 49, 1 }, 6662 { "ERxError", 48, 1 }, 6663 { "SanityFailed", 47, 1 }, 6664 { "SpuriousMsg", 46, 1 }, 6665 { "FlushInputMsg", 45, 1 }, 6666 { "FlushInputCpl", 44, 1 }, 6667 { "RssUpBit", 43, 1 }, 6668 { "RssFilterHit", 42, 1 }, 6669 { "Tid", 32, 10 }, 6670 { "InitTcb", 31, 1 }, 6671 { "LineNumber", 24, 7 }, 6672 { "Emsg", 23, 1 }, 6673 { "EdataOut", 22, 1 }, 6674 { "Cmsg", 21, 1 }, 6675 { "CdataOut", 20, 1 }, 6676 { "EreadPdu", 19, 1 }, 6677 { "CreadPdu", 18, 1 }, 6678 { "TunnelPkt", 17, 1 }, 6679 { "RcfPeerFin", 16, 1 }, 6680 { "RcfReasonOut", 12, 4 }, 6681 { "TxCchannel", 10, 2 }, 6682 { "RcfTxChannel", 8, 2 }, 6683 { "RxEchannel", 6, 2 }, 6684 { "RcfRxChannel", 5, 1 }, 6685 { "RcfDataOutSrdy", 4, 1 }, 6686 { "RxDvld", 3, 1 }, 6687 { "RxOoDvld", 2, 1 }, 6688 { "RxCongestion", 1, 1 }, 6689 { "TxCongestion", 0, 1 }, 6690 { NULL } 6691 }; 6692 6693 static struct field_desc tp_la1[] = { 6694 { "CplCmdIn", 56, 8 }, 6695 { "CplCmdOut", 48, 8 }, 6696 { "ESynOut", 47, 1 }, 6697 { "EAckOut", 46, 1 }, 6698 { "EFinOut", 45, 1 }, 6699 { "ERstOut", 44, 1 }, 6700 { "SynIn", 43, 1 }, 6701 { "AckIn", 42, 1 }, 6702 { "FinIn", 41, 1 }, 6703 { "RstIn", 40, 1 }, 6704 { "DataIn", 39, 1 }, 6705 { "DataInVld", 38, 1 }, 6706 { "PadIn", 37, 1 }, 6707 { "RxBufEmpty", 36, 1 }, 6708 { "RxDdp", 35, 1 }, 6709 { "RxFbCongestion", 34, 1 }, 6710 { "TxFbCongestion", 33, 1 }, 6711 { "TxPktSumSrdy", 32, 1 }, 6712 { "RcfUlpType", 28, 4 }, 6713 { "Eread", 27, 1 }, 6714 { "Ebypass", 26, 1 }, 6715 { "Esave", 25, 1 }, 6716 { "Static0", 24, 1 }, 6717 { "Cread", 23, 1 }, 6718 { "Cbypass", 22, 1 }, 6719 { "Csave", 21, 1 }, 6720 { "CPktOut", 20, 1 }, 6721 { "RxPagePoolFull", 18, 2 }, 6722 { "RxLpbkPkt", 17, 1 }, 6723 { "TxLpbkPkt", 16, 1 }, 6724 { "RxVfValid", 15, 1 }, 6725 { "SynLearned", 14, 1 }, 6726 { "SetDelEntry", 13, 1 }, 6727 { "SetInvEntry", 12, 1 }, 6728 { "CpcmdDvld", 11, 1 }, 6729 { "CpcmdSave", 10, 1 }, 6730 { "RxPstructsFull", 8, 2 }, 6731 { "EpcmdDvld", 7, 1 }, 6732 { "EpcmdFlush", 6, 1 }, 6733 { "EpcmdTrimPrefix", 5, 1 }, 6734 { "EpcmdTrimPostfix", 4, 1 }, 6735 { "ERssIp4Pkt", 3, 1 }, 6736 { "ERssIp6Pkt", 2, 1 }, 6737 { "ERssTcpUdpPkt", 1, 1 }, 6738 { "ERssFceFipPkt", 0, 1 }, 6739 { NULL } 6740 }; 6741 6742 static struct field_desc tp_la2[] = { 6743 { "CplCmdIn", 56, 8 }, 6744 { "MpsVfVld", 55, 1 }, 6745 { "MpsPf", 52, 3 }, 6746 { "MpsVf", 44, 8 }, 6747 { "SynIn", 43, 1 }, 6748 { "AckIn", 42, 1 }, 6749 { "FinIn", 41, 1 }, 6750 { "RstIn", 40, 1 }, 6751 { "DataIn", 39, 1 }, 6752 { "DataInVld", 38, 1 }, 6753 { "PadIn", 37, 1 }, 6754 { "RxBufEmpty", 36, 1 }, 6755 { "RxDdp", 35, 1 }, 6756 { "RxFbCongestion", 34, 1 }, 6757 { "TxFbCongestion", 33, 1 }, 6758 { "TxPktSumSrdy", 32, 1 }, 6759 { "RcfUlpType", 28, 4 }, 6760 { "Eread", 27, 1 }, 6761 { "Ebypass", 26, 1 }, 6762 { "Esave", 25, 1 }, 6763 { "Static0", 24, 1 }, 6764 { "Cread", 23, 1 }, 6765 { "Cbypass", 22, 1 }, 6766 { "Csave", 21, 1 }, 6767 { "CPktOut", 20, 1 }, 6768 { "RxPagePoolFull", 18, 2 }, 6769 { "RxLpbkPkt", 17, 1 }, 6770 { "TxLpbkPkt", 16, 1 }, 6771 { "RxVfValid", 15, 1 }, 6772 { "SynLearned", 14, 1 }, 6773 { "SetDelEntry", 13, 1 }, 6774 { "SetInvEntry", 12, 1 }, 6775 { "CpcmdDvld", 11, 1 }, 6776 { "CpcmdSave", 10, 1 }, 6777 { "RxPstructsFull", 8, 2 }, 6778 { "EpcmdDvld", 7, 1 }, 6779 { "EpcmdFlush", 6, 1 }, 6780 { "EpcmdTrimPrefix", 5, 1 }, 6781 { "EpcmdTrimPostfix", 4, 1 }, 6782 { "ERssIp4Pkt", 3, 1 }, 6783 { "ERssIp6Pkt", 2, 1 }, 6784 { "ERssTcpUdpPkt", 1, 1 }, 6785 { "ERssFceFipPkt", 0, 1 }, 6786 { NULL } 6787 }; 6788 6789 static void 6790 tp_la_show(struct sbuf *sb, uint64_t *p, int idx) 6791 { 6792 6793 field_desc_show(sb, *p, tp_la0); 6794 } 6795 6796 static void 6797 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) 6798 { 6799 6800 if (idx) 6801 sbuf_printf(sb, "\n"); 6802 field_desc_show(sb, p[0], tp_la0); 6803 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6804 field_desc_show(sb, p[1], tp_la0); 6805 } 6806 6807 static void 6808 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) 6809 { 6810 6811 if (idx) 6812 sbuf_printf(sb, "\n"); 6813 field_desc_show(sb, p[0], tp_la0); 6814 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6815 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); 6816 } 6817 6818 static int 6819 sysctl_tp_la(SYSCTL_HANDLER_ARGS) 6820 { 6821 struct adapter *sc = arg1; 6822 struct sbuf *sb; 6823 uint64_t *buf, *p; 6824 int rc; 6825 u_int i, inc; 6826 void (*show_func)(struct sbuf *, uint64_t *, int); 6827 6828 rc = sysctl_wire_old_buffer(req, 0); 6829 if (rc != 0) 6830 return (rc); 6831 6832 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6833 if (sb == NULL) 6834 return (ENOMEM); 6835 6836 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); 6837 6838 t4_tp_read_la(sc, buf, NULL); 6839 p = buf; 6840 6841 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { 6842 case 2: 6843 inc = 2; 6844 show_func = tp_la_show2; 6845 break; 6846 case 3: 6847 inc = 2; 6848 show_func = tp_la_show3; 6849 break; 6850 default: 6851 inc = 1; 6852 show_func = tp_la_show; 6853 } 6854 6855 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) 6856 (*show_func)(sb, p, i); 6857 6858 rc = sbuf_finish(sb); 6859 sbuf_delete(sb); 6860 free(buf, M_CXGBE); 6861 return (rc); 6862 } 6863 6864 static int 6865 sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 6866 { 6867 struct adapter *sc = arg1; 6868 struct sbuf *sb; 6869 int rc; 6870 u64 nrate[NCHAN], orate[NCHAN]; 6871 6872 rc = sysctl_wire_old_buffer(req, 0); 6873 if (rc != 0) 6874 return (rc); 6875 6876 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6877 if (sb == NULL) 6878 return (ENOMEM); 6879 6880 t4_get_chan_txrate(sc, nrate, orate); 6881 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6882 "channel 3\n"); 6883 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 6884 nrate[0], nrate[1], nrate[2], nrate[3]); 6885 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 6886 orate[0], orate[1], orate[2], orate[3]); 6887 6888 rc = sbuf_finish(sb); 6889 sbuf_delete(sb); 6890 6891 return (rc); 6892 } 6893 6894 static int 6895 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) 6896 { 6897 struct adapter *sc = arg1; 6898 struct sbuf *sb; 6899 uint32_t *buf, *p; 6900 int rc, i; 6901 6902 rc = sysctl_wire_old_buffer(req, 0); 6903 if (rc != 0) 6904 return (rc); 6905 6906 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6907 if (sb == NULL) 6908 return (ENOMEM); 6909 6910 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, 6911 M_ZERO | M_WAITOK); 6912 6913 t4_ulprx_read_la(sc, buf); 6914 p = buf; 6915 6916 sbuf_printf(sb, " Pcmd Type Message" 6917 " Data"); 6918 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { 6919 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", 6920 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); 6921 } 6922 6923 rc = sbuf_finish(sb); 6924 sbuf_delete(sb); 6925 free(buf, M_CXGBE); 6926 return (rc); 6927 } 6928 6929 static int 6930 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 6931 { 6932 struct adapter *sc = arg1; 6933 struct sbuf *sb; 6934 int rc, v; 6935 6936 rc = sysctl_wire_old_buffer(req, 0); 6937 if (rc != 0) 6938 return (rc); 6939 6940 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6941 if (sb == NULL) 6942 return (ENOMEM); 6943 6944 v = t4_read_reg(sc, A_SGE_STAT_CFG); 6945 if (G_STATSOURCE_T5(v) == 7) { 6946 if (G_STATMODE(v) == 0) { 6947 sbuf_printf(sb, "total %d, incomplete %d", 6948 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6949 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6950 } else if (G_STATMODE(v) == 1) { 6951 sbuf_printf(sb, "total %d, data overflow %d", 6952 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6953 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6954 } 6955 } 6956 rc = sbuf_finish(sb); 6957 sbuf_delete(sb); 6958 6959 return (rc); 6960 } 6961 #endif 6962 6963 static uint32_t 6964 fconf_to_mode(uint32_t fconf) 6965 { 6966 uint32_t mode; 6967 6968 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 6969 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 6970 6971 if (fconf & F_FRAGMENTATION) 6972 mode |= T4_FILTER_IP_FRAGMENT; 6973 6974 if (fconf & F_MPSHITTYPE) 6975 mode |= T4_FILTER_MPS_HIT_TYPE; 6976 6977 if (fconf & F_MACMATCH) 6978 mode |= T4_FILTER_MAC_IDX; 6979 6980 if (fconf & F_ETHERTYPE) 6981 mode |= T4_FILTER_ETH_TYPE; 6982 6983 if (fconf & F_PROTOCOL) 6984 mode |= T4_FILTER_IP_PROTO; 6985 6986 if (fconf & F_TOS) 6987 mode |= T4_FILTER_IP_TOS; 6988 6989 if (fconf & F_VLAN) 6990 mode |= T4_FILTER_VLAN; 6991 6992 if (fconf & F_VNIC_ID) 6993 mode |= T4_FILTER_VNIC; 6994 6995 if (fconf & F_PORT) 6996 mode |= T4_FILTER_PORT; 6997 6998 if (fconf & F_FCOE) 6999 mode |= T4_FILTER_FCoE; 7000 7001 return (mode); 7002 } 7003 7004 static uint32_t 7005 mode_to_fconf(uint32_t mode) 7006 { 7007 uint32_t fconf = 0; 7008 7009 if (mode & T4_FILTER_IP_FRAGMENT) 7010 fconf |= F_FRAGMENTATION; 7011 7012 if (mode & T4_FILTER_MPS_HIT_TYPE) 7013 fconf |= F_MPSHITTYPE; 7014 7015 if (mode & T4_FILTER_MAC_IDX) 7016 fconf |= F_MACMATCH; 7017 7018 if (mode & T4_FILTER_ETH_TYPE) 7019 fconf |= F_ETHERTYPE; 7020 7021 if (mode & T4_FILTER_IP_PROTO) 7022 fconf |= F_PROTOCOL; 7023 7024 if (mode & T4_FILTER_IP_TOS) 7025 fconf |= F_TOS; 7026 7027 if (mode & T4_FILTER_VLAN) 7028 fconf |= F_VLAN; 7029 7030 if (mode & T4_FILTER_VNIC) 7031 fconf |= F_VNIC_ID; 7032 7033 if (mode & T4_FILTER_PORT) 7034 fconf |= F_PORT; 7035 7036 if (mode & T4_FILTER_FCoE) 7037 fconf |= F_FCOE; 7038 7039 return (fconf); 7040 } 7041 7042 static uint32_t 7043 fspec_to_fconf(struct t4_filter_specification *fs) 7044 { 7045 uint32_t fconf = 0; 7046 7047 if (fs->val.frag || fs->mask.frag) 7048 fconf |= F_FRAGMENTATION; 7049 7050 if (fs->val.matchtype || fs->mask.matchtype) 7051 fconf |= F_MPSHITTYPE; 7052 7053 if (fs->val.macidx || fs->mask.macidx) 7054 fconf |= F_MACMATCH; 7055 7056 if (fs->val.ethtype || fs->mask.ethtype) 7057 fconf |= F_ETHERTYPE; 7058 7059 if (fs->val.proto || fs->mask.proto) 7060 fconf |= F_PROTOCOL; 7061 7062 if (fs->val.tos || fs->mask.tos) 7063 fconf |= F_TOS; 7064 7065 if (fs->val.vlan_vld || fs->mask.vlan_vld) 7066 fconf |= F_VLAN; 7067 7068 if (fs->val.vnic_vld || fs->mask.vnic_vld) 7069 fconf |= F_VNIC_ID; 7070 7071 if (fs->val.iport || fs->mask.iport) 7072 fconf |= F_PORT; 7073 7074 if (fs->val.fcoe || fs->mask.fcoe) 7075 fconf |= F_FCOE; 7076 7077 return (fconf); 7078 } 7079 7080 static int 7081 get_filter_mode(struct adapter *sc, uint32_t *mode) 7082 { 7083 int rc; 7084 uint32_t fconf; 7085 7086 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7087 "t4getfm"); 7088 if (rc) 7089 return (rc); 7090 7091 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 7092 A_TP_VLAN_PRI_MAP); 7093 7094 if (sc->params.tp.vlan_pri_map != fconf) { 7095 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 7096 device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map, 7097 fconf); 7098 sc->params.tp.vlan_pri_map = fconf; 7099 } 7100 7101 *mode = fconf_to_mode(sc->params.tp.vlan_pri_map); 7102 7103 end_synchronized_op(sc, LOCK_HELD); 7104 return (0); 7105 } 7106 7107 static int 7108 set_filter_mode(struct adapter *sc, uint32_t mode) 7109 { 7110 uint32_t fconf; 7111 int rc; 7112 7113 fconf = mode_to_fconf(mode); 7114 7115 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7116 "t4setfm"); 7117 if (rc) 7118 return (rc); 7119 7120 if (sc->tids.ftids_in_use > 0) { 7121 rc = EBUSY; 7122 goto done; 7123 } 7124 7125 #ifdef TCP_OFFLOAD 7126 if (sc->offload_map) { 7127 rc = EBUSY; 7128 goto done; 7129 } 7130 #endif 7131 7132 #ifdef notyet 7133 rc = -t4_set_filter_mode(sc, fconf); 7134 if (rc == 0) 7135 sc->filter_mode = fconf; 7136 #else 7137 rc = ENOTSUP; 7138 #endif 7139 7140 done: 7141 end_synchronized_op(sc, LOCK_HELD); 7142 return (rc); 7143 } 7144 7145 static inline uint64_t 7146 get_filter_hits(struct adapter *sc, uint32_t fid) 7147 { 7148 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 7149 uint64_t hits; 7150 7151 memwin_info(sc, 0, &mw_base, NULL); 7152 off = position_memwin(sc, 0, 7153 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 7154 if (is_t4(sc)) { 7155 hits = t4_read_reg64(sc, mw_base + off + 16); 7156 hits = be64toh(hits); 7157 } else { 7158 hits = t4_read_reg(sc, mw_base + off + 24); 7159 hits = be32toh(hits); 7160 } 7161 7162 return (hits); 7163 } 7164 7165 static int 7166 get_filter(struct adapter *sc, struct t4_filter *t) 7167 { 7168 int i, rc, nfilters = sc->tids.nftids; 7169 struct filter_entry *f; 7170 7171 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7172 "t4getf"); 7173 if (rc) 7174 return (rc); 7175 7176 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 7177 t->idx >= nfilters) { 7178 t->idx = 0xffffffff; 7179 goto done; 7180 } 7181 7182 f = &sc->tids.ftid_tab[t->idx]; 7183 for (i = t->idx; i < nfilters; i++, f++) { 7184 if (f->valid) { 7185 t->idx = i; 7186 t->l2tidx = f->l2t ? f->l2t->idx : 0; 7187 t->smtidx = f->smtidx; 7188 if (f->fs.hitcnts) 7189 t->hits = get_filter_hits(sc, t->idx); 7190 else 7191 t->hits = UINT64_MAX; 7192 t->fs = f->fs; 7193 7194 goto done; 7195 } 7196 } 7197 7198 t->idx = 0xffffffff; 7199 done: 7200 end_synchronized_op(sc, LOCK_HELD); 7201 return (0); 7202 } 7203 7204 static int 7205 set_filter(struct adapter *sc, struct t4_filter *t) 7206 { 7207 unsigned int nfilters, nports; 7208 struct filter_entry *f; 7209 int i, rc; 7210 7211 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 7212 if (rc) 7213 return (rc); 7214 7215 nfilters = sc->tids.nftids; 7216 nports = sc->params.nports; 7217 7218 if (nfilters == 0) { 7219 rc = ENOTSUP; 7220 goto done; 7221 } 7222 7223 if (!(sc->flags & FULL_INIT_DONE)) { 7224 rc = EAGAIN; 7225 goto done; 7226 } 7227 7228 if (t->idx >= nfilters) { 7229 rc = EINVAL; 7230 goto done; 7231 } 7232 7233 /* Validate against the global filter mode */ 7234 if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) != 7235 sc->params.tp.vlan_pri_map) { 7236 rc = E2BIG; 7237 goto done; 7238 } 7239 7240 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 7241 rc = EINVAL; 7242 goto done; 7243 } 7244 7245 if (t->fs.val.iport >= nports) { 7246 rc = EINVAL; 7247 goto done; 7248 } 7249 7250 /* Can't specify an iq if not steering to it */ 7251 if (!t->fs.dirsteer && t->fs.iq) { 7252 rc = EINVAL; 7253 goto done; 7254 } 7255 7256 /* IPv6 filter idx must be 4 aligned */ 7257 if (t->fs.type == 1 && 7258 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 7259 rc = EINVAL; 7260 goto done; 7261 } 7262 7263 if (sc->tids.ftid_tab == NULL) { 7264 KASSERT(sc->tids.ftids_in_use == 0, 7265 ("%s: no memory allocated but filters_in_use > 0", 7266 __func__)); 7267 7268 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 7269 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 7270 if (sc->tids.ftid_tab == NULL) { 7271 rc = ENOMEM; 7272 goto done; 7273 } 7274 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 7275 } 7276 7277 for (i = 0; i < 4; i++) { 7278 f = &sc->tids.ftid_tab[t->idx + i]; 7279 7280 if (f->pending || f->valid) { 7281 rc = EBUSY; 7282 goto done; 7283 } 7284 if (f->locked) { 7285 rc = EPERM; 7286 goto done; 7287 } 7288 7289 if (t->fs.type == 0) 7290 break; 7291 } 7292 7293 f = &sc->tids.ftid_tab[t->idx]; 7294 f->fs = t->fs; 7295 7296 rc = set_filter_wr(sc, t->idx); 7297 done: 7298 end_synchronized_op(sc, 0); 7299 7300 if (rc == 0) { 7301 mtx_lock(&sc->tids.ftid_lock); 7302 for (;;) { 7303 if (f->pending == 0) { 7304 rc = f->valid ? 0 : EIO; 7305 break; 7306 } 7307 7308 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7309 PCATCH, "t4setfw", 0)) { 7310 rc = EINPROGRESS; 7311 break; 7312 } 7313 } 7314 mtx_unlock(&sc->tids.ftid_lock); 7315 } 7316 return (rc); 7317 } 7318 7319 static int 7320 del_filter(struct adapter *sc, struct t4_filter *t) 7321 { 7322 unsigned int nfilters; 7323 struct filter_entry *f; 7324 int rc; 7325 7326 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 7327 if (rc) 7328 return (rc); 7329 7330 nfilters = sc->tids.nftids; 7331 7332 if (nfilters == 0) { 7333 rc = ENOTSUP; 7334 goto done; 7335 } 7336 7337 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 7338 t->idx >= nfilters) { 7339 rc = EINVAL; 7340 goto done; 7341 } 7342 7343 if (!(sc->flags & FULL_INIT_DONE)) { 7344 rc = EAGAIN; 7345 goto done; 7346 } 7347 7348 f = &sc->tids.ftid_tab[t->idx]; 7349 7350 if (f->pending) { 7351 rc = EBUSY; 7352 goto done; 7353 } 7354 if (f->locked) { 7355 rc = EPERM; 7356 goto done; 7357 } 7358 7359 if (f->valid) { 7360 t->fs = f->fs; /* extra info for the caller */ 7361 rc = del_filter_wr(sc, t->idx); 7362 } 7363 7364 done: 7365 end_synchronized_op(sc, 0); 7366 7367 if (rc == 0) { 7368 mtx_lock(&sc->tids.ftid_lock); 7369 for (;;) { 7370 if (f->pending == 0) { 7371 rc = f->valid ? EIO : 0; 7372 break; 7373 } 7374 7375 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7376 PCATCH, "t4delfw", 0)) { 7377 rc = EINPROGRESS; 7378 break; 7379 } 7380 } 7381 mtx_unlock(&sc->tids.ftid_lock); 7382 } 7383 7384 return (rc); 7385 } 7386 7387 static void 7388 clear_filter(struct filter_entry *f) 7389 { 7390 if (f->l2t) 7391 t4_l2t_release(f->l2t); 7392 7393 bzero(f, sizeof (*f)); 7394 } 7395 7396 static int 7397 set_filter_wr(struct adapter *sc, int fidx) 7398 { 7399 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7400 struct fw_filter_wr *fwr; 7401 unsigned int ftid; 7402 struct wrq_cookie cookie; 7403 7404 ASSERT_SYNCHRONIZED_OP(sc); 7405 7406 if (f->fs.newdmac || f->fs.newvlan) { 7407 /* This filter needs an L2T entry; allocate one. */ 7408 f->l2t = t4_l2t_alloc_switching(sc->l2t); 7409 if (f->l2t == NULL) 7410 return (EAGAIN); 7411 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 7412 f->fs.dmac)) { 7413 t4_l2t_release(f->l2t); 7414 f->l2t = NULL; 7415 return (ENOMEM); 7416 } 7417 } 7418 7419 ftid = sc->tids.ftid_base + fidx; 7420 7421 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 7422 if (fwr == NULL) 7423 return (ENOMEM); 7424 bzero(fwr, sizeof(*fwr)); 7425 7426 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 7427 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 7428 fwr->tid_to_iq = 7429 htobe32(V_FW_FILTER_WR_TID(ftid) | 7430 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 7431 V_FW_FILTER_WR_NOREPLY(0) | 7432 V_FW_FILTER_WR_IQ(f->fs.iq)); 7433 fwr->del_filter_to_l2tix = 7434 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 7435 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 7436 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 7437 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 7438 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 7439 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 7440 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 7441 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 7442 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 7443 f->fs.newvlan == VLAN_REWRITE) | 7444 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 7445 f->fs.newvlan == VLAN_REWRITE) | 7446 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 7447 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 7448 V_FW_FILTER_WR_PRIO(f->fs.prio) | 7449 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 7450 fwr->ethtype = htobe16(f->fs.val.ethtype); 7451 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 7452 fwr->frag_to_ovlan_vldm = 7453 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 7454 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 7455 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 7456 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 7457 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 7458 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 7459 fwr->smac_sel = 0; 7460 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 7461 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 7462 fwr->maci_to_matchtypem = 7463 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 7464 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 7465 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 7466 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 7467 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 7468 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 7469 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 7470 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 7471 fwr->ptcl = f->fs.val.proto; 7472 fwr->ptclm = f->fs.mask.proto; 7473 fwr->ttyp = f->fs.val.tos; 7474 fwr->ttypm = f->fs.mask.tos; 7475 fwr->ivlan = htobe16(f->fs.val.vlan); 7476 fwr->ivlanm = htobe16(f->fs.mask.vlan); 7477 fwr->ovlan = htobe16(f->fs.val.vnic); 7478 fwr->ovlanm = htobe16(f->fs.mask.vnic); 7479 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 7480 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 7481 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 7482 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 7483 fwr->lp = htobe16(f->fs.val.dport); 7484 fwr->lpm = htobe16(f->fs.mask.dport); 7485 fwr->fp = htobe16(f->fs.val.sport); 7486 fwr->fpm = htobe16(f->fs.mask.sport); 7487 if (f->fs.newsmac) 7488 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 7489 7490 f->pending = 1; 7491 sc->tids.ftids_in_use++; 7492 7493 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 7494 return (0); 7495 } 7496 7497 static int 7498 del_filter_wr(struct adapter *sc, int fidx) 7499 { 7500 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7501 struct fw_filter_wr *fwr; 7502 unsigned int ftid; 7503 struct wrq_cookie cookie; 7504 7505 ftid = sc->tids.ftid_base + fidx; 7506 7507 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 7508 if (fwr == NULL) 7509 return (ENOMEM); 7510 bzero(fwr, sizeof (*fwr)); 7511 7512 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 7513 7514 f->pending = 1; 7515 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 7516 return (0); 7517 } 7518 7519 int 7520 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 7521 { 7522 struct adapter *sc = iq->adapter; 7523 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 7524 unsigned int idx = GET_TID(rpl); 7525 unsigned int rc; 7526 struct filter_entry *f; 7527 7528 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 7529 rss->opcode)); 7530 7531 if (is_ftid(sc, idx)) { 7532 7533 idx -= sc->tids.ftid_base; 7534 f = &sc->tids.ftid_tab[idx]; 7535 rc = G_COOKIE(rpl->cookie); 7536 7537 mtx_lock(&sc->tids.ftid_lock); 7538 if (rc == FW_FILTER_WR_FLT_ADDED) { 7539 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 7540 __func__, idx)); 7541 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 7542 f->pending = 0; /* asynchronous setup completed */ 7543 f->valid = 1; 7544 } else { 7545 if (rc != FW_FILTER_WR_FLT_DELETED) { 7546 /* Add or delete failed, display an error */ 7547 log(LOG_ERR, 7548 "filter %u setup failed with error %u\n", 7549 idx, rc); 7550 } 7551 7552 clear_filter(f); 7553 sc->tids.ftids_in_use--; 7554 } 7555 wakeup(&sc->tids.ftid_tab); 7556 mtx_unlock(&sc->tids.ftid_lock); 7557 } 7558 7559 return (0); 7560 } 7561 7562 static int 7563 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 7564 { 7565 int rc; 7566 7567 if (cntxt->cid > M_CTXTQID) 7568 return (EINVAL); 7569 7570 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 7571 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 7572 return (EINVAL); 7573 7574 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 7575 if (rc) 7576 return (rc); 7577 7578 if (sc->flags & FW_OK) { 7579 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 7580 &cntxt->data[0]); 7581 if (rc == 0) 7582 goto done; 7583 } 7584 7585 /* 7586 * Read via firmware failed or wasn't even attempted. Read directly via 7587 * the backdoor. 7588 */ 7589 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 7590 done: 7591 end_synchronized_op(sc, 0); 7592 return (rc); 7593 } 7594 7595 static int 7596 load_fw(struct adapter *sc, struct t4_data *fw) 7597 { 7598 int rc; 7599 uint8_t *fw_data; 7600 7601 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 7602 if (rc) 7603 return (rc); 7604 7605 if (sc->flags & FULL_INIT_DONE) { 7606 rc = EBUSY; 7607 goto done; 7608 } 7609 7610 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 7611 if (fw_data == NULL) { 7612 rc = ENOMEM; 7613 goto done; 7614 } 7615 7616 rc = copyin(fw->data, fw_data, fw->len); 7617 if (rc == 0) 7618 rc = -t4_load_fw(sc, fw_data, fw->len); 7619 7620 free(fw_data, M_CXGBE); 7621 done: 7622 end_synchronized_op(sc, 0); 7623 return (rc); 7624 } 7625 7626 static int 7627 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 7628 { 7629 uint32_t addr, off, remaining, i, n; 7630 uint32_t *buf, *b; 7631 uint32_t mw_base, mw_aperture; 7632 int rc; 7633 uint8_t *dst; 7634 7635 rc = validate_mem_range(sc, mr->addr, mr->len); 7636 if (rc != 0) 7637 return (rc); 7638 7639 memwin_info(sc, win, &mw_base, &mw_aperture); 7640 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 7641 addr = mr->addr; 7642 remaining = mr->len; 7643 dst = (void *)mr->data; 7644 7645 while (remaining) { 7646 off = position_memwin(sc, win, addr); 7647 7648 /* number of bytes that we'll copy in the inner loop */ 7649 n = min(remaining, mw_aperture - off); 7650 for (i = 0; i < n; i += 4) 7651 *b++ = t4_read_reg(sc, mw_base + off + i); 7652 7653 rc = copyout(buf, dst, n); 7654 if (rc != 0) 7655 break; 7656 7657 b = buf; 7658 dst += n; 7659 remaining -= n; 7660 addr += n; 7661 } 7662 7663 free(buf, M_CXGBE); 7664 return (rc); 7665 } 7666 7667 static int 7668 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 7669 { 7670 int rc; 7671 7672 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 7673 return (EINVAL); 7674 7675 if (i2cd->len > sizeof(i2cd->data)) 7676 return (EFBIG); 7677 7678 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 7679 if (rc) 7680 return (rc); 7681 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 7682 i2cd->offset, i2cd->len, &i2cd->data[0]); 7683 end_synchronized_op(sc, 0); 7684 7685 return (rc); 7686 } 7687 7688 static int 7689 in_range(int val, int lo, int hi) 7690 { 7691 7692 return (val < 0 || (val <= hi && val >= lo)); 7693 } 7694 7695 static int 7696 set_sched_class(struct adapter *sc, struct t4_sched_params *p) 7697 { 7698 int fw_subcmd, fw_type, rc; 7699 7700 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc"); 7701 if (rc) 7702 return (rc); 7703 7704 if (!(sc->flags & FULL_INIT_DONE)) { 7705 rc = EAGAIN; 7706 goto done; 7707 } 7708 7709 /* 7710 * Translate the cxgbetool parameters into T4 firmware parameters. (The 7711 * sub-command and type are in common locations.) 7712 */ 7713 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG) 7714 fw_subcmd = FW_SCHED_SC_CONFIG; 7715 else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS) 7716 fw_subcmd = FW_SCHED_SC_PARAMS; 7717 else { 7718 rc = EINVAL; 7719 goto done; 7720 } 7721 if (p->type == SCHED_CLASS_TYPE_PACKET) 7722 fw_type = FW_SCHED_TYPE_PKTSCHED; 7723 else { 7724 rc = EINVAL; 7725 goto done; 7726 } 7727 7728 if (fw_subcmd == FW_SCHED_SC_CONFIG) { 7729 /* Vet our parameters ..*/ 7730 if (p->u.config.minmax < 0) { 7731 rc = EINVAL; 7732 goto done; 7733 } 7734 7735 /* And pass the request to the firmware ...*/ 7736 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1); 7737 goto done; 7738 } 7739 7740 if (fw_subcmd == FW_SCHED_SC_PARAMS) { 7741 int fw_level; 7742 int fw_mode; 7743 int fw_rateunit; 7744 int fw_ratemode; 7745 7746 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL) 7747 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL; 7748 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) 7749 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR; 7750 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) 7751 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL; 7752 else { 7753 rc = EINVAL; 7754 goto done; 7755 } 7756 7757 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS) 7758 fw_mode = FW_SCHED_PARAMS_MODE_CLASS; 7759 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW) 7760 fw_mode = FW_SCHED_PARAMS_MODE_FLOW; 7761 else { 7762 rc = EINVAL; 7763 goto done; 7764 } 7765 7766 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS) 7767 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 7768 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS) 7769 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE; 7770 else { 7771 rc = EINVAL; 7772 goto done; 7773 } 7774 7775 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL) 7776 fw_ratemode = FW_SCHED_PARAMS_RATE_REL; 7777 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS) 7778 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS; 7779 else { 7780 rc = EINVAL; 7781 goto done; 7782 } 7783 7784 /* Vet our parameters ... */ 7785 if (!in_range(p->u.params.channel, 0, 3) || 7786 !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) || 7787 !in_range(p->u.params.minrate, 0, 10000000) || 7788 !in_range(p->u.params.maxrate, 0, 10000000) || 7789 !in_range(p->u.params.weight, 0, 100)) { 7790 rc = ERANGE; 7791 goto done; 7792 } 7793 7794 /* 7795 * Translate any unset parameters into the firmware's 7796 * nomenclature and/or fail the call if the parameters 7797 * are required ... 7798 */ 7799 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 || 7800 p->u.params.channel < 0 || p->u.params.cl < 0) { 7801 rc = EINVAL; 7802 goto done; 7803 } 7804 if (p->u.params.minrate < 0) 7805 p->u.params.minrate = 0; 7806 if (p->u.params.maxrate < 0) { 7807 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7808 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7809 rc = EINVAL; 7810 goto done; 7811 } else 7812 p->u.params.maxrate = 0; 7813 } 7814 if (p->u.params.weight < 0) { 7815 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) { 7816 rc = EINVAL; 7817 goto done; 7818 } else 7819 p->u.params.weight = 0; 7820 } 7821 if (p->u.params.pktsize < 0) { 7822 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7823 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7824 rc = EINVAL; 7825 goto done; 7826 } else 7827 p->u.params.pktsize = 0; 7828 } 7829 7830 /* See what the firmware thinks of the request ... */ 7831 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode, 7832 fw_rateunit, fw_ratemode, p->u.params.channel, 7833 p->u.params.cl, p->u.params.minrate, p->u.params.maxrate, 7834 p->u.params.weight, p->u.params.pktsize, 1); 7835 goto done; 7836 } 7837 7838 rc = EINVAL; 7839 done: 7840 end_synchronized_op(sc, 0); 7841 return (rc); 7842 } 7843 7844 static int 7845 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p) 7846 { 7847 struct port_info *pi = NULL; 7848 struct sge_txq *txq; 7849 uint32_t fw_mnem, fw_queue, fw_class; 7850 int i, rc; 7851 7852 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq"); 7853 if (rc) 7854 return (rc); 7855 7856 if (!(sc->flags & FULL_INIT_DONE)) { 7857 rc = EAGAIN; 7858 goto done; 7859 } 7860 7861 if (p->port >= sc->params.nports) { 7862 rc = EINVAL; 7863 goto done; 7864 } 7865 7866 pi = sc->port[p->port]; 7867 if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) { 7868 rc = EINVAL; 7869 goto done; 7870 } 7871 7872 /* 7873 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX 7874 * Scheduling Class in this case). 7875 */ 7876 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 7877 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH)); 7878 fw_class = p->cl < 0 ? 0xffffffff : p->cl; 7879 7880 /* 7881 * If op.queue is non-negative, then we're only changing the scheduling 7882 * on a single specified TX queue. 7883 */ 7884 if (p->queue >= 0) { 7885 txq = &sc->sge.txq[pi->first_txq + p->queue]; 7886 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7887 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7888 &fw_class); 7889 goto done; 7890 } 7891 7892 /* 7893 * Change the scheduling on all the TX queues for the 7894 * interface. 7895 */ 7896 for_each_txq(pi, i, txq) { 7897 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7898 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7899 &fw_class); 7900 if (rc) 7901 goto done; 7902 } 7903 7904 rc = 0; 7905 done: 7906 end_synchronized_op(sc, 0); 7907 return (rc); 7908 } 7909 7910 int 7911 t4_os_find_pci_capability(struct adapter *sc, int cap) 7912 { 7913 int i; 7914 7915 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 7916 } 7917 7918 int 7919 t4_os_pci_save_state(struct adapter *sc) 7920 { 7921 device_t dev; 7922 struct pci_devinfo *dinfo; 7923 7924 dev = sc->dev; 7925 dinfo = device_get_ivars(dev); 7926 7927 pci_cfg_save(dev, dinfo, 0); 7928 return (0); 7929 } 7930 7931 int 7932 t4_os_pci_restore_state(struct adapter *sc) 7933 { 7934 device_t dev; 7935 struct pci_devinfo *dinfo; 7936 7937 dev = sc->dev; 7938 dinfo = device_get_ivars(dev); 7939 7940 pci_cfg_restore(dev, dinfo); 7941 return (0); 7942 } 7943 7944 void 7945 t4_os_portmod_changed(const struct adapter *sc, int idx) 7946 { 7947 struct port_info *pi = sc->port[idx]; 7948 static const char *mod_str[] = { 7949 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 7950 }; 7951 7952 build_medialist(pi, &pi->media); 7953 #ifdef DEV_NETMAP 7954 build_medialist(pi, &pi->nm_media); 7955 #endif 7956 7957 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 7958 if_printf(pi->ifp, "transceiver unplugged.\n"); 7959 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 7960 if_printf(pi->ifp, "unknown transceiver inserted.\n"); 7961 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 7962 if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 7963 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 7964 if_printf(pi->ifp, "%s transceiver inserted.\n", 7965 mod_str[pi->mod_type]); 7966 } else { 7967 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 7968 pi->mod_type); 7969 } 7970 } 7971 7972 void 7973 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason) 7974 { 7975 struct port_info *pi = sc->port[idx]; 7976 struct ifnet *ifp = pi->ifp; 7977 7978 if (link_stat) { 7979 pi->linkdnrc = -1; 7980 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 7981 if_link_state_change(ifp, LINK_STATE_UP); 7982 } else { 7983 if (reason >= 0) 7984 pi->linkdnrc = reason; 7985 if_link_state_change(ifp, LINK_STATE_DOWN); 7986 } 7987 } 7988 7989 void 7990 t4_iterate(void (*func)(struct adapter *, void *), void *arg) 7991 { 7992 struct adapter *sc; 7993 7994 sx_slock(&t4_list_lock); 7995 SLIST_FOREACH(sc, &t4_list, link) { 7996 /* 7997 * func should not make any assumptions about what state sc is 7998 * in - the only guarantee is that sc->sc_lock is a valid lock. 7999 */ 8000 func(sc, arg); 8001 } 8002 sx_sunlock(&t4_list_lock); 8003 } 8004 8005 static int 8006 t4_open(struct cdev *dev, int flags, int type, struct thread *td) 8007 { 8008 return (0); 8009 } 8010 8011 static int 8012 t4_close(struct cdev *dev, int flags, int type, struct thread *td) 8013 { 8014 return (0); 8015 } 8016 8017 static int 8018 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 8019 struct thread *td) 8020 { 8021 int rc; 8022 struct adapter *sc = dev->si_drv1; 8023 8024 rc = priv_check(td, PRIV_DRIVER); 8025 if (rc != 0) 8026 return (rc); 8027 8028 switch (cmd) { 8029 case CHELSIO_T4_GETREG: { 8030 struct t4_reg *edata = (struct t4_reg *)data; 8031 8032 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8033 return (EFAULT); 8034 8035 if (edata->size == 4) 8036 edata->val = t4_read_reg(sc, edata->addr); 8037 else if (edata->size == 8) 8038 edata->val = t4_read_reg64(sc, edata->addr); 8039 else 8040 return (EINVAL); 8041 8042 break; 8043 } 8044 case CHELSIO_T4_SETREG: { 8045 struct t4_reg *edata = (struct t4_reg *)data; 8046 8047 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8048 return (EFAULT); 8049 8050 if (edata->size == 4) { 8051 if (edata->val & 0xffffffff00000000) 8052 return (EINVAL); 8053 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 8054 } else if (edata->size == 8) 8055 t4_write_reg64(sc, edata->addr, edata->val); 8056 else 8057 return (EINVAL); 8058 break; 8059 } 8060 case CHELSIO_T4_REGDUMP: { 8061 struct t4_regdump *regs = (struct t4_regdump *)data; 8062 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 8063 uint8_t *buf; 8064 8065 if (regs->len < reglen) { 8066 regs->len = reglen; /* hint to the caller */ 8067 return (ENOBUFS); 8068 } 8069 8070 regs->len = reglen; 8071 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 8072 t4_get_regs(sc, regs, buf); 8073 rc = copyout(buf, regs->data, reglen); 8074 free(buf, M_CXGBE); 8075 break; 8076 } 8077 case CHELSIO_T4_GET_FILTER_MODE: 8078 rc = get_filter_mode(sc, (uint32_t *)data); 8079 break; 8080 case CHELSIO_T4_SET_FILTER_MODE: 8081 rc = set_filter_mode(sc, *(uint32_t *)data); 8082 break; 8083 case CHELSIO_T4_GET_FILTER: 8084 rc = get_filter(sc, (struct t4_filter *)data); 8085 break; 8086 case CHELSIO_T4_SET_FILTER: 8087 rc = set_filter(sc, (struct t4_filter *)data); 8088 break; 8089 case CHELSIO_T4_DEL_FILTER: 8090 rc = del_filter(sc, (struct t4_filter *)data); 8091 break; 8092 case CHELSIO_T4_GET_SGE_CONTEXT: 8093 rc = get_sge_context(sc, (struct t4_sge_context *)data); 8094 break; 8095 case CHELSIO_T4_LOAD_FW: 8096 rc = load_fw(sc, (struct t4_data *)data); 8097 break; 8098 case CHELSIO_T4_GET_MEM: 8099 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 8100 break; 8101 case CHELSIO_T4_GET_I2C: 8102 rc = read_i2c(sc, (struct t4_i2c_data *)data); 8103 break; 8104 case CHELSIO_T4_CLEAR_STATS: { 8105 int i; 8106 u_int port_id = *(uint32_t *)data; 8107 struct port_info *pi; 8108 8109 if (port_id >= sc->params.nports) 8110 return (EINVAL); 8111 pi = sc->port[port_id]; 8112 8113 /* MAC stats */ 8114 t4_clr_port_stats(sc, pi->tx_chan); 8115 pi->tx_parse_error = 0; 8116 8117 if (pi->flags & PORT_INIT_DONE) { 8118 struct sge_rxq *rxq; 8119 struct sge_txq *txq; 8120 struct sge_wrq *wrq; 8121 8122 for_each_rxq(pi, i, rxq) { 8123 #if defined(INET) || defined(INET6) 8124 rxq->lro.lro_queued = 0; 8125 rxq->lro.lro_flushed = 0; 8126 #endif 8127 rxq->rxcsum = 0; 8128 rxq->vlan_extraction = 0; 8129 } 8130 8131 for_each_txq(pi, i, txq) { 8132 txq->txcsum = 0; 8133 txq->tso_wrs = 0; 8134 txq->vlan_insertion = 0; 8135 txq->imm_wrs = 0; 8136 txq->sgl_wrs = 0; 8137 txq->txpkt_wrs = 0; 8138 txq->txpkts0_wrs = 0; 8139 txq->txpkts1_wrs = 0; 8140 txq->txpkts0_pkts = 0; 8141 txq->txpkts1_pkts = 0; 8142 mp_ring_reset_stats(txq->r); 8143 } 8144 8145 #ifdef TCP_OFFLOAD 8146 /* nothing to clear for each ofld_rxq */ 8147 8148 for_each_ofld_txq(pi, i, wrq) { 8149 wrq->tx_wrs_direct = 0; 8150 wrq->tx_wrs_copied = 0; 8151 } 8152 #endif 8153 wrq = &sc->sge.ctrlq[pi->port_id]; 8154 wrq->tx_wrs_direct = 0; 8155 wrq->tx_wrs_copied = 0; 8156 } 8157 break; 8158 } 8159 case CHELSIO_T4_SCHED_CLASS: 8160 rc = set_sched_class(sc, (struct t4_sched_params *)data); 8161 break; 8162 case CHELSIO_T4_SCHED_QUEUE: 8163 rc = set_sched_queue(sc, (struct t4_sched_queue *)data); 8164 break; 8165 case CHELSIO_T4_GET_TRACER: 8166 rc = t4_get_tracer(sc, (struct t4_tracer *)data); 8167 break; 8168 case CHELSIO_T4_SET_TRACER: 8169 rc = t4_set_tracer(sc, (struct t4_tracer *)data); 8170 break; 8171 default: 8172 rc = EINVAL; 8173 } 8174 8175 return (rc); 8176 } 8177 8178 #ifdef TCP_OFFLOAD 8179 void 8180 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask, 8181 const unsigned int *pgsz_order) 8182 { 8183 struct port_info *pi = ifp->if_softc; 8184 struct adapter *sc = pi->adapter; 8185 8186 t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask); 8187 t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) | 8188 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) | 8189 V_HPZ3(pgsz_order[3])); 8190 } 8191 8192 static int 8193 toe_capability(struct port_info *pi, int enable) 8194 { 8195 int rc; 8196 struct adapter *sc = pi->adapter; 8197 8198 ASSERT_SYNCHRONIZED_OP(sc); 8199 8200 if (!is_offload(sc)) 8201 return (ENODEV); 8202 8203 if (enable) { 8204 if (!(sc->flags & FULL_INIT_DONE)) { 8205 rc = cxgbe_init_synchronized(pi); 8206 if (rc) 8207 return (rc); 8208 } 8209 8210 if (isset(&sc->offload_map, pi->port_id)) 8211 return (0); 8212 8213 if (!(sc->flags & TOM_INIT_DONE)) { 8214 rc = t4_activate_uld(sc, ULD_TOM); 8215 if (rc == EAGAIN) { 8216 log(LOG_WARNING, 8217 "You must kldload t4_tom.ko before trying " 8218 "to enable TOE on a cxgbe interface.\n"); 8219 } 8220 if (rc != 0) 8221 return (rc); 8222 KASSERT(sc->tom_softc != NULL, 8223 ("%s: TOM activated but softc NULL", __func__)); 8224 KASSERT(sc->flags & TOM_INIT_DONE, 8225 ("%s: TOM activated but flag not set", __func__)); 8226 } 8227 8228 setbit(&sc->offload_map, pi->port_id); 8229 } else { 8230 if (!isset(&sc->offload_map, pi->port_id)) 8231 return (0); 8232 8233 KASSERT(sc->flags & TOM_INIT_DONE, 8234 ("%s: TOM never initialized?", __func__)); 8235 clrbit(&sc->offload_map, pi->port_id); 8236 } 8237 8238 return (0); 8239 } 8240 8241 /* 8242 * Add an upper layer driver to the global list. 8243 */ 8244 int 8245 t4_register_uld(struct uld_info *ui) 8246 { 8247 int rc = 0; 8248 struct uld_info *u; 8249 8250 sx_xlock(&t4_uld_list_lock); 8251 SLIST_FOREACH(u, &t4_uld_list, link) { 8252 if (u->uld_id == ui->uld_id) { 8253 rc = EEXIST; 8254 goto done; 8255 } 8256 } 8257 8258 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 8259 ui->refcount = 0; 8260 done: 8261 sx_xunlock(&t4_uld_list_lock); 8262 return (rc); 8263 } 8264 8265 int 8266 t4_unregister_uld(struct uld_info *ui) 8267 { 8268 int rc = EINVAL; 8269 struct uld_info *u; 8270 8271 sx_xlock(&t4_uld_list_lock); 8272 8273 SLIST_FOREACH(u, &t4_uld_list, link) { 8274 if (u == ui) { 8275 if (ui->refcount > 0) { 8276 rc = EBUSY; 8277 goto done; 8278 } 8279 8280 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 8281 rc = 0; 8282 goto done; 8283 } 8284 } 8285 done: 8286 sx_xunlock(&t4_uld_list_lock); 8287 return (rc); 8288 } 8289 8290 int 8291 t4_activate_uld(struct adapter *sc, int id) 8292 { 8293 int rc = EAGAIN; 8294 struct uld_info *ui; 8295 8296 ASSERT_SYNCHRONIZED_OP(sc); 8297 8298 sx_slock(&t4_uld_list_lock); 8299 8300 SLIST_FOREACH(ui, &t4_uld_list, link) { 8301 if (ui->uld_id == id) { 8302 if (!(sc->flags & FULL_INIT_DONE)) { 8303 rc = adapter_full_init(sc); 8304 if (rc != 0) 8305 goto done; 8306 } 8307 8308 rc = ui->activate(sc); 8309 if (rc == 0) 8310 ui->refcount++; 8311 goto done; 8312 } 8313 } 8314 done: 8315 sx_sunlock(&t4_uld_list_lock); 8316 8317 return (rc); 8318 } 8319 8320 int 8321 t4_deactivate_uld(struct adapter *sc, int id) 8322 { 8323 int rc = EINVAL; 8324 struct uld_info *ui; 8325 8326 ASSERT_SYNCHRONIZED_OP(sc); 8327 8328 sx_slock(&t4_uld_list_lock); 8329 8330 SLIST_FOREACH(ui, &t4_uld_list, link) { 8331 if (ui->uld_id == id) { 8332 rc = ui->deactivate(sc); 8333 if (rc == 0) 8334 ui->refcount--; 8335 goto done; 8336 } 8337 } 8338 done: 8339 sx_sunlock(&t4_uld_list_lock); 8340 8341 return (rc); 8342 } 8343 #endif 8344 8345 /* 8346 * Come up with reasonable defaults for some of the tunables, provided they're 8347 * not set by the user (in which case we'll use the values as is). 8348 */ 8349 static void 8350 tweak_tunables(void) 8351 { 8352 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 8353 8354 if (t4_ntxq10g < 1) 8355 t4_ntxq10g = min(nc, NTXQ_10G); 8356 8357 if (t4_ntxq1g < 1) 8358 t4_ntxq1g = min(nc, NTXQ_1G); 8359 8360 if (t4_nrxq10g < 1) 8361 t4_nrxq10g = min(nc, NRXQ_10G); 8362 8363 if (t4_nrxq1g < 1) 8364 t4_nrxq1g = min(nc, NRXQ_1G); 8365 8366 #ifdef TCP_OFFLOAD 8367 if (t4_nofldtxq10g < 1) 8368 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 8369 8370 if (t4_nofldtxq1g < 1) 8371 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 8372 8373 if (t4_nofldrxq10g < 1) 8374 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 8375 8376 if (t4_nofldrxq1g < 1) 8377 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 8378 8379 if (t4_toecaps_allowed == -1) 8380 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 8381 #else 8382 if (t4_toecaps_allowed == -1) 8383 t4_toecaps_allowed = 0; 8384 #endif 8385 8386 #ifdef DEV_NETMAP 8387 if (t4_nnmtxq10g < 1) 8388 t4_nnmtxq10g = min(nc, NNMTXQ_10G); 8389 8390 if (t4_nnmtxq1g < 1) 8391 t4_nnmtxq1g = min(nc, NNMTXQ_1G); 8392 8393 if (t4_nnmrxq10g < 1) 8394 t4_nnmrxq10g = min(nc, NNMRXQ_10G); 8395 8396 if (t4_nnmrxq1g < 1) 8397 t4_nnmrxq1g = min(nc, NNMRXQ_1G); 8398 #endif 8399 8400 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 8401 t4_tmr_idx_10g = TMR_IDX_10G; 8402 8403 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 8404 t4_pktc_idx_10g = PKTC_IDX_10G; 8405 8406 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 8407 t4_tmr_idx_1g = TMR_IDX_1G; 8408 8409 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 8410 t4_pktc_idx_1g = PKTC_IDX_1G; 8411 8412 if (t4_qsize_txq < 128) 8413 t4_qsize_txq = 128; 8414 8415 if (t4_qsize_rxq < 128) 8416 t4_qsize_rxq = 128; 8417 while (t4_qsize_rxq & 7) 8418 t4_qsize_rxq++; 8419 8420 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 8421 } 8422 8423 static struct sx mlu; /* mod load unload */ 8424 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload"); 8425 8426 static int 8427 mod_event(module_t mod, int cmd, void *arg) 8428 { 8429 int rc = 0; 8430 static int loaded = 0; 8431 8432 switch (cmd) { 8433 case MOD_LOAD: 8434 sx_xlock(&mlu); 8435 if (loaded++ == 0) { 8436 t4_sge_modload(); 8437 sx_init(&t4_list_lock, "T4/T5 adapters"); 8438 SLIST_INIT(&t4_list); 8439 #ifdef TCP_OFFLOAD 8440 sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); 8441 SLIST_INIT(&t4_uld_list); 8442 #endif 8443 t4_tracer_modload(); 8444 tweak_tunables(); 8445 } 8446 sx_xunlock(&mlu); 8447 break; 8448 8449 case MOD_UNLOAD: 8450 sx_xlock(&mlu); 8451 if (--loaded == 0) { 8452 int tries; 8453 8454 sx_slock(&t4_list_lock); 8455 if (!SLIST_EMPTY(&t4_list)) { 8456 rc = EBUSY; 8457 sx_sunlock(&t4_list_lock); 8458 goto done_unload; 8459 } 8460 #ifdef TCP_OFFLOAD 8461 sx_slock(&t4_uld_list_lock); 8462 if (!SLIST_EMPTY(&t4_uld_list)) { 8463 rc = EBUSY; 8464 sx_sunlock(&t4_uld_list_lock); 8465 sx_sunlock(&t4_list_lock); 8466 goto done_unload; 8467 } 8468 #endif 8469 tries = 0; 8470 while (tries++ < 5 && t4_sge_extfree_refs() != 0) { 8471 uprintf("%ju clusters with custom free routine " 8472 "still is use.\n", t4_sge_extfree_refs()); 8473 pause("t4unload", 2 * hz); 8474 } 8475 #ifdef TCP_OFFLOAD 8476 sx_sunlock(&t4_uld_list_lock); 8477 #endif 8478 sx_sunlock(&t4_list_lock); 8479 8480 if (t4_sge_extfree_refs() == 0) { 8481 t4_tracer_modunload(); 8482 #ifdef TCP_OFFLOAD 8483 sx_destroy(&t4_uld_list_lock); 8484 #endif 8485 sx_destroy(&t4_list_lock); 8486 t4_sge_modunload(); 8487 loaded = 0; 8488 } else { 8489 rc = EBUSY; 8490 loaded++; /* undo earlier decrement */ 8491 } 8492 } 8493 done_unload: 8494 sx_xunlock(&mlu); 8495 break; 8496 } 8497 8498 return (rc); 8499 } 8500 8501 static devclass_t t4_devclass, t5_devclass; 8502 static devclass_t cxgbe_devclass, cxl_devclass; 8503 8504 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 8505 MODULE_VERSION(t4nex, 1); 8506 MODULE_DEPEND(t4nex, firmware, 1, 1, 1); 8507 8508 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 8509 MODULE_VERSION(t5nex, 1); 8510 MODULE_DEPEND(t5nex, firmware, 1, 1, 1); 8511 8512 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 8513 MODULE_VERSION(cxgbe, 1); 8514 8515 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 8516 MODULE_VERSION(cxl, 1); 8517