1 /*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_inet.h" 32 #include "opt_inet6.h" 33 #include "opt_rss.h" 34 35 #include <sys/param.h> 36 #include <sys/conf.h> 37 #include <sys/priv.h> 38 #include <sys/kernel.h> 39 #include <sys/bus.h> 40 #include <sys/module.h> 41 #include <sys/malloc.h> 42 #include <sys/queue.h> 43 #include <sys/taskqueue.h> 44 #include <sys/pciio.h> 45 #include <dev/pci/pcireg.h> 46 #include <dev/pci/pcivar.h> 47 #include <dev/pci/pci_private.h> 48 #include <sys/firmware.h> 49 #include <sys/sbuf.h> 50 #include <sys/smp.h> 51 #include <sys/socket.h> 52 #include <sys/sockio.h> 53 #include <sys/sysctl.h> 54 #include <net/ethernet.h> 55 #include <net/if.h> 56 #include <net/if_types.h> 57 #include <net/if_dl.h> 58 #include <net/if_vlan_var.h> 59 #ifdef RSS 60 #include <net/rss_config.h> 61 #endif 62 #if defined(__i386__) || defined(__amd64__) 63 #include <vm/vm.h> 64 #include <vm/pmap.h> 65 #endif 66 67 #include "common/common.h" 68 #include "common/t4_msg.h" 69 #include "common/t4_regs.h" 70 #include "common/t4_regs_values.h" 71 #include "t4_ioctl.h" 72 #include "t4_l2t.h" 73 #include "t4_mp_ring.h" 74 75 /* T4 bus driver interface */ 76 static int t4_probe(device_t); 77 static int t4_attach(device_t); 78 static int t4_detach(device_t); 79 static device_method_t t4_methods[] = { 80 DEVMETHOD(device_probe, t4_probe), 81 DEVMETHOD(device_attach, t4_attach), 82 DEVMETHOD(device_detach, t4_detach), 83 84 DEVMETHOD_END 85 }; 86 static driver_t t4_driver = { 87 "t4nex", 88 t4_methods, 89 sizeof(struct adapter) 90 }; 91 92 93 /* T4 port (cxgbe) interface */ 94 static int cxgbe_probe(device_t); 95 static int cxgbe_attach(device_t); 96 static int cxgbe_detach(device_t); 97 static device_method_t cxgbe_methods[] = { 98 DEVMETHOD(device_probe, cxgbe_probe), 99 DEVMETHOD(device_attach, cxgbe_attach), 100 DEVMETHOD(device_detach, cxgbe_detach), 101 { 0, 0 } 102 }; 103 static driver_t cxgbe_driver = { 104 "cxgbe", 105 cxgbe_methods, 106 sizeof(struct port_info) 107 }; 108 109 /* T4 VI (vcxgbe) interface */ 110 static int vcxgbe_probe(device_t); 111 static int vcxgbe_attach(device_t); 112 static int vcxgbe_detach(device_t); 113 static device_method_t vcxgbe_methods[] = { 114 DEVMETHOD(device_probe, vcxgbe_probe), 115 DEVMETHOD(device_attach, vcxgbe_attach), 116 DEVMETHOD(device_detach, vcxgbe_detach), 117 { 0, 0 } 118 }; 119 static driver_t vcxgbe_driver = { 120 "vcxgbe", 121 vcxgbe_methods, 122 sizeof(struct vi_info) 123 }; 124 125 static d_ioctl_t t4_ioctl; 126 static d_open_t t4_open; 127 static d_close_t t4_close; 128 129 static struct cdevsw t4_cdevsw = { 130 .d_version = D_VERSION, 131 .d_flags = 0, 132 .d_open = t4_open, 133 .d_close = t4_close, 134 .d_ioctl = t4_ioctl, 135 .d_name = "t4nex", 136 }; 137 138 /* T5 bus driver interface */ 139 static int t5_probe(device_t); 140 static device_method_t t5_methods[] = { 141 DEVMETHOD(device_probe, t5_probe), 142 DEVMETHOD(device_attach, t4_attach), 143 DEVMETHOD(device_detach, t4_detach), 144 145 DEVMETHOD_END 146 }; 147 static driver_t t5_driver = { 148 "t5nex", 149 t5_methods, 150 sizeof(struct adapter) 151 }; 152 153 154 /* T5 port (cxl) interface */ 155 static driver_t cxl_driver = { 156 "cxl", 157 cxgbe_methods, 158 sizeof(struct port_info) 159 }; 160 161 /* T5 VI (vcxl) interface */ 162 static driver_t vcxl_driver = { 163 "vcxl", 164 vcxgbe_methods, 165 sizeof(struct vi_info) 166 }; 167 168 static struct cdevsw t5_cdevsw = { 169 .d_version = D_VERSION, 170 .d_flags = 0, 171 .d_open = t4_open, 172 .d_close = t4_close, 173 .d_ioctl = t4_ioctl, 174 .d_name = "t5nex", 175 }; 176 177 /* ifnet + media interface */ 178 static void cxgbe_init(void *); 179 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 180 static int cxgbe_transmit(struct ifnet *, struct mbuf *); 181 static void cxgbe_qflush(struct ifnet *); 182 static int cxgbe_media_change(struct ifnet *); 183 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 184 185 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 186 187 /* 188 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 189 * then ADAPTER_LOCK, then t4_uld_list_lock. 190 */ 191 static struct sx t4_list_lock; 192 SLIST_HEAD(, adapter) t4_list; 193 #ifdef TCP_OFFLOAD 194 static struct sx t4_uld_list_lock; 195 SLIST_HEAD(, uld_info) t4_uld_list; 196 #endif 197 198 /* 199 * Tunables. See tweak_tunables() too. 200 * 201 * Each tunable is set to a default value here if it's known at compile-time. 202 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 203 * provide a reasonable default when the driver is loaded. 204 * 205 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 206 * T5 are under hw.cxl. 207 */ 208 209 /* 210 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 211 */ 212 #define NTXQ_10G 16 213 static int t4_ntxq10g = -1; 214 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 215 216 #define NRXQ_10G 8 217 static int t4_nrxq10g = -1; 218 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 219 220 #define NTXQ_1G 4 221 static int t4_ntxq1g = -1; 222 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 223 224 #define NRXQ_1G 2 225 static int t4_nrxq1g = -1; 226 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 227 228 static int t4_rsrv_noflowq = 0; 229 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq); 230 231 #ifdef TCP_OFFLOAD 232 #define NOFLDTXQ_10G 8 233 static int t4_nofldtxq10g = -1; 234 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 235 236 #define NOFLDRXQ_10G 2 237 static int t4_nofldrxq10g = -1; 238 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 239 240 #define NOFLDTXQ_1G 2 241 static int t4_nofldtxq1g = -1; 242 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 243 244 #define NOFLDRXQ_1G 1 245 static int t4_nofldrxq1g = -1; 246 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 247 #endif 248 249 #ifdef DEV_NETMAP 250 #define NNMTXQ_10G 2 251 static int t4_nnmtxq10g = -1; 252 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g); 253 254 #define NNMRXQ_10G 2 255 static int t4_nnmrxq10g = -1; 256 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g); 257 258 #define NNMTXQ_1G 1 259 static int t4_nnmtxq1g = -1; 260 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g); 261 262 #define NNMRXQ_1G 1 263 static int t4_nnmrxq1g = -1; 264 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g); 265 #endif 266 267 /* 268 * Holdoff parameters for 10G and 1G ports. 269 */ 270 #define TMR_IDX_10G 1 271 static int t4_tmr_idx_10g = TMR_IDX_10G; 272 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 273 274 #define PKTC_IDX_10G (-1) 275 static int t4_pktc_idx_10g = PKTC_IDX_10G; 276 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 277 278 #define TMR_IDX_1G 1 279 static int t4_tmr_idx_1g = TMR_IDX_1G; 280 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 281 282 #define PKTC_IDX_1G (-1) 283 static int t4_pktc_idx_1g = PKTC_IDX_1G; 284 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 285 286 /* 287 * Size (# of entries) of each tx and rx queue. 288 */ 289 static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 290 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 291 292 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 293 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 294 295 /* 296 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 297 */ 298 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 299 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 300 301 /* 302 * Configuration file. 303 */ 304 #define DEFAULT_CF "default" 305 #define FLASH_CF "flash" 306 #define UWIRE_CF "uwire" 307 #define FPGA_CF "fpga" 308 static char t4_cfg_file[32] = DEFAULT_CF; 309 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 310 311 /* 312 * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively). 313 * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them. 314 * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water 315 * mark or when signalled to do so, 0 to never emit PAUSE. 316 */ 317 static int t4_pause_settings = PAUSE_TX | PAUSE_RX; 318 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings); 319 320 /* 321 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 322 * encouraged respectively). 323 */ 324 static unsigned int t4_fw_install = 1; 325 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 326 327 /* 328 * ASIC features that will be used. Disable the ones you don't want so that the 329 * chip resources aren't wasted on features that will not be used. 330 */ 331 static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 332 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 333 334 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 335 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 336 337 static int t4_toecaps_allowed = -1; 338 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 339 340 static int t4_rdmacaps_allowed = 0; 341 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 342 343 static int t4_iscsicaps_allowed = 0; 344 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 345 346 static int t4_fcoecaps_allowed = 0; 347 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 348 349 static int t5_write_combine = 0; 350 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 351 352 static int t4_num_vis = 1; 353 TUNABLE_INT("hw.cxgbe.num_vis", &t4_num_vis); 354 355 /* Functions used by extra VIs to obtain unique MAC addresses for each VI. */ 356 static int vi_mac_funcs[] = { 357 FW_VI_FUNC_OFLD, 358 FW_VI_FUNC_IWARP, 359 FW_VI_FUNC_OPENISCSI, 360 FW_VI_FUNC_OPENFCOE, 361 FW_VI_FUNC_FOISCSI, 362 FW_VI_FUNC_FOFCOE, 363 }; 364 365 struct intrs_and_queues { 366 uint16_t intr_type; /* INTx, MSI, or MSI-X */ 367 uint16_t nirq; /* Total # of vectors */ 368 uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */ 369 uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */ 370 uint16_t ntxq10g; /* # of NIC txq's for each 10G port */ 371 uint16_t nrxq10g; /* # of NIC rxq's for each 10G port */ 372 uint16_t ntxq1g; /* # of NIC txq's for each 1G port */ 373 uint16_t nrxq1g; /* # of NIC rxq's for each 1G port */ 374 uint16_t rsrv_noflowq; /* Flag whether to reserve queue 0 */ 375 #ifdef TCP_OFFLOAD 376 uint16_t nofldtxq10g; /* # of TOE txq's for each 10G port */ 377 uint16_t nofldrxq10g; /* # of TOE rxq's for each 10G port */ 378 uint16_t nofldtxq1g; /* # of TOE txq's for each 1G port */ 379 uint16_t nofldrxq1g; /* # of TOE rxq's for each 1G port */ 380 #endif 381 #ifdef DEV_NETMAP 382 uint16_t nnmtxq10g; /* # of netmap txq's for each 10G port */ 383 uint16_t nnmrxq10g; /* # of netmap rxq's for each 10G port */ 384 uint16_t nnmtxq1g; /* # of netmap txq's for each 1G port */ 385 uint16_t nnmrxq1g; /* # of netmap rxq's for each 1G port */ 386 #endif 387 }; 388 389 struct filter_entry { 390 uint32_t valid:1; /* filter allocated and valid */ 391 uint32_t locked:1; /* filter is administratively locked */ 392 uint32_t pending:1; /* filter action is pending firmware reply */ 393 uint32_t smtidx:8; /* Source MAC Table index for smac */ 394 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 395 396 struct t4_filter_specification fs; 397 }; 398 399 static int map_bars_0_and_4(struct adapter *); 400 static int map_bar_2(struct adapter *); 401 static void setup_memwin(struct adapter *); 402 static int validate_mem_range(struct adapter *, uint32_t, int); 403 static int fwmtype_to_hwmtype(int); 404 static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 405 uint32_t *); 406 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 407 static uint32_t position_memwin(struct adapter *, int, uint32_t); 408 static int cfg_itype_and_nqueues(struct adapter *, int, int, int, 409 struct intrs_and_queues *); 410 static int prep_firmware(struct adapter *); 411 static int partition_resources(struct adapter *, const struct firmware *, 412 const char *); 413 static int get_params__pre_init(struct adapter *); 414 static int get_params__post_init(struct adapter *); 415 static int set_params__post_init(struct adapter *); 416 static void t4_set_desc(struct adapter *); 417 static void build_medialist(struct port_info *, struct ifmedia *); 418 static int cxgbe_init_synchronized(struct vi_info *); 419 static int cxgbe_uninit_synchronized(struct vi_info *); 420 static int setup_intr_handlers(struct adapter *); 421 static void quiesce_txq(struct adapter *, struct sge_txq *); 422 static void quiesce_wrq(struct adapter *, struct sge_wrq *); 423 static void quiesce_iq(struct adapter *, struct sge_iq *); 424 static void quiesce_fl(struct adapter *, struct sge_fl *); 425 static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 426 driver_intr_t *, void *, char *); 427 static int t4_free_irq(struct adapter *, struct irq *); 428 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 429 unsigned int); 430 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 431 static void vi_refresh_stats(struct adapter *, struct vi_info *); 432 static void cxgbe_refresh_stats(struct adapter *, struct port_info *); 433 static void cxgbe_tick(void *); 434 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 435 static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 436 struct mbuf *); 437 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 438 static int fw_msg_not_handled(struct adapter *, const __be64 *); 439 static void t4_sysctls(struct adapter *); 440 static void cxgbe_sysctls(struct port_info *); 441 static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 442 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 443 static int sysctl_btphy(SYSCTL_HANDLER_ARGS); 444 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); 445 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 446 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 447 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 448 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 449 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS); 450 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 451 static int sysctl_temperature(SYSCTL_HANDLER_ARGS); 452 #ifdef SBUF_DRAIN 453 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 454 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 455 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 456 static int sysctl_cim_la_t6(SYSCTL_HANDLER_ARGS); 457 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); 458 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); 459 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 460 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 461 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 462 static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 463 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 464 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 465 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 466 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); 467 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 468 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); 469 static int sysctl_mps_tcam_t6(SYSCTL_HANDLER_ARGS); 470 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 471 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 472 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 473 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 474 static int sysctl_tids(SYSCTL_HANDLER_ARGS); 475 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 476 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 477 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 478 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 479 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 480 #endif 481 static uint32_t fconf_to_mode(uint32_t); 482 static uint32_t mode_to_fconf(uint32_t); 483 static uint32_t fspec_to_fconf(struct t4_filter_specification *); 484 static int get_filter_mode(struct adapter *, uint32_t *); 485 static int set_filter_mode(struct adapter *, uint32_t); 486 static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 487 static int get_filter(struct adapter *, struct t4_filter *); 488 static int set_filter(struct adapter *, struct t4_filter *); 489 static int del_filter(struct adapter *, struct t4_filter *); 490 static void clear_filter(struct filter_entry *); 491 static int set_filter_wr(struct adapter *, int); 492 static int del_filter_wr(struct adapter *, int); 493 static int get_sge_context(struct adapter *, struct t4_sge_context *); 494 static int load_fw(struct adapter *, struct t4_data *); 495 static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 496 static int read_i2c(struct adapter *, struct t4_i2c_data *); 497 static int set_sched_class(struct adapter *, struct t4_sched_params *); 498 static int set_sched_queue(struct adapter *, struct t4_sched_queue *); 499 #ifdef TCP_OFFLOAD 500 static int toe_capability(struct vi_info *, int); 501 #endif 502 static int mod_event(module_t, int, void *); 503 504 struct { 505 uint16_t device; 506 char *desc; 507 } t4_pciids[] = { 508 {0xa000, "Chelsio Terminator 4 FPGA"}, 509 {0x4400, "Chelsio T440-dbg"}, 510 {0x4401, "Chelsio T420-CR"}, 511 {0x4402, "Chelsio T422-CR"}, 512 {0x4403, "Chelsio T440-CR"}, 513 {0x4404, "Chelsio T420-BCH"}, 514 {0x4405, "Chelsio T440-BCH"}, 515 {0x4406, "Chelsio T440-CH"}, 516 {0x4407, "Chelsio T420-SO"}, 517 {0x4408, "Chelsio T420-CX"}, 518 {0x4409, "Chelsio T420-BT"}, 519 {0x440a, "Chelsio T404-BT"}, 520 {0x440e, "Chelsio T440-LP-CR"}, 521 }, t5_pciids[] = { 522 {0xb000, "Chelsio Terminator 5 FPGA"}, 523 {0x5400, "Chelsio T580-dbg"}, 524 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ 525 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ 526 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ 527 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ 528 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ 529 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ 530 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ 531 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ 532 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 533 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ 534 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ 535 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ 536 {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */ 537 #ifdef notyet 538 {0x5404, "Chelsio T520-BCH"}, 539 {0x5405, "Chelsio T540-BCH"}, 540 {0x5406, "Chelsio T540-CH"}, 541 {0x5408, "Chelsio T520-CX"}, 542 {0x540b, "Chelsio B520-SR"}, 543 {0x540c, "Chelsio B504-BT"}, 544 {0x540f, "Chelsio Amsterdam"}, 545 {0x5413, "Chelsio T580-CHR"}, 546 #endif 547 }; 548 549 #ifdef TCP_OFFLOAD 550 /* 551 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 552 * exactly the same for both rxq and ofld_rxq. 553 */ 554 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 555 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 556 #endif 557 558 /* No easy way to include t4_msg.h before adapter.h so we check this way */ 559 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 560 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 561 562 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); 563 564 static int 565 t4_probe(device_t dev) 566 { 567 int i; 568 uint16_t v = pci_get_vendor(dev); 569 uint16_t d = pci_get_device(dev); 570 uint8_t f = pci_get_function(dev); 571 572 if (v != PCI_VENDOR_ID_CHELSIO) 573 return (ENXIO); 574 575 /* Attach only to PF0 of the FPGA */ 576 if (d == 0xa000 && f != 0) 577 return (ENXIO); 578 579 for (i = 0; i < nitems(t4_pciids); i++) { 580 if (d == t4_pciids[i].device) { 581 device_set_desc(dev, t4_pciids[i].desc); 582 return (BUS_PROBE_DEFAULT); 583 } 584 } 585 586 return (ENXIO); 587 } 588 589 static int 590 t5_probe(device_t dev) 591 { 592 int i; 593 uint16_t v = pci_get_vendor(dev); 594 uint16_t d = pci_get_device(dev); 595 uint8_t f = pci_get_function(dev); 596 597 if (v != PCI_VENDOR_ID_CHELSIO) 598 return (ENXIO); 599 600 /* Attach only to PF0 of the FPGA */ 601 if (d == 0xb000 && f != 0) 602 return (ENXIO); 603 604 for (i = 0; i < nitems(t5_pciids); i++) { 605 if (d == t5_pciids[i].device) { 606 device_set_desc(dev, t5_pciids[i].desc); 607 return (BUS_PROBE_DEFAULT); 608 } 609 } 610 611 return (ENXIO); 612 } 613 614 static void 615 t5_attribute_workaround(device_t dev) 616 { 617 device_t root_port; 618 uint32_t v; 619 620 /* 621 * The T5 chips do not properly echo the No Snoop and Relaxed 622 * Ordering attributes when replying to a TLP from a Root 623 * Port. As a workaround, find the parent Root Port and 624 * disable No Snoop and Relaxed Ordering. Note that this 625 * affects all devices under this root port. 626 */ 627 root_port = pci_find_pcie_root_port(dev); 628 if (root_port == NULL) { 629 device_printf(dev, "Unable to find parent root port\n"); 630 return; 631 } 632 633 v = pcie_adjust_config(root_port, PCIER_DEVICE_CTL, 634 PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE, 0, 2); 635 if ((v & (PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE)) != 636 0) 637 device_printf(dev, "Disabled No Snoop/Relaxed Ordering on %s\n", 638 device_get_nameunit(root_port)); 639 } 640 641 static int 642 t4_attach(device_t dev) 643 { 644 struct adapter *sc; 645 int rc = 0, i, j, n10g, n1g, rqidx, tqidx; 646 struct intrs_and_queues iaq; 647 struct sge *s; 648 #ifdef TCP_OFFLOAD 649 int ofld_rqidx, ofld_tqidx; 650 #endif 651 #ifdef DEV_NETMAP 652 int nm_rqidx, nm_tqidx; 653 #endif 654 int num_vis; 655 656 sc = device_get_softc(dev); 657 sc->dev = dev; 658 TUNABLE_INT_FETCH("hw.cxgbe.debug_flags", &sc->debug_flags); 659 660 if ((pci_get_device(dev) & 0xff00) == 0x5400) 661 t5_attribute_workaround(dev); 662 pci_enable_busmaster(dev); 663 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 664 uint32_t v; 665 666 pci_set_max_read_req(dev, 4096); 667 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 668 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 669 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 670 671 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5); 672 } 673 674 sc->traceq = -1; 675 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); 676 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", 677 device_get_nameunit(dev)); 678 679 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 680 device_get_nameunit(dev)); 681 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 682 sx_xlock(&t4_list_lock); 683 SLIST_INSERT_HEAD(&t4_list, sc, link); 684 sx_xunlock(&t4_list_lock); 685 686 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 687 TAILQ_INIT(&sc->sfl); 688 callout_init_mtx(&sc->sfl_callout, &sc->sfl_lock, 0); 689 690 mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF); 691 692 rc = map_bars_0_and_4(sc); 693 if (rc != 0) 694 goto done; /* error message displayed already */ 695 696 /* 697 * This is the real PF# to which we're attaching. Works from within PCI 698 * passthrough environments too, where pci_get_function() could return a 699 * different PF# depending on the passthrough configuration. We need to 700 * use the real PF# in all our communication with the firmware. 701 */ 702 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 703 sc->mbox = sc->pf; 704 705 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 706 sc->an_handler = an_not_handled; 707 for (i = 0; i < nitems(sc->cpl_handler); i++) 708 sc->cpl_handler[i] = cpl_not_handled; 709 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 710 sc->fw_msg_handler[i] = fw_msg_not_handled; 711 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 712 t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt); 713 t4_register_cpl_handler(sc, CPL_T5_TRACE_PKT, t5_trace_pkt); 714 t4_init_sge_cpl_handlers(sc); 715 716 /* Prepare the adapter for operation */ 717 rc = -t4_prep_adapter(sc); 718 if (rc != 0) { 719 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 720 goto done; 721 } 722 723 /* 724 * Do this really early, with the memory windows set up even before the 725 * character device. The userland tool's register i/o and mem read 726 * will work even in "recovery mode". 727 */ 728 setup_memwin(sc); 729 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 730 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 731 device_get_nameunit(dev)); 732 if (sc->cdev == NULL) 733 device_printf(dev, "failed to create nexus char device.\n"); 734 else 735 sc->cdev->si_drv1 = sc; 736 737 /* Go no further if recovery mode has been requested. */ 738 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 739 device_printf(dev, "recovery mode.\n"); 740 goto done; 741 } 742 743 #if defined(__i386__) 744 if ((cpu_feature & CPUID_CX8) == 0) { 745 device_printf(dev, "64 bit atomics not available.\n"); 746 rc = ENOTSUP; 747 goto done; 748 } 749 #endif 750 751 /* Prepare the firmware for operation */ 752 rc = prep_firmware(sc); 753 if (rc != 0) 754 goto done; /* error message displayed already */ 755 756 rc = get_params__post_init(sc); 757 if (rc != 0) 758 goto done; /* error message displayed already */ 759 760 rc = set_params__post_init(sc); 761 if (rc != 0) 762 goto done; /* error message displayed already */ 763 764 rc = map_bar_2(sc); 765 if (rc != 0) 766 goto done; /* error message displayed already */ 767 768 rc = t4_create_dma_tag(sc); 769 if (rc != 0) 770 goto done; /* error message displayed already */ 771 772 /* 773 * Number of VIs to create per-port. The first VI is the 774 * "main" regular VI for the port. The second VI is used for 775 * netmap if present, and any remaining VIs are used for 776 * additional virtual interfaces. 777 * 778 * Limit the number of VIs per port to the number of available 779 * MAC addresses per port. 780 */ 781 if (t4_num_vis >= 1) 782 num_vis = t4_num_vis; 783 else 784 num_vis = 1; 785 #ifdef DEV_NETMAP 786 num_vis++; 787 #endif 788 if (num_vis > nitems(vi_mac_funcs)) { 789 num_vis = nitems(vi_mac_funcs); 790 device_printf(dev, "Number of VIs limited to %d\n", num_vis); 791 } 792 793 /* 794 * First pass over all the ports - allocate VIs and initialize some 795 * basic parameters like mac address, port type, etc. We also figure 796 * out whether a port is 10G or 1G and use that information when 797 * calculating how many interrupts to attempt to allocate. 798 */ 799 n10g = n1g = 0; 800 for_each_port(sc, i) { 801 struct port_info *pi; 802 struct vi_info *vi; 803 804 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 805 sc->port[i] = pi; 806 807 /* These must be set before t4_port_init */ 808 pi->adapter = sc; 809 pi->port_id = i; 810 pi->nvi = num_vis; 811 pi->vi = malloc(sizeof(struct vi_info) * num_vis, M_CXGBE, 812 M_ZERO | M_WAITOK); 813 814 /* 815 * Allocate the "main" VI and initialize parameters 816 * like mac addr. 817 */ 818 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 819 if (rc != 0) { 820 device_printf(dev, "unable to initialize port %d: %d\n", 821 i, rc); 822 free(pi->vi, M_CXGBE); 823 free(pi, M_CXGBE); 824 sc->port[i] = NULL; 825 goto done; 826 } 827 828 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX); 829 pi->link_cfg.requested_fc |= t4_pause_settings; 830 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX); 831 pi->link_cfg.fc |= t4_pause_settings; 832 833 rc = -t4_link_l1cfg(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 834 if (rc != 0) { 835 device_printf(dev, "port %d l1cfg failed: %d\n", i, rc); 836 free(pi->vi, M_CXGBE); 837 free(pi, M_CXGBE); 838 sc->port[i] = NULL; 839 goto done; 840 } 841 842 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 843 device_get_nameunit(dev), i); 844 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 845 sc->chan_map[pi->tx_chan] = i; 846 847 if (is_10G_port(pi) || is_40G_port(pi)) { 848 n10g++; 849 for_each_vi(pi, j, vi) { 850 vi->tmr_idx = t4_tmr_idx_10g; 851 vi->pktc_idx = t4_pktc_idx_10g; 852 } 853 } else { 854 n1g++; 855 for_each_vi(pi, j, vi) { 856 vi->tmr_idx = t4_tmr_idx_1g; 857 vi->pktc_idx = t4_pktc_idx_1g; 858 } 859 } 860 861 pi->linkdnrc = -1; 862 863 for_each_vi(pi, j, vi) { 864 vi->qsize_rxq = t4_qsize_rxq; 865 vi->qsize_txq = t4_qsize_txq; 866 vi->pi = pi; 867 } 868 869 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 870 if (pi->dev == NULL) { 871 device_printf(dev, 872 "failed to add device for port %d.\n", i); 873 rc = ENXIO; 874 goto done; 875 } 876 pi->vi[0].dev = pi->dev; 877 device_set_softc(pi->dev, pi); 878 } 879 880 /* 881 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 882 */ 883 #ifdef DEV_NETMAP 884 num_vis--; 885 #endif 886 rc = cfg_itype_and_nqueues(sc, n10g, n1g, num_vis, &iaq); 887 if (rc != 0) 888 goto done; /* error message displayed already */ 889 890 sc->intr_type = iaq.intr_type; 891 sc->intr_count = iaq.nirq; 892 893 s = &sc->sge; 894 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 895 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 896 if (num_vis > 1) { 897 s->nrxq += (n10g + n1g) * (num_vis - 1); 898 s->ntxq += (n10g + n1g) * (num_vis - 1); 899 } 900 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 901 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 902 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 903 #ifdef TCP_OFFLOAD 904 if (is_offload(sc)) { 905 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 906 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 907 if (num_vis > 1) { 908 s->nofldrxq += (n10g + n1g) * (num_vis - 1); 909 s->nofldtxq += (n10g + n1g) * (num_vis - 1); 910 } 911 s->neq += s->nofldtxq + s->nofldrxq; 912 s->niq += s->nofldrxq; 913 914 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 915 M_CXGBE, M_ZERO | M_WAITOK); 916 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 917 M_CXGBE, M_ZERO | M_WAITOK); 918 } 919 #endif 920 #ifdef DEV_NETMAP 921 s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g; 922 s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g; 923 s->neq += s->nnmtxq + s->nnmrxq; 924 s->niq += s->nnmrxq; 925 926 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq), 927 M_CXGBE, M_ZERO | M_WAITOK); 928 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq), 929 M_CXGBE, M_ZERO | M_WAITOK); 930 #endif 931 932 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 933 M_ZERO | M_WAITOK); 934 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 935 M_ZERO | M_WAITOK); 936 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 937 M_ZERO | M_WAITOK); 938 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 939 M_ZERO | M_WAITOK); 940 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 941 M_ZERO | M_WAITOK); 942 943 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 944 M_ZERO | M_WAITOK); 945 946 t4_init_l2t(sc, M_WAITOK); 947 948 /* 949 * Second pass over the ports. This time we know the number of rx and 950 * tx queues that each port should get. 951 */ 952 rqidx = tqidx = 0; 953 #ifdef TCP_OFFLOAD 954 ofld_rqidx = ofld_tqidx = 0; 955 #endif 956 #ifdef DEV_NETMAP 957 nm_rqidx = nm_tqidx = 0; 958 #endif 959 for_each_port(sc, i) { 960 struct port_info *pi = sc->port[i]; 961 struct vi_info *vi; 962 963 if (pi == NULL) 964 continue; 965 966 for_each_vi(pi, j, vi) { 967 #ifdef DEV_NETMAP 968 if (j == 1) { 969 vi->flags |= VI_NETMAP | INTR_RXQ; 970 vi->first_rxq = nm_rqidx; 971 vi->first_txq = nm_tqidx; 972 if (is_10G_port(pi) || is_40G_port(pi)) { 973 vi->nrxq = iaq.nnmrxq10g; 974 vi->ntxq = iaq.nnmtxq10g; 975 } else { 976 vi->nrxq = iaq.nnmrxq1g; 977 vi->ntxq = iaq.nnmtxq1g; 978 } 979 nm_rqidx += vi->nrxq; 980 nm_tqidx += vi->ntxq; 981 continue; 982 } 983 #endif 984 985 vi->first_rxq = rqidx; 986 vi->first_txq = tqidx; 987 if (is_10G_port(pi) || is_40G_port(pi)) { 988 vi->flags |= iaq.intr_flags_10g & INTR_RXQ; 989 vi->nrxq = j == 0 ? iaq.nrxq10g : 1; 990 vi->ntxq = j == 0 ? iaq.ntxq10g : 1; 991 } else { 992 vi->flags |= iaq.intr_flags_1g & INTR_RXQ; 993 vi->nrxq = j == 0 ? iaq.nrxq1g : 1; 994 vi->ntxq = j == 0 ? iaq.ntxq1g : 1; 995 } 996 997 if (vi->ntxq > 1) 998 vi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0; 999 else 1000 vi->rsrv_noflowq = 0; 1001 1002 rqidx += vi->nrxq; 1003 tqidx += vi->ntxq; 1004 1005 #ifdef TCP_OFFLOAD 1006 if (!is_offload(sc)) 1007 continue; 1008 vi->first_ofld_rxq = ofld_rqidx; 1009 vi->first_ofld_txq = ofld_tqidx; 1010 if (is_10G_port(pi) || is_40G_port(pi)) { 1011 vi->flags |= iaq.intr_flags_10g & INTR_OFLD_RXQ; 1012 vi->nofldrxq = j == 0 ? iaq.nofldrxq10g : 1; 1013 vi->nofldtxq = j == 0 ? iaq.nofldtxq10g : 1; 1014 } else { 1015 vi->flags |= iaq.intr_flags_1g & INTR_OFLD_RXQ; 1016 vi->nofldrxq = j == 0 ? iaq.nofldrxq1g : 1; 1017 vi->nofldtxq = j == 0 ? iaq.nofldtxq1g : 1; 1018 } 1019 ofld_rqidx += vi->nofldrxq; 1020 ofld_tqidx += vi->nofldtxq; 1021 #endif 1022 } 1023 } 1024 1025 rc = setup_intr_handlers(sc); 1026 if (rc != 0) { 1027 device_printf(dev, 1028 "failed to setup interrupt handlers: %d\n", rc); 1029 goto done; 1030 } 1031 1032 rc = bus_generic_attach(dev); 1033 if (rc != 0) { 1034 device_printf(dev, 1035 "failed to attach all child ports: %d\n", rc); 1036 goto done; 1037 } 1038 1039 device_printf(dev, 1040 "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 1041 sc->params.pci.speed, sc->params.pci.width, sc->params.nports, 1042 sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" : 1043 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 1044 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 1045 1046 t4_set_desc(sc); 1047 1048 done: 1049 if (rc != 0 && sc->cdev) { 1050 /* cdev was created and so cxgbetool works; recover that way. */ 1051 device_printf(dev, 1052 "error during attach, adapter is now in recovery mode.\n"); 1053 rc = 0; 1054 } 1055 1056 if (rc != 0) 1057 t4_detach(dev); 1058 else 1059 t4_sysctls(sc); 1060 1061 return (rc); 1062 } 1063 1064 /* 1065 * Idempotent 1066 */ 1067 static int 1068 t4_detach(device_t dev) 1069 { 1070 struct adapter *sc; 1071 struct port_info *pi; 1072 int i, rc; 1073 1074 sc = device_get_softc(dev); 1075 1076 if (sc->flags & FULL_INIT_DONE) 1077 t4_intr_disable(sc); 1078 1079 if (sc->cdev) { 1080 destroy_dev(sc->cdev); 1081 sc->cdev = NULL; 1082 } 1083 1084 rc = bus_generic_detach(dev); 1085 if (rc) { 1086 device_printf(dev, 1087 "failed to detach child devices: %d\n", rc); 1088 return (rc); 1089 } 1090 1091 for (i = 0; i < sc->intr_count; i++) 1092 t4_free_irq(sc, &sc->irq[i]); 1093 1094 for (i = 0; i < MAX_NPORTS; i++) { 1095 pi = sc->port[i]; 1096 if (pi) { 1097 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->vi[0].viid); 1098 if (pi->dev) 1099 device_delete_child(dev, pi->dev); 1100 1101 mtx_destroy(&pi->pi_lock); 1102 free(pi->vi, M_CXGBE); 1103 free(pi, M_CXGBE); 1104 } 1105 } 1106 1107 if (sc->flags & FULL_INIT_DONE) 1108 adapter_full_uninit(sc); 1109 1110 if (sc->flags & FW_OK) 1111 t4_fw_bye(sc, sc->mbox); 1112 1113 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 1114 pci_release_msi(dev); 1115 1116 if (sc->regs_res) 1117 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 1118 sc->regs_res); 1119 1120 if (sc->udbs_res) 1121 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 1122 sc->udbs_res); 1123 1124 if (sc->msix_res) 1125 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 1126 sc->msix_res); 1127 1128 if (sc->l2t) 1129 t4_free_l2t(sc->l2t); 1130 1131 #ifdef TCP_OFFLOAD 1132 free(sc->sge.ofld_rxq, M_CXGBE); 1133 free(sc->sge.ofld_txq, M_CXGBE); 1134 #endif 1135 #ifdef DEV_NETMAP 1136 free(sc->sge.nm_rxq, M_CXGBE); 1137 free(sc->sge.nm_txq, M_CXGBE); 1138 #endif 1139 free(sc->irq, M_CXGBE); 1140 free(sc->sge.rxq, M_CXGBE); 1141 free(sc->sge.txq, M_CXGBE); 1142 free(sc->sge.ctrlq, M_CXGBE); 1143 free(sc->sge.iqmap, M_CXGBE); 1144 free(sc->sge.eqmap, M_CXGBE); 1145 free(sc->tids.ftid_tab, M_CXGBE); 1146 t4_destroy_dma_tag(sc); 1147 if (mtx_initialized(&sc->sc_lock)) { 1148 sx_xlock(&t4_list_lock); 1149 SLIST_REMOVE(&t4_list, sc, adapter, link); 1150 sx_xunlock(&t4_list_lock); 1151 mtx_destroy(&sc->sc_lock); 1152 } 1153 1154 callout_drain(&sc->sfl_callout); 1155 if (mtx_initialized(&sc->tids.ftid_lock)) 1156 mtx_destroy(&sc->tids.ftid_lock); 1157 if (mtx_initialized(&sc->sfl_lock)) 1158 mtx_destroy(&sc->sfl_lock); 1159 if (mtx_initialized(&sc->ifp_lock)) 1160 mtx_destroy(&sc->ifp_lock); 1161 if (mtx_initialized(&sc->regwin_lock)) 1162 mtx_destroy(&sc->regwin_lock); 1163 1164 bzero(sc, sizeof(*sc)); 1165 1166 return (0); 1167 } 1168 1169 static int 1170 cxgbe_probe(device_t dev) 1171 { 1172 char buf[128]; 1173 struct port_info *pi = device_get_softc(dev); 1174 1175 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 1176 device_set_desc_copy(dev, buf); 1177 1178 return (BUS_PROBE_DEFAULT); 1179 } 1180 1181 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 1182 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 1183 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS) 1184 #define T4_CAP_ENABLE (T4_CAP) 1185 1186 static int 1187 cxgbe_vi_attach(device_t dev, struct vi_info *vi) 1188 { 1189 struct ifnet *ifp; 1190 struct sbuf *sb; 1191 1192 vi->xact_addr_filt = -1; 1193 callout_init(&vi->tick, 1); 1194 1195 /* Allocate an ifnet and set it up */ 1196 ifp = if_alloc(IFT_ETHER); 1197 if (ifp == NULL) { 1198 device_printf(dev, "Cannot allocate ifnet\n"); 1199 return (ENOMEM); 1200 } 1201 vi->ifp = ifp; 1202 ifp->if_softc = vi; 1203 1204 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1205 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1206 1207 ifp->if_init = cxgbe_init; 1208 ifp->if_ioctl = cxgbe_ioctl; 1209 ifp->if_transmit = cxgbe_transmit; 1210 ifp->if_qflush = cxgbe_qflush; 1211 ifp->if_get_counter = cxgbe_get_counter; 1212 1213 ifp->if_capabilities = T4_CAP; 1214 #ifdef TCP_OFFLOAD 1215 if (vi->nofldrxq != 0) 1216 ifp->if_capabilities |= IFCAP_TOE; 1217 #endif 1218 ifp->if_capenable = T4_CAP_ENABLE; 1219 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1220 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1221 1222 ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1223 ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS; 1224 ifp->if_hw_tsomaxsegsize = 65536; 1225 1226 /* Initialize ifmedia for this VI */ 1227 ifmedia_init(&vi->media, IFM_IMASK, cxgbe_media_change, 1228 cxgbe_media_status); 1229 build_medialist(vi->pi, &vi->media); 1230 1231 vi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 1232 EVENTHANDLER_PRI_ANY); 1233 1234 ether_ifattach(ifp, vi->hw_addr); 1235 1236 sb = sbuf_new_auto(); 1237 sbuf_printf(sb, "%d txq, %d rxq (NIC)", vi->ntxq, vi->nrxq); 1238 #ifdef TCP_OFFLOAD 1239 if (ifp->if_capabilities & IFCAP_TOE) 1240 sbuf_printf(sb, "; %d txq, %d rxq (TOE)", 1241 vi->nofldtxq, vi->nofldrxq); 1242 #endif 1243 sbuf_finish(sb); 1244 device_printf(dev, "%s\n", sbuf_data(sb)); 1245 sbuf_delete(sb); 1246 1247 vi_sysctls(vi); 1248 1249 return (0); 1250 } 1251 1252 static int 1253 cxgbe_attach(device_t dev) 1254 { 1255 struct port_info *pi = device_get_softc(dev); 1256 struct vi_info *vi; 1257 int i, rc; 1258 1259 callout_init_mtx(&pi->tick, &pi->pi_lock, 0); 1260 1261 rc = cxgbe_vi_attach(dev, &pi->vi[0]); 1262 if (rc) 1263 return (rc); 1264 1265 for_each_vi(pi, i, vi) { 1266 if (i == 0) 1267 continue; 1268 #ifdef DEV_NETMAP 1269 if (vi->flags & VI_NETMAP) { 1270 /* 1271 * media handled here to keep 1272 * implementation private to this file 1273 */ 1274 ifmedia_init(&vi->media, IFM_IMASK, cxgbe_media_change, 1275 cxgbe_media_status); 1276 build_medialist(pi, &vi->media); 1277 vi->dev = device_add_child(dev, is_t4(pi->adapter) ? 1278 "ncxgbe" : "ncxl", device_get_unit(dev)); 1279 } else 1280 #endif 1281 vi->dev = device_add_child(dev, is_t4(pi->adapter) ? 1282 "vcxgbe" : "vcxl", -1); 1283 if (vi->dev == NULL) { 1284 device_printf(dev, "failed to add VI %d\n", i); 1285 continue; 1286 } 1287 device_set_softc(vi->dev, vi); 1288 } 1289 1290 cxgbe_sysctls(pi); 1291 1292 bus_generic_attach(dev); 1293 1294 return (0); 1295 } 1296 1297 static void 1298 cxgbe_vi_detach(struct vi_info *vi) 1299 { 1300 struct ifnet *ifp = vi->ifp; 1301 1302 ether_ifdetach(ifp); 1303 1304 if (vi->vlan_c) 1305 EVENTHANDLER_DEREGISTER(vlan_config, vi->vlan_c); 1306 1307 /* Let detach proceed even if these fail. */ 1308 cxgbe_uninit_synchronized(vi); 1309 callout_drain(&vi->tick); 1310 vi_full_uninit(vi); 1311 1312 ifmedia_removeall(&vi->media); 1313 if_free(vi->ifp); 1314 vi->ifp = NULL; 1315 } 1316 1317 static int 1318 cxgbe_detach(device_t dev) 1319 { 1320 struct port_info *pi = device_get_softc(dev); 1321 struct adapter *sc = pi->adapter; 1322 int rc; 1323 1324 /* Detach the extra VIs first. */ 1325 rc = bus_generic_detach(dev); 1326 if (rc) 1327 return (rc); 1328 device_delete_children(dev); 1329 1330 doom_vi(sc, &pi->vi[0]); 1331 1332 if (pi->flags & HAS_TRACEQ) { 1333 sc->traceq = -1; /* cloner should not create ifnet */ 1334 t4_tracer_port_detach(sc); 1335 } 1336 1337 cxgbe_vi_detach(&pi->vi[0]); 1338 callout_drain(&pi->tick); 1339 1340 end_synchronized_op(sc, 0); 1341 1342 return (0); 1343 } 1344 1345 static void 1346 cxgbe_init(void *arg) 1347 { 1348 struct vi_info *vi = arg; 1349 struct adapter *sc = vi->pi->adapter; 1350 1351 if (begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4init") != 0) 1352 return; 1353 cxgbe_init_synchronized(vi); 1354 end_synchronized_op(sc, 0); 1355 } 1356 1357 static int 1358 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1359 { 1360 int rc = 0, mtu, flags, can_sleep; 1361 struct vi_info *vi = ifp->if_softc; 1362 struct adapter *sc = vi->pi->adapter; 1363 struct ifreq *ifr = (struct ifreq *)data; 1364 uint32_t mask; 1365 1366 switch (cmd) { 1367 case SIOCSIFMTU: 1368 mtu = ifr->ifr_mtu; 1369 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1370 return (EINVAL); 1371 1372 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4mtu"); 1373 if (rc) 1374 return (rc); 1375 ifp->if_mtu = mtu; 1376 if (vi->flags & VI_INIT_DONE) { 1377 t4_update_fl_bufsize(ifp); 1378 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1379 rc = update_mac_settings(ifp, XGMAC_MTU); 1380 } 1381 end_synchronized_op(sc, 0); 1382 break; 1383 1384 case SIOCSIFFLAGS: 1385 can_sleep = 0; 1386 redo_sifflags: 1387 rc = begin_synchronized_op(sc, vi, 1388 can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg"); 1389 if (rc) 1390 return (rc); 1391 1392 if (ifp->if_flags & IFF_UP) { 1393 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1394 flags = vi->if_flags; 1395 if ((ifp->if_flags ^ flags) & 1396 (IFF_PROMISC | IFF_ALLMULTI)) { 1397 if (can_sleep == 1) { 1398 end_synchronized_op(sc, 0); 1399 can_sleep = 0; 1400 goto redo_sifflags; 1401 } 1402 rc = update_mac_settings(ifp, 1403 XGMAC_PROMISC | XGMAC_ALLMULTI); 1404 } 1405 } else { 1406 if (can_sleep == 0) { 1407 end_synchronized_op(sc, LOCK_HELD); 1408 can_sleep = 1; 1409 goto redo_sifflags; 1410 } 1411 rc = cxgbe_init_synchronized(vi); 1412 } 1413 vi->if_flags = ifp->if_flags; 1414 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1415 if (can_sleep == 0) { 1416 end_synchronized_op(sc, LOCK_HELD); 1417 can_sleep = 1; 1418 goto redo_sifflags; 1419 } 1420 rc = cxgbe_uninit_synchronized(vi); 1421 } 1422 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD); 1423 break; 1424 1425 case SIOCADDMULTI: 1426 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1427 rc = begin_synchronized_op(sc, vi, HOLD_LOCK, "t4multi"); 1428 if (rc) 1429 return (rc); 1430 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1431 rc = update_mac_settings(ifp, XGMAC_MCADDRS); 1432 end_synchronized_op(sc, LOCK_HELD); 1433 break; 1434 1435 case SIOCSIFCAP: 1436 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4cap"); 1437 if (rc) 1438 return (rc); 1439 1440 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1441 if (mask & IFCAP_TXCSUM) { 1442 ifp->if_capenable ^= IFCAP_TXCSUM; 1443 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1444 1445 if (IFCAP_TSO4 & ifp->if_capenable && 1446 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1447 ifp->if_capenable &= ~IFCAP_TSO4; 1448 if_printf(ifp, 1449 "tso4 disabled due to -txcsum.\n"); 1450 } 1451 } 1452 if (mask & IFCAP_TXCSUM_IPV6) { 1453 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1454 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1455 1456 if (IFCAP_TSO6 & ifp->if_capenable && 1457 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1458 ifp->if_capenable &= ~IFCAP_TSO6; 1459 if_printf(ifp, 1460 "tso6 disabled due to -txcsum6.\n"); 1461 } 1462 } 1463 if (mask & IFCAP_RXCSUM) 1464 ifp->if_capenable ^= IFCAP_RXCSUM; 1465 if (mask & IFCAP_RXCSUM_IPV6) 1466 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1467 1468 /* 1469 * Note that we leave CSUM_TSO alone (it is always set). The 1470 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1471 * sending a TSO request our way, so it's sufficient to toggle 1472 * IFCAP_TSOx only. 1473 */ 1474 if (mask & IFCAP_TSO4) { 1475 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1476 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1477 if_printf(ifp, "enable txcsum first.\n"); 1478 rc = EAGAIN; 1479 goto fail; 1480 } 1481 ifp->if_capenable ^= IFCAP_TSO4; 1482 } 1483 if (mask & IFCAP_TSO6) { 1484 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1485 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1486 if_printf(ifp, "enable txcsum6 first.\n"); 1487 rc = EAGAIN; 1488 goto fail; 1489 } 1490 ifp->if_capenable ^= IFCAP_TSO6; 1491 } 1492 if (mask & IFCAP_LRO) { 1493 #if defined(INET) || defined(INET6) 1494 int i; 1495 struct sge_rxq *rxq; 1496 1497 ifp->if_capenable ^= IFCAP_LRO; 1498 for_each_rxq(vi, i, rxq) { 1499 if (ifp->if_capenable & IFCAP_LRO) 1500 rxq->iq.flags |= IQ_LRO_ENABLED; 1501 else 1502 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1503 } 1504 #endif 1505 } 1506 #ifdef TCP_OFFLOAD 1507 if (mask & IFCAP_TOE) { 1508 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1509 1510 rc = toe_capability(vi, enable); 1511 if (rc != 0) 1512 goto fail; 1513 1514 ifp->if_capenable ^= mask; 1515 } 1516 #endif 1517 if (mask & IFCAP_VLAN_HWTAGGING) { 1518 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1519 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1520 rc = update_mac_settings(ifp, XGMAC_VLANEX); 1521 } 1522 if (mask & IFCAP_VLAN_MTU) { 1523 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1524 1525 /* Need to find out how to disable auto-mtu-inflation */ 1526 } 1527 if (mask & IFCAP_VLAN_HWTSO) 1528 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1529 if (mask & IFCAP_VLAN_HWCSUM) 1530 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1531 1532 #ifdef VLAN_CAPABILITIES 1533 VLAN_CAPABILITIES(ifp); 1534 #endif 1535 fail: 1536 end_synchronized_op(sc, 0); 1537 break; 1538 1539 case SIOCSIFMEDIA: 1540 case SIOCGIFMEDIA: 1541 ifmedia_ioctl(ifp, ifr, &vi->media, cmd); 1542 break; 1543 1544 case SIOCGI2C: { 1545 struct ifi2creq i2c; 1546 1547 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); 1548 if (rc != 0) 1549 break; 1550 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) { 1551 rc = EPERM; 1552 break; 1553 } 1554 if (i2c.len > sizeof(i2c.data)) { 1555 rc = EINVAL; 1556 break; 1557 } 1558 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4i2c"); 1559 if (rc) 1560 return (rc); 1561 rc = -t4_i2c_rd(sc, sc->mbox, vi->pi->port_id, i2c.dev_addr, 1562 i2c.offset, i2c.len, &i2c.data[0]); 1563 end_synchronized_op(sc, 0); 1564 if (rc == 0) 1565 rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); 1566 break; 1567 } 1568 1569 default: 1570 rc = ether_ioctl(ifp, cmd, data); 1571 } 1572 1573 return (rc); 1574 } 1575 1576 static int 1577 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1578 { 1579 struct vi_info *vi = ifp->if_softc; 1580 struct port_info *pi = vi->pi; 1581 struct adapter *sc = pi->adapter; 1582 struct sge_txq *txq; 1583 void *items[1]; 1584 int rc; 1585 1586 M_ASSERTPKTHDR(m); 1587 MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */ 1588 1589 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1590 m_freem(m); 1591 return (ENETDOWN); 1592 } 1593 1594 rc = parse_pkt(&m); 1595 if (__predict_false(rc != 0)) { 1596 MPASS(m == NULL); /* was freed already */ 1597 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */ 1598 return (rc); 1599 } 1600 1601 /* Select a txq. */ 1602 txq = &sc->sge.txq[vi->first_txq]; 1603 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 1604 txq += ((m->m_pkthdr.flowid % (vi->ntxq - vi->rsrv_noflowq)) + 1605 vi->rsrv_noflowq); 1606 1607 items[0] = m; 1608 rc = mp_ring_enqueue(txq->r, items, 1, 4096); 1609 if (__predict_false(rc != 0)) 1610 m_freem(m); 1611 1612 return (rc); 1613 } 1614 1615 static void 1616 cxgbe_qflush(struct ifnet *ifp) 1617 { 1618 struct vi_info *vi = ifp->if_softc; 1619 struct sge_txq *txq; 1620 int i; 1621 1622 /* queues do not exist if !VI_INIT_DONE. */ 1623 if (vi->flags & VI_INIT_DONE) { 1624 for_each_txq(vi, i, txq) { 1625 TXQ_LOCK(txq); 1626 txq->eq.flags &= ~EQ_ENABLED; 1627 TXQ_UNLOCK(txq); 1628 while (!mp_ring_is_idle(txq->r)) { 1629 mp_ring_check_drainage(txq->r, 0); 1630 pause("qflush", 1); 1631 } 1632 } 1633 } 1634 if_qflush(ifp); 1635 } 1636 1637 static uint64_t 1638 vi_get_counter(struct ifnet *ifp, ift_counter c) 1639 { 1640 struct vi_info *vi = ifp->if_softc; 1641 struct fw_vi_stats_vf *s = &vi->stats; 1642 1643 vi_refresh_stats(vi->pi->adapter, vi); 1644 1645 switch (c) { 1646 case IFCOUNTER_IPACKETS: 1647 return (s->rx_bcast_frames + s->rx_mcast_frames + 1648 s->rx_ucast_frames); 1649 case IFCOUNTER_IERRORS: 1650 return (s->rx_err_frames); 1651 case IFCOUNTER_OPACKETS: 1652 return (s->tx_bcast_frames + s->tx_mcast_frames + 1653 s->tx_ucast_frames + s->tx_offload_frames); 1654 case IFCOUNTER_OERRORS: 1655 return (s->tx_drop_frames); 1656 case IFCOUNTER_IBYTES: 1657 return (s->rx_bcast_bytes + s->rx_mcast_bytes + 1658 s->rx_ucast_bytes); 1659 case IFCOUNTER_OBYTES: 1660 return (s->tx_bcast_bytes + s->tx_mcast_bytes + 1661 s->tx_ucast_bytes + s->tx_offload_bytes); 1662 case IFCOUNTER_IMCASTS: 1663 return (s->rx_mcast_frames); 1664 case IFCOUNTER_OMCASTS: 1665 return (s->tx_mcast_frames); 1666 case IFCOUNTER_OQDROPS: { 1667 uint64_t drops; 1668 1669 drops = 0; 1670 if ((vi->flags & (VI_INIT_DONE | VI_NETMAP)) == VI_INIT_DONE) { 1671 int i; 1672 struct sge_txq *txq; 1673 1674 for_each_txq(vi, i, txq) 1675 drops += counter_u64_fetch(txq->r->drops); 1676 } 1677 1678 return (drops); 1679 1680 } 1681 1682 default: 1683 return (if_get_counter_default(ifp, c)); 1684 } 1685 } 1686 1687 uint64_t 1688 cxgbe_get_counter(struct ifnet *ifp, ift_counter c) 1689 { 1690 struct vi_info *vi = ifp->if_softc; 1691 struct port_info *pi = vi->pi; 1692 struct adapter *sc = pi->adapter; 1693 struct port_stats *s = &pi->stats; 1694 1695 if (pi->nvi > 1) 1696 return (vi_get_counter(ifp, c)); 1697 1698 cxgbe_refresh_stats(sc, pi); 1699 1700 switch (c) { 1701 case IFCOUNTER_IPACKETS: 1702 return (s->rx_frames - s->rx_pause); 1703 1704 case IFCOUNTER_IERRORS: 1705 return (s->rx_jabber + s->rx_runt + s->rx_too_long + 1706 s->rx_fcs_err + s->rx_len_err); 1707 1708 case IFCOUNTER_OPACKETS: 1709 return (s->tx_frames - s->tx_pause); 1710 1711 case IFCOUNTER_OERRORS: 1712 return (s->tx_error_frames); 1713 1714 case IFCOUNTER_IBYTES: 1715 return (s->rx_octets - s->rx_pause * 64); 1716 1717 case IFCOUNTER_OBYTES: 1718 return (s->tx_octets - s->tx_pause * 64); 1719 1720 case IFCOUNTER_IMCASTS: 1721 return (s->rx_mcast_frames - s->rx_pause); 1722 1723 case IFCOUNTER_OMCASTS: 1724 return (s->tx_mcast_frames - s->tx_pause); 1725 1726 case IFCOUNTER_IQDROPS: 1727 return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 1728 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 1729 s->rx_trunc3 + pi->tnl_cong_drops); 1730 1731 case IFCOUNTER_OQDROPS: { 1732 uint64_t drops; 1733 1734 drops = s->tx_drop; 1735 if (vi->flags & VI_INIT_DONE) { 1736 int i; 1737 struct sge_txq *txq; 1738 1739 for_each_txq(vi, i, txq) 1740 drops += counter_u64_fetch(txq->r->drops); 1741 } 1742 1743 return (drops); 1744 1745 } 1746 1747 default: 1748 return (if_get_counter_default(ifp, c)); 1749 } 1750 } 1751 1752 static int 1753 cxgbe_media_change(struct ifnet *ifp) 1754 { 1755 struct vi_info *vi = ifp->if_softc; 1756 1757 device_printf(vi->dev, "%s unimplemented.\n", __func__); 1758 1759 return (EOPNOTSUPP); 1760 } 1761 1762 static void 1763 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1764 { 1765 struct vi_info *vi = ifp->if_softc; 1766 struct port_info *pi = vi->pi; 1767 struct ifmedia_entry *cur; 1768 int speed = pi->link_cfg.speed; 1769 1770 cur = vi->media.ifm_cur; 1771 1772 ifmr->ifm_status = IFM_AVALID; 1773 if (!pi->link_cfg.link_ok) 1774 return; 1775 1776 ifmr->ifm_status |= IFM_ACTIVE; 1777 1778 /* active and current will differ iff current media is autoselect. */ 1779 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1780 return; 1781 1782 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1783 if (speed == SPEED_10000) 1784 ifmr->ifm_active |= IFM_10G_T; 1785 else if (speed == SPEED_1000) 1786 ifmr->ifm_active |= IFM_1000_T; 1787 else if (speed == SPEED_100) 1788 ifmr->ifm_active |= IFM_100_TX; 1789 else if (speed == SPEED_10) 1790 ifmr->ifm_active |= IFM_10_T; 1791 else 1792 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1793 speed)); 1794 } 1795 1796 static int 1797 vcxgbe_probe(device_t dev) 1798 { 1799 char buf[128]; 1800 struct vi_info *vi = device_get_softc(dev); 1801 1802 snprintf(buf, sizeof(buf), "port %d vi %td", vi->pi->port_id, 1803 vi - vi->pi->vi); 1804 device_set_desc_copy(dev, buf); 1805 1806 return (BUS_PROBE_DEFAULT); 1807 } 1808 1809 static int 1810 vcxgbe_attach(device_t dev) 1811 { 1812 struct vi_info *vi; 1813 struct port_info *pi; 1814 struct adapter *sc; 1815 int func, index, rc; 1816 u32 param, val; 1817 1818 vi = device_get_softc(dev); 1819 pi = vi->pi; 1820 sc = pi->adapter; 1821 1822 index = vi - pi->vi; 1823 KASSERT(index < nitems(vi_mac_funcs), 1824 ("%s: VI %s doesn't have a MAC func", __func__, 1825 device_get_nameunit(dev))); 1826 func = vi_mac_funcs[index]; 1827 rc = t4_alloc_vi_func(sc, sc->mbox, pi->tx_chan, sc->pf, 0, 1, 1828 vi->hw_addr, &vi->rss_size, func, 0); 1829 if (rc < 0) { 1830 device_printf(dev, "Failed to allocate virtual interface " 1831 "for port %d: %d\n", pi->port_id, -rc); 1832 return (-rc); 1833 } 1834 vi->viid = rc; 1835 1836 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 1837 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_RSSINFO) | 1838 V_FW_PARAMS_PARAM_YZ(vi->viid); 1839 rc = t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 1840 if (rc) 1841 vi->rss_base = 0xffff; 1842 else { 1843 /* MPASS((val >> 16) == rss_size); */ 1844 vi->rss_base = val & 0xffff; 1845 } 1846 1847 rc = cxgbe_vi_attach(dev, vi); 1848 if (rc) { 1849 t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid); 1850 return (rc); 1851 } 1852 return (0); 1853 } 1854 1855 static int 1856 vcxgbe_detach(device_t dev) 1857 { 1858 struct vi_info *vi; 1859 struct adapter *sc; 1860 1861 vi = device_get_softc(dev); 1862 sc = vi->pi->adapter; 1863 1864 doom_vi(sc, vi); 1865 1866 cxgbe_vi_detach(vi); 1867 t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid); 1868 1869 end_synchronized_op(sc, 0); 1870 1871 return (0); 1872 } 1873 1874 void 1875 t4_fatal_err(struct adapter *sc) 1876 { 1877 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1878 t4_intr_disable(sc); 1879 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1880 device_get_nameunit(sc->dev)); 1881 } 1882 1883 static int 1884 map_bars_0_and_4(struct adapter *sc) 1885 { 1886 sc->regs_rid = PCIR_BAR(0); 1887 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1888 &sc->regs_rid, RF_ACTIVE); 1889 if (sc->regs_res == NULL) { 1890 device_printf(sc->dev, "cannot map registers.\n"); 1891 return (ENXIO); 1892 } 1893 sc->bt = rman_get_bustag(sc->regs_res); 1894 sc->bh = rman_get_bushandle(sc->regs_res); 1895 sc->mmio_len = rman_get_size(sc->regs_res); 1896 setbit(&sc->doorbells, DOORBELL_KDB); 1897 1898 sc->msix_rid = PCIR_BAR(4); 1899 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1900 &sc->msix_rid, RF_ACTIVE); 1901 if (sc->msix_res == NULL) { 1902 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1903 return (ENXIO); 1904 } 1905 1906 return (0); 1907 } 1908 1909 static int 1910 map_bar_2(struct adapter *sc) 1911 { 1912 1913 /* 1914 * T4: only iWARP driver uses the userspace doorbells. There is no need 1915 * to map it if RDMA is disabled. 1916 */ 1917 if (is_t4(sc) && sc->rdmacaps == 0) 1918 return (0); 1919 1920 sc->udbs_rid = PCIR_BAR(2); 1921 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1922 &sc->udbs_rid, RF_ACTIVE); 1923 if (sc->udbs_res == NULL) { 1924 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1925 return (ENXIO); 1926 } 1927 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1928 1929 if (is_t5(sc)) { 1930 setbit(&sc->doorbells, DOORBELL_UDB); 1931 #if defined(__i386__) || defined(__amd64__) 1932 if (t5_write_combine) { 1933 int rc; 1934 1935 /* 1936 * Enable write combining on BAR2. This is the 1937 * userspace doorbell BAR and is split into 128B 1938 * (UDBS_SEG_SIZE) doorbell regions, each associated 1939 * with an egress queue. The first 64B has the doorbell 1940 * and the second 64B can be used to submit a tx work 1941 * request with an implicit doorbell. 1942 */ 1943 1944 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1945 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1946 if (rc == 0) { 1947 clrbit(&sc->doorbells, DOORBELL_UDB); 1948 setbit(&sc->doorbells, DOORBELL_WCWR); 1949 setbit(&sc->doorbells, DOORBELL_UDBWC); 1950 } else { 1951 device_printf(sc->dev, 1952 "couldn't enable write combining: %d\n", 1953 rc); 1954 } 1955 1956 t4_write_reg(sc, A_SGE_STAT_CFG, 1957 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1958 } 1959 #endif 1960 } 1961 1962 return (0); 1963 } 1964 1965 static const struct memwin t4_memwin[] = { 1966 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1967 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1968 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1969 }; 1970 1971 static const struct memwin t5_memwin[] = { 1972 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1973 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1974 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1975 }; 1976 1977 static void 1978 setup_memwin(struct adapter *sc) 1979 { 1980 const struct memwin *mw; 1981 int i, n; 1982 uint32_t bar0; 1983 1984 if (is_t4(sc)) { 1985 /* 1986 * Read low 32b of bar0 indirectly via the hardware backdoor 1987 * mechanism. Works from within PCI passthrough environments 1988 * too, where rman_get_start() can return a different value. We 1989 * need to program the T4 memory window decoders with the actual 1990 * addresses that will be coming across the PCIe link. 1991 */ 1992 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1993 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1994 1995 mw = &t4_memwin[0]; 1996 n = nitems(t4_memwin); 1997 } else { 1998 /* T5 uses the relative offset inside the PCIe BAR */ 1999 bar0 = 0; 2000 2001 mw = &t5_memwin[0]; 2002 n = nitems(t5_memwin); 2003 } 2004 2005 for (i = 0; i < n; i++, mw++) { 2006 t4_write_reg(sc, 2007 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 2008 (mw->base + bar0) | V_BIR(0) | 2009 V_WINDOW(ilog2(mw->aperture) - 10)); 2010 } 2011 2012 /* flush */ 2013 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 2014 } 2015 2016 /* 2017 * Verify that the memory range specified by the addr/len pair is valid and lies 2018 * entirely within a single region (EDCx or MCx). 2019 */ 2020 static int 2021 validate_mem_range(struct adapter *sc, uint32_t addr, int len) 2022 { 2023 uint32_t em, addr_len, maddr, mlen; 2024 2025 /* Memory can only be accessed in naturally aligned 4 byte units */ 2026 if (addr & 3 || len & 3 || len == 0) 2027 return (EINVAL); 2028 2029 /* Enabled memories */ 2030 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 2031 if (em & F_EDRAM0_ENABLE) { 2032 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 2033 maddr = G_EDRAM0_BASE(addr_len) << 20; 2034 mlen = G_EDRAM0_SIZE(addr_len) << 20; 2035 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 2036 addr + len <= maddr + mlen) 2037 return (0); 2038 } 2039 if (em & F_EDRAM1_ENABLE) { 2040 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 2041 maddr = G_EDRAM1_BASE(addr_len) << 20; 2042 mlen = G_EDRAM1_SIZE(addr_len) << 20; 2043 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 2044 addr + len <= maddr + mlen) 2045 return (0); 2046 } 2047 if (em & F_EXT_MEM_ENABLE) { 2048 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 2049 maddr = G_EXT_MEM_BASE(addr_len) << 20; 2050 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 2051 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 2052 addr + len <= maddr + mlen) 2053 return (0); 2054 } 2055 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 2056 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 2057 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 2058 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 2059 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 2060 addr + len <= maddr + mlen) 2061 return (0); 2062 } 2063 2064 return (EFAULT); 2065 } 2066 2067 static int 2068 fwmtype_to_hwmtype(int mtype) 2069 { 2070 2071 switch (mtype) { 2072 case FW_MEMTYPE_EDC0: 2073 return (MEM_EDC0); 2074 case FW_MEMTYPE_EDC1: 2075 return (MEM_EDC1); 2076 case FW_MEMTYPE_EXTMEM: 2077 return (MEM_MC0); 2078 case FW_MEMTYPE_EXTMEM1: 2079 return (MEM_MC1); 2080 default: 2081 panic("%s: cannot translate fw mtype %d.", __func__, mtype); 2082 } 2083 } 2084 2085 /* 2086 * Verify that the memory range specified by the memtype/offset/len pair is 2087 * valid and lies entirely within the memtype specified. The global address of 2088 * the start of the range is returned in addr. 2089 */ 2090 static int 2091 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 2092 uint32_t *addr) 2093 { 2094 uint32_t em, addr_len, maddr, mlen; 2095 2096 /* Memory can only be accessed in naturally aligned 4 byte units */ 2097 if (off & 3 || len & 3 || len == 0) 2098 return (EINVAL); 2099 2100 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 2101 switch (fwmtype_to_hwmtype(mtype)) { 2102 case MEM_EDC0: 2103 if (!(em & F_EDRAM0_ENABLE)) 2104 return (EINVAL); 2105 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 2106 maddr = G_EDRAM0_BASE(addr_len) << 20; 2107 mlen = G_EDRAM0_SIZE(addr_len) << 20; 2108 break; 2109 case MEM_EDC1: 2110 if (!(em & F_EDRAM1_ENABLE)) 2111 return (EINVAL); 2112 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 2113 maddr = G_EDRAM1_BASE(addr_len) << 20; 2114 mlen = G_EDRAM1_SIZE(addr_len) << 20; 2115 break; 2116 case MEM_MC: 2117 if (!(em & F_EXT_MEM_ENABLE)) 2118 return (EINVAL); 2119 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 2120 maddr = G_EXT_MEM_BASE(addr_len) << 20; 2121 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 2122 break; 2123 case MEM_MC1: 2124 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 2125 return (EINVAL); 2126 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 2127 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 2128 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 2129 break; 2130 default: 2131 return (EINVAL); 2132 } 2133 2134 if (mlen > 0 && off < mlen && off + len <= mlen) { 2135 *addr = maddr + off; /* global address */ 2136 return (0); 2137 } 2138 2139 return (EFAULT); 2140 } 2141 2142 static void 2143 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 2144 { 2145 const struct memwin *mw; 2146 2147 if (is_t4(sc)) { 2148 KASSERT(win >= 0 && win < nitems(t4_memwin), 2149 ("%s: incorrect memwin# (%d)", __func__, win)); 2150 mw = &t4_memwin[win]; 2151 } else { 2152 KASSERT(win >= 0 && win < nitems(t5_memwin), 2153 ("%s: incorrect memwin# (%d)", __func__, win)); 2154 mw = &t5_memwin[win]; 2155 } 2156 2157 if (base != NULL) 2158 *base = mw->base; 2159 if (aperture != NULL) 2160 *aperture = mw->aperture; 2161 } 2162 2163 /* 2164 * Positions the memory window such that it can be used to access the specified 2165 * address in the chip's address space. The return value is the offset of addr 2166 * from the start of the window. 2167 */ 2168 static uint32_t 2169 position_memwin(struct adapter *sc, int n, uint32_t addr) 2170 { 2171 uint32_t start, pf; 2172 uint32_t reg; 2173 2174 KASSERT(n >= 0 && n <= 3, 2175 ("%s: invalid window %d.", __func__, n)); 2176 KASSERT((addr & 3) == 0, 2177 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 2178 2179 if (is_t4(sc)) { 2180 pf = 0; 2181 start = addr & ~0xf; /* start must be 16B aligned */ 2182 } else { 2183 pf = V_PFNUM(sc->pf); 2184 start = addr & ~0x7f; /* start must be 128B aligned */ 2185 } 2186 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 2187 2188 t4_write_reg(sc, reg, start | pf); 2189 t4_read_reg(sc, reg); 2190 2191 return (addr - start); 2192 } 2193 2194 static int 2195 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, int num_vis, 2196 struct intrs_and_queues *iaq) 2197 { 2198 int rc, itype, navail, nrxq10g, nrxq1g, n; 2199 int nofldrxq10g = 0, nofldrxq1g = 0; 2200 int nnmrxq10g = 0, nnmrxq1g = 0; 2201 2202 bzero(iaq, sizeof(*iaq)); 2203 2204 iaq->ntxq10g = t4_ntxq10g; 2205 iaq->ntxq1g = t4_ntxq1g; 2206 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 2207 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 2208 iaq->rsrv_noflowq = t4_rsrv_noflowq; 2209 #ifdef TCP_OFFLOAD 2210 if (is_offload(sc)) { 2211 iaq->nofldtxq10g = t4_nofldtxq10g; 2212 iaq->nofldtxq1g = t4_nofldtxq1g; 2213 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 2214 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 2215 } 2216 #endif 2217 #ifdef DEV_NETMAP 2218 iaq->nnmtxq10g = t4_nnmtxq10g; 2219 iaq->nnmtxq1g = t4_nnmtxq1g; 2220 iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g; 2221 iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g; 2222 #endif 2223 2224 for (itype = INTR_MSIX; itype; itype >>= 1) { 2225 2226 if ((itype & t4_intr_types) == 0) 2227 continue; /* not allowed */ 2228 2229 if (itype == INTR_MSIX) 2230 navail = pci_msix_count(sc->dev); 2231 else if (itype == INTR_MSI) 2232 navail = pci_msi_count(sc->dev); 2233 else 2234 navail = 1; 2235 restart: 2236 if (navail == 0) 2237 continue; 2238 2239 iaq->intr_type = itype; 2240 iaq->intr_flags_10g = 0; 2241 iaq->intr_flags_1g = 0; 2242 2243 /* 2244 * Best option: an interrupt vector for errors, one for the 2245 * firmware event queue, and one for every rxq (NIC, TOE, and 2246 * netmap). 2247 */ 2248 iaq->nirq = T4_EXTRA_INTR; 2249 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g); 2250 iaq->nirq += n10g * 2 * (num_vis - 1); 2251 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g); 2252 iaq->nirq += n1g * 2 * (num_vis - 1); 2253 if (iaq->nirq <= navail && 2254 (itype != INTR_MSI || powerof2(iaq->nirq))) { 2255 iaq->intr_flags_10g = INTR_ALL; 2256 iaq->intr_flags_1g = INTR_ALL; 2257 goto allocate; 2258 } 2259 2260 /* 2261 * Second best option: a vector for errors, one for the firmware 2262 * event queue, and vectors for either all the NIC rx queues or 2263 * all the TOE rx queues. The queues that don't get vectors 2264 * will forward their interrupts to those that do. 2265 * 2266 * Note: netmap rx queues cannot be created early and so they 2267 * can't be setup to receive forwarded interrupts for others. 2268 */ 2269 iaq->nirq = T4_EXTRA_INTR; 2270 if (nrxq10g >= nofldrxq10g) { 2271 iaq->intr_flags_10g = INTR_RXQ; 2272 iaq->nirq += n10g * nrxq10g; 2273 iaq->nirq += n10g * (num_vis - 1); 2274 #ifdef DEV_NETMAP 2275 iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g); 2276 #endif 2277 } else { 2278 iaq->intr_flags_10g = INTR_OFLD_RXQ; 2279 iaq->nirq += n10g * nofldrxq10g; 2280 #ifdef DEV_NETMAP 2281 iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g); 2282 #endif 2283 } 2284 if (nrxq1g >= nofldrxq1g) { 2285 iaq->intr_flags_1g = INTR_RXQ; 2286 iaq->nirq += n1g * nrxq1g; 2287 iaq->nirq += n1g * (num_vis - 1); 2288 #ifdef DEV_NETMAP 2289 iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g); 2290 #endif 2291 } else { 2292 iaq->intr_flags_1g = INTR_OFLD_RXQ; 2293 iaq->nirq += n1g * nofldrxq1g; 2294 #ifdef DEV_NETMAP 2295 iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g); 2296 #endif 2297 } 2298 if (iaq->nirq <= navail && 2299 (itype != INTR_MSI || powerof2(iaq->nirq))) 2300 goto allocate; 2301 2302 /* 2303 * Next best option: an interrupt vector for errors, one for the 2304 * firmware event queue, and at least one per VI. At this 2305 * point we know we'll have to downsize nrxq and/or nofldrxq 2306 * and/or nnmrxq to fit what's available to us. 2307 */ 2308 iaq->nirq = T4_EXTRA_INTR; 2309 iaq->nirq += (n10g + n1g) * num_vis; 2310 if (iaq->nirq <= navail) { 2311 int leftover = navail - iaq->nirq; 2312 2313 if (n10g > 0) { 2314 int target = max(nrxq10g, nofldrxq10g); 2315 2316 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ? 2317 INTR_RXQ : INTR_OFLD_RXQ; 2318 2319 n = 1; 2320 while (n < target && leftover >= n10g) { 2321 leftover -= n10g; 2322 iaq->nirq += n10g; 2323 n++; 2324 } 2325 iaq->nrxq10g = min(n, nrxq10g); 2326 #ifdef TCP_OFFLOAD 2327 iaq->nofldrxq10g = min(n, nofldrxq10g); 2328 #endif 2329 #ifdef DEV_NETMAP 2330 iaq->nnmrxq10g = min(n, nnmrxq10g); 2331 #endif 2332 } 2333 2334 if (n1g > 0) { 2335 int target = max(nrxq1g, nofldrxq1g); 2336 2337 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ? 2338 INTR_RXQ : INTR_OFLD_RXQ; 2339 2340 n = 1; 2341 while (n < target && leftover >= n1g) { 2342 leftover -= n1g; 2343 iaq->nirq += n1g; 2344 n++; 2345 } 2346 iaq->nrxq1g = min(n, nrxq1g); 2347 #ifdef TCP_OFFLOAD 2348 iaq->nofldrxq1g = min(n, nofldrxq1g); 2349 #endif 2350 #ifdef DEV_NETMAP 2351 iaq->nnmrxq1g = min(n, nnmrxq1g); 2352 #endif 2353 } 2354 2355 if (itype != INTR_MSI || powerof2(iaq->nirq)) 2356 goto allocate; 2357 } 2358 2359 /* 2360 * Least desirable option: one interrupt vector for everything. 2361 */ 2362 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 2363 iaq->intr_flags_10g = iaq->intr_flags_1g = 0; 2364 #ifdef TCP_OFFLOAD 2365 if (is_offload(sc)) 2366 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 2367 #endif 2368 #ifdef DEV_NETMAP 2369 iaq->nnmrxq10g = iaq->nnmrxq1g = 1; 2370 #endif 2371 2372 allocate: 2373 navail = iaq->nirq; 2374 rc = 0; 2375 if (itype == INTR_MSIX) 2376 rc = pci_alloc_msix(sc->dev, &navail); 2377 else if (itype == INTR_MSI) 2378 rc = pci_alloc_msi(sc->dev, &navail); 2379 2380 if (rc == 0) { 2381 if (navail == iaq->nirq) 2382 return (0); 2383 2384 /* 2385 * Didn't get the number requested. Use whatever number 2386 * the kernel is willing to allocate (it's in navail). 2387 */ 2388 device_printf(sc->dev, "fewer vectors than requested, " 2389 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 2390 itype, iaq->nirq, navail); 2391 pci_release_msi(sc->dev); 2392 goto restart; 2393 } 2394 2395 device_printf(sc->dev, 2396 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 2397 itype, rc, iaq->nirq, navail); 2398 } 2399 2400 device_printf(sc->dev, 2401 "failed to find a usable interrupt type. " 2402 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 2403 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 2404 2405 return (ENXIO); 2406 } 2407 2408 #define FW_VERSION(chip) ( \ 2409 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ 2410 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ 2411 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ 2412 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) 2413 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) 2414 2415 struct fw_info { 2416 uint8_t chip; 2417 char *kld_name; 2418 char *fw_mod_name; 2419 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 2420 } fw_info[] = { 2421 { 2422 .chip = CHELSIO_T4, 2423 .kld_name = "t4fw_cfg", 2424 .fw_mod_name = "t4fw", 2425 .fw_hdr = { 2426 .chip = FW_HDR_CHIP_T4, 2427 .fw_ver = htobe32_const(FW_VERSION(T4)), 2428 .intfver_nic = FW_INTFVER(T4, NIC), 2429 .intfver_vnic = FW_INTFVER(T4, VNIC), 2430 .intfver_ofld = FW_INTFVER(T4, OFLD), 2431 .intfver_ri = FW_INTFVER(T4, RI), 2432 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 2433 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 2434 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 2435 .intfver_fcoe = FW_INTFVER(T4, FCOE), 2436 }, 2437 }, { 2438 .chip = CHELSIO_T5, 2439 .kld_name = "t5fw_cfg", 2440 .fw_mod_name = "t5fw", 2441 .fw_hdr = { 2442 .chip = FW_HDR_CHIP_T5, 2443 .fw_ver = htobe32_const(FW_VERSION(T5)), 2444 .intfver_nic = FW_INTFVER(T5, NIC), 2445 .intfver_vnic = FW_INTFVER(T5, VNIC), 2446 .intfver_ofld = FW_INTFVER(T5, OFLD), 2447 .intfver_ri = FW_INTFVER(T5, RI), 2448 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 2449 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 2450 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 2451 .intfver_fcoe = FW_INTFVER(T5, FCOE), 2452 }, 2453 } 2454 }; 2455 2456 static struct fw_info * 2457 find_fw_info(int chip) 2458 { 2459 int i; 2460 2461 for (i = 0; i < nitems(fw_info); i++) { 2462 if (fw_info[i].chip == chip) 2463 return (&fw_info[i]); 2464 } 2465 return (NULL); 2466 } 2467 2468 /* 2469 * Is the given firmware API compatible with the one the driver was compiled 2470 * with? 2471 */ 2472 static int 2473 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 2474 { 2475 2476 /* short circuit if it's the exact same firmware version */ 2477 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 2478 return (1); 2479 2480 /* 2481 * XXX: Is this too conservative? Perhaps I should limit this to the 2482 * features that are supported in the driver. 2483 */ 2484 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 2485 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 2486 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 2487 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 2488 return (1); 2489 #undef SAME_INTF 2490 2491 return (0); 2492 } 2493 2494 /* 2495 * The firmware in the KLD is usable, but should it be installed? This routine 2496 * explains itself in detail if it indicates the KLD firmware should be 2497 * installed. 2498 */ 2499 static int 2500 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 2501 { 2502 const char *reason; 2503 2504 if (!card_fw_usable) { 2505 reason = "incompatible or unusable"; 2506 goto install; 2507 } 2508 2509 if (k > c) { 2510 reason = "older than the version bundled with this driver"; 2511 goto install; 2512 } 2513 2514 if (t4_fw_install == 2 && k != c) { 2515 reason = "different than the version bundled with this driver"; 2516 goto install; 2517 } 2518 2519 return (0); 2520 2521 install: 2522 if (t4_fw_install == 0) { 2523 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2524 "but the driver is prohibited from installing a different " 2525 "firmware on the card.\n", 2526 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2527 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 2528 2529 return (0); 2530 } 2531 2532 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2533 "installing firmware %u.%u.%u.%u on card.\n", 2534 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2535 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 2536 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2537 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2538 2539 return (1); 2540 } 2541 /* 2542 * Establish contact with the firmware and determine if we are the master driver 2543 * or not, and whether we are responsible for chip initialization. 2544 */ 2545 static int 2546 prep_firmware(struct adapter *sc) 2547 { 2548 const struct firmware *fw = NULL, *default_cfg; 2549 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 2550 enum dev_state state; 2551 struct fw_info *fw_info; 2552 struct fw_hdr *card_fw; /* fw on the card */ 2553 const struct fw_hdr *kld_fw; /* fw in the KLD */ 2554 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 2555 against */ 2556 2557 /* Contact firmware. */ 2558 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 2559 if (rc < 0 || state == DEV_STATE_ERR) { 2560 rc = -rc; 2561 device_printf(sc->dev, 2562 "failed to connect to the firmware: %d, %d.\n", rc, state); 2563 return (rc); 2564 } 2565 pf = rc; 2566 if (pf == sc->mbox) 2567 sc->flags |= MASTER_PF; 2568 else if (state == DEV_STATE_UNINIT) { 2569 /* 2570 * We didn't get to be the master so we definitely won't be 2571 * configuring the chip. It's a bug if someone else hasn't 2572 * configured it already. 2573 */ 2574 device_printf(sc->dev, "couldn't be master(%d), " 2575 "device not already initialized either(%d).\n", rc, state); 2576 return (EDOOFUS); 2577 } 2578 2579 /* This is the firmware whose headers the driver was compiled against */ 2580 fw_info = find_fw_info(chip_id(sc)); 2581 if (fw_info == NULL) { 2582 device_printf(sc->dev, 2583 "unable to look up firmware information for chip %d.\n", 2584 chip_id(sc)); 2585 return (EINVAL); 2586 } 2587 drv_fw = &fw_info->fw_hdr; 2588 2589 /* 2590 * The firmware KLD contains many modules. The KLD name is also the 2591 * name of the module that contains the default config file. 2592 */ 2593 default_cfg = firmware_get(fw_info->kld_name); 2594 2595 /* Read the header of the firmware on the card */ 2596 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 2597 rc = -t4_read_flash(sc, FLASH_FW_START, 2598 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 2599 if (rc == 0) 2600 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 2601 else { 2602 device_printf(sc->dev, 2603 "Unable to read card's firmware header: %d\n", rc); 2604 card_fw_usable = 0; 2605 } 2606 2607 /* This is the firmware in the KLD */ 2608 fw = firmware_get(fw_info->fw_mod_name); 2609 if (fw != NULL) { 2610 kld_fw = (const void *)fw->data; 2611 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 2612 } else { 2613 kld_fw = NULL; 2614 kld_fw_usable = 0; 2615 } 2616 2617 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2618 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) { 2619 /* 2620 * Common case: the firmware on the card is an exact match and 2621 * the KLD is an exact match too, or the KLD is 2622 * absent/incompatible. Note that t4_fw_install = 2 is ignored 2623 * here -- use cxgbetool loadfw if you want to reinstall the 2624 * same firmware as the one on the card. 2625 */ 2626 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2627 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2628 be32toh(card_fw->fw_ver))) { 2629 2630 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 2631 if (rc != 0) { 2632 device_printf(sc->dev, 2633 "failed to install firmware: %d\n", rc); 2634 goto done; 2635 } 2636 2637 /* Installed successfully, update the cached header too. */ 2638 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2639 card_fw_usable = 1; 2640 need_fw_reset = 0; /* already reset as part of load_fw */ 2641 } 2642 2643 if (!card_fw_usable) { 2644 uint32_t d, c, k; 2645 2646 d = ntohl(drv_fw->fw_ver); 2647 c = ntohl(card_fw->fw_ver); 2648 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2649 2650 device_printf(sc->dev, "Cannot find a usable firmware: " 2651 "fw_install %d, chip state %d, " 2652 "driver compiled with %d.%d.%d.%d, " 2653 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2654 t4_fw_install, state, 2655 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2656 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2657 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2658 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2659 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2660 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2661 rc = EINVAL; 2662 goto done; 2663 } 2664 2665 /* We're using whatever's on the card and it's known to be good. */ 2666 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2667 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2668 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2669 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2670 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2671 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2672 t4_get_tp_version(sc, &sc->params.tp_vers); 2673 2674 /* Reset device */ 2675 if (need_fw_reset && 2676 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2677 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2678 if (rc != ETIMEDOUT && rc != EIO) 2679 t4_fw_bye(sc, sc->mbox); 2680 goto done; 2681 } 2682 sc->flags |= FW_OK; 2683 2684 rc = get_params__pre_init(sc); 2685 if (rc != 0) 2686 goto done; /* error message displayed already */ 2687 2688 /* Partition adapter resources as specified in the config file. */ 2689 if (state == DEV_STATE_UNINIT) { 2690 2691 KASSERT(sc->flags & MASTER_PF, 2692 ("%s: trying to change chip settings when not master.", 2693 __func__)); 2694 2695 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2696 if (rc != 0) 2697 goto done; /* error message displayed already */ 2698 2699 t4_tweak_chip_settings(sc); 2700 2701 /* get basic stuff going */ 2702 rc = -t4_fw_initialize(sc, sc->mbox); 2703 if (rc != 0) { 2704 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2705 goto done; 2706 } 2707 } else { 2708 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2709 sc->cfcsum = 0; 2710 } 2711 2712 done: 2713 free(card_fw, M_CXGBE); 2714 if (fw != NULL) 2715 firmware_put(fw, FIRMWARE_UNLOAD); 2716 if (default_cfg != NULL) 2717 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2718 2719 return (rc); 2720 } 2721 2722 #define FW_PARAM_DEV(param) \ 2723 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2724 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2725 #define FW_PARAM_PFVF(param) \ 2726 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2727 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2728 2729 /* 2730 * Partition chip resources for use between various PFs, VFs, etc. 2731 */ 2732 static int 2733 partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2734 const char *name_prefix) 2735 { 2736 const struct firmware *cfg = NULL; 2737 int rc = 0; 2738 struct fw_caps_config_cmd caps; 2739 uint32_t mtype, moff, finicsum, cfcsum; 2740 2741 /* 2742 * Figure out what configuration file to use. Pick the default config 2743 * file for the card if the user hasn't specified one explicitly. 2744 */ 2745 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2746 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2747 /* Card specific overrides go here. */ 2748 if (pci_get_device(sc->dev) == 0x440a) 2749 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2750 if (is_fpga(sc)) 2751 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2752 } 2753 2754 /* 2755 * We need to load another module if the profile is anything except 2756 * "default" or "flash". 2757 */ 2758 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2759 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2760 char s[32]; 2761 2762 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2763 cfg = firmware_get(s); 2764 if (cfg == NULL) { 2765 if (default_cfg != NULL) { 2766 device_printf(sc->dev, 2767 "unable to load module \"%s\" for " 2768 "configuration profile \"%s\", will use " 2769 "the default config file instead.\n", 2770 s, sc->cfg_file); 2771 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2772 "%s", DEFAULT_CF); 2773 } else { 2774 device_printf(sc->dev, 2775 "unable to load module \"%s\" for " 2776 "configuration profile \"%s\", will use " 2777 "the config file on the card's flash " 2778 "instead.\n", s, sc->cfg_file); 2779 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2780 "%s", FLASH_CF); 2781 } 2782 } 2783 } 2784 2785 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2786 default_cfg == NULL) { 2787 device_printf(sc->dev, 2788 "default config file not available, will use the config " 2789 "file on the card's flash instead.\n"); 2790 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2791 } 2792 2793 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2794 u_int cflen, i, n; 2795 const uint32_t *cfdata; 2796 uint32_t param, val, addr, off, mw_base, mw_aperture; 2797 2798 KASSERT(cfg != NULL || default_cfg != NULL, 2799 ("%s: no config to upload", __func__)); 2800 2801 /* 2802 * Ask the firmware where it wants us to upload the config file. 2803 */ 2804 param = FW_PARAM_DEV(CF); 2805 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2806 if (rc != 0) { 2807 /* No support for config file? Shouldn't happen. */ 2808 device_printf(sc->dev, 2809 "failed to query config file location: %d.\n", rc); 2810 goto done; 2811 } 2812 mtype = G_FW_PARAMS_PARAM_Y(val); 2813 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2814 2815 /* 2816 * XXX: sheer laziness. We deliberately added 4 bytes of 2817 * useless stuffing/comments at the end of the config file so 2818 * it's ok to simply throw away the last remaining bytes when 2819 * the config file is not an exact multiple of 4. This also 2820 * helps with the validate_mt_off_len check. 2821 */ 2822 if (cfg != NULL) { 2823 cflen = cfg->datasize & ~3; 2824 cfdata = cfg->data; 2825 } else { 2826 cflen = default_cfg->datasize & ~3; 2827 cfdata = default_cfg->data; 2828 } 2829 2830 if (cflen > FLASH_CFG_MAX_SIZE) { 2831 device_printf(sc->dev, 2832 "config file too long (%d, max allowed is %d). " 2833 "Will try to use the config on the card, if any.\n", 2834 cflen, FLASH_CFG_MAX_SIZE); 2835 goto use_config_on_flash; 2836 } 2837 2838 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2839 if (rc != 0) { 2840 device_printf(sc->dev, 2841 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2842 "Will try to use the config on the card, if any.\n", 2843 __func__, mtype, moff, cflen, rc); 2844 goto use_config_on_flash; 2845 } 2846 2847 memwin_info(sc, 2, &mw_base, &mw_aperture); 2848 while (cflen) { 2849 off = position_memwin(sc, 2, addr); 2850 n = min(cflen, mw_aperture - off); 2851 for (i = 0; i < n; i += 4) 2852 t4_write_reg(sc, mw_base + off + i, *cfdata++); 2853 cflen -= n; 2854 addr += n; 2855 } 2856 } else { 2857 use_config_on_flash: 2858 mtype = FW_MEMTYPE_FLASH; 2859 moff = t4_flash_cfg_addr(sc); 2860 } 2861 2862 bzero(&caps, sizeof(caps)); 2863 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2864 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2865 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2866 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2867 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2868 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2869 if (rc != 0) { 2870 device_printf(sc->dev, 2871 "failed to pre-process config file: %d " 2872 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2873 goto done; 2874 } 2875 2876 finicsum = be32toh(caps.finicsum); 2877 cfcsum = be32toh(caps.cfcsum); 2878 if (finicsum != cfcsum) { 2879 device_printf(sc->dev, 2880 "WARNING: config file checksum mismatch: %08x %08x\n", 2881 finicsum, cfcsum); 2882 } 2883 sc->cfcsum = cfcsum; 2884 2885 #define LIMIT_CAPS(x) do { \ 2886 caps.x &= htobe16(t4_##x##_allowed); \ 2887 } while (0) 2888 2889 /* 2890 * Let the firmware know what features will (not) be used so it can tune 2891 * things accordingly. 2892 */ 2893 LIMIT_CAPS(linkcaps); 2894 LIMIT_CAPS(niccaps); 2895 LIMIT_CAPS(toecaps); 2896 LIMIT_CAPS(rdmacaps); 2897 LIMIT_CAPS(iscsicaps); 2898 LIMIT_CAPS(fcoecaps); 2899 #undef LIMIT_CAPS 2900 2901 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2902 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2903 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2904 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2905 if (rc != 0) { 2906 device_printf(sc->dev, 2907 "failed to process config file: %d.\n", rc); 2908 } 2909 done: 2910 if (cfg != NULL) 2911 firmware_put(cfg, FIRMWARE_UNLOAD); 2912 return (rc); 2913 } 2914 2915 /* 2916 * Retrieve parameters that are needed (or nice to have) very early. 2917 */ 2918 static int 2919 get_params__pre_init(struct adapter *sc) 2920 { 2921 int rc; 2922 uint32_t param[2], val[2]; 2923 struct fw_devlog_cmd cmd; 2924 struct devlog_params *dlog = &sc->params.devlog; 2925 2926 param[0] = FW_PARAM_DEV(PORTVEC); 2927 param[1] = FW_PARAM_DEV(CCLK); 2928 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2929 if (rc != 0) { 2930 device_printf(sc->dev, 2931 "failed to query parameters (pre_init): %d.\n", rc); 2932 return (rc); 2933 } 2934 2935 sc->params.portvec = val[0]; 2936 sc->params.nports = bitcount32(val[0]); 2937 sc->params.vpd.cclk = val[1]; 2938 2939 /* Read device log parameters. */ 2940 bzero(&cmd, sizeof(cmd)); 2941 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2942 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2943 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2944 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2945 if (rc != 0) { 2946 device_printf(sc->dev, 2947 "failed to get devlog parameters: %d.\n", rc); 2948 bzero(dlog, sizeof (*dlog)); 2949 rc = 0; /* devlog isn't critical for device operation */ 2950 } else { 2951 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2952 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2953 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2954 dlog->size = be32toh(cmd.memsize_devlog); 2955 } 2956 2957 return (rc); 2958 } 2959 2960 /* 2961 * Retrieve various parameters that are of interest to the driver. The device 2962 * has been initialized by the firmware at this point. 2963 */ 2964 static int 2965 get_params__post_init(struct adapter *sc) 2966 { 2967 int rc; 2968 uint32_t param[7], val[7]; 2969 struct fw_caps_config_cmd caps; 2970 2971 param[0] = FW_PARAM_PFVF(IQFLINT_START); 2972 param[1] = FW_PARAM_PFVF(EQ_START); 2973 param[2] = FW_PARAM_PFVF(FILTER_START); 2974 param[3] = FW_PARAM_PFVF(FILTER_END); 2975 param[4] = FW_PARAM_PFVF(L2T_START); 2976 param[5] = FW_PARAM_PFVF(L2T_END); 2977 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2978 if (rc != 0) { 2979 device_printf(sc->dev, 2980 "failed to query parameters (post_init): %d.\n", rc); 2981 return (rc); 2982 } 2983 2984 sc->sge.iq_start = val[0]; 2985 sc->sge.eq_start = val[1]; 2986 sc->tids.ftid_base = val[2]; 2987 sc->tids.nftids = val[3] - val[2] + 1; 2988 sc->params.ftid_min = val[2]; 2989 sc->params.ftid_max = val[3]; 2990 sc->vres.l2t.start = val[4]; 2991 sc->vres.l2t.size = val[5] - val[4] + 1; 2992 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2993 ("%s: L2 table size (%u) larger than expected (%u)", 2994 __func__, sc->vres.l2t.size, L2T_SIZE)); 2995 2996 /* get capabilites */ 2997 bzero(&caps, sizeof(caps)); 2998 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2999 F_FW_CMD_REQUEST | F_FW_CMD_READ); 3000 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 3001 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 3002 if (rc != 0) { 3003 device_printf(sc->dev, 3004 "failed to get card capabilities: %d.\n", rc); 3005 return (rc); 3006 } 3007 3008 #define READ_CAPS(x) do { \ 3009 sc->x = htobe16(caps.x); \ 3010 } while (0) 3011 READ_CAPS(linkcaps); 3012 READ_CAPS(niccaps); 3013 READ_CAPS(toecaps); 3014 READ_CAPS(rdmacaps); 3015 READ_CAPS(iscsicaps); 3016 READ_CAPS(fcoecaps); 3017 3018 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { 3019 param[0] = FW_PARAM_PFVF(ETHOFLD_START); 3020 param[1] = FW_PARAM_PFVF(ETHOFLD_END); 3021 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 3022 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); 3023 if (rc != 0) { 3024 device_printf(sc->dev, 3025 "failed to query NIC parameters: %d.\n", rc); 3026 return (rc); 3027 } 3028 sc->tids.etid_base = val[0]; 3029 sc->params.etid_min = val[0]; 3030 sc->tids.netids = val[1] - val[0] + 1; 3031 sc->params.netids = sc->tids.netids; 3032 sc->params.eo_wr_cred = val[2]; 3033 sc->params.ethoffload = 1; 3034 } 3035 3036 if (sc->toecaps) { 3037 /* query offload-related parameters */ 3038 param[0] = FW_PARAM_DEV(NTID); 3039 param[1] = FW_PARAM_PFVF(SERVER_START); 3040 param[2] = FW_PARAM_PFVF(SERVER_END); 3041 param[3] = FW_PARAM_PFVF(TDDP_START); 3042 param[4] = FW_PARAM_PFVF(TDDP_END); 3043 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 3044 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 3045 if (rc != 0) { 3046 device_printf(sc->dev, 3047 "failed to query TOE parameters: %d.\n", rc); 3048 return (rc); 3049 } 3050 sc->tids.ntids = val[0]; 3051 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 3052 sc->tids.stid_base = val[1]; 3053 sc->tids.nstids = val[2] - val[1] + 1; 3054 sc->vres.ddp.start = val[3]; 3055 sc->vres.ddp.size = val[4] - val[3] + 1; 3056 sc->params.ofldq_wr_cred = val[5]; 3057 sc->params.offload = 1; 3058 } 3059 if (sc->rdmacaps) { 3060 param[0] = FW_PARAM_PFVF(STAG_START); 3061 param[1] = FW_PARAM_PFVF(STAG_END); 3062 param[2] = FW_PARAM_PFVF(RQ_START); 3063 param[3] = FW_PARAM_PFVF(RQ_END); 3064 param[4] = FW_PARAM_PFVF(PBL_START); 3065 param[5] = FW_PARAM_PFVF(PBL_END); 3066 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 3067 if (rc != 0) { 3068 device_printf(sc->dev, 3069 "failed to query RDMA parameters(1): %d.\n", rc); 3070 return (rc); 3071 } 3072 sc->vres.stag.start = val[0]; 3073 sc->vres.stag.size = val[1] - val[0] + 1; 3074 sc->vres.rq.start = val[2]; 3075 sc->vres.rq.size = val[3] - val[2] + 1; 3076 sc->vres.pbl.start = val[4]; 3077 sc->vres.pbl.size = val[5] - val[4] + 1; 3078 3079 param[0] = FW_PARAM_PFVF(SQRQ_START); 3080 param[1] = FW_PARAM_PFVF(SQRQ_END); 3081 param[2] = FW_PARAM_PFVF(CQ_START); 3082 param[3] = FW_PARAM_PFVF(CQ_END); 3083 param[4] = FW_PARAM_PFVF(OCQ_START); 3084 param[5] = FW_PARAM_PFVF(OCQ_END); 3085 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 3086 if (rc != 0) { 3087 device_printf(sc->dev, 3088 "failed to query RDMA parameters(2): %d.\n", rc); 3089 return (rc); 3090 } 3091 sc->vres.qp.start = val[0]; 3092 sc->vres.qp.size = val[1] - val[0] + 1; 3093 sc->vres.cq.start = val[2]; 3094 sc->vres.cq.size = val[3] - val[2] + 1; 3095 sc->vres.ocq.start = val[4]; 3096 sc->vres.ocq.size = val[5] - val[4] + 1; 3097 } 3098 if (sc->iscsicaps) { 3099 param[0] = FW_PARAM_PFVF(ISCSI_START); 3100 param[1] = FW_PARAM_PFVF(ISCSI_END); 3101 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 3102 if (rc != 0) { 3103 device_printf(sc->dev, 3104 "failed to query iSCSI parameters: %d.\n", rc); 3105 return (rc); 3106 } 3107 sc->vres.iscsi.start = val[0]; 3108 sc->vres.iscsi.size = val[1] - val[0] + 1; 3109 } 3110 3111 /* 3112 * We've got the params we wanted to query via the firmware. Now grab 3113 * some others directly from the chip. 3114 */ 3115 rc = t4_read_chip_settings(sc); 3116 3117 return (rc); 3118 } 3119 3120 static int 3121 set_params__post_init(struct adapter *sc) 3122 { 3123 uint32_t param, val; 3124 3125 /* ask for encapsulated CPLs */ 3126 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 3127 val = 1; 3128 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 3129 3130 return (0); 3131 } 3132 3133 #undef FW_PARAM_PFVF 3134 #undef FW_PARAM_DEV 3135 3136 static void 3137 t4_set_desc(struct adapter *sc) 3138 { 3139 char buf[128]; 3140 struct adapter_params *p = &sc->params; 3141 3142 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, " 3143 "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "", 3144 chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec); 3145 3146 device_set_desc_copy(sc->dev, buf); 3147 } 3148 3149 static void 3150 build_medialist(struct port_info *pi, struct ifmedia *media) 3151 { 3152 int m; 3153 3154 PORT_LOCK(pi); 3155 3156 ifmedia_removeall(media); 3157 3158 m = IFM_ETHER | IFM_FDX; 3159 3160 switch(pi->port_type) { 3161 case FW_PORT_TYPE_BT_XFI: 3162 case FW_PORT_TYPE_BT_XAUI: 3163 ifmedia_add(media, m | IFM_10G_T, 0, NULL); 3164 /* fall through */ 3165 3166 case FW_PORT_TYPE_BT_SGMII: 3167 ifmedia_add(media, m | IFM_1000_T, 0, NULL); 3168 ifmedia_add(media, m | IFM_100_TX, 0, NULL); 3169 ifmedia_add(media, IFM_ETHER | IFM_AUTO, 0, NULL); 3170 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 3171 break; 3172 3173 case FW_PORT_TYPE_CX4: 3174 ifmedia_add(media, m | IFM_10G_CX4, 0, NULL); 3175 ifmedia_set(media, m | IFM_10G_CX4); 3176 break; 3177 3178 case FW_PORT_TYPE_QSFP_10G: 3179 case FW_PORT_TYPE_SFP: 3180 case FW_PORT_TYPE_FIBER_XFI: 3181 case FW_PORT_TYPE_FIBER_XAUI: 3182 switch (pi->mod_type) { 3183 3184 case FW_PORT_MOD_TYPE_LR: 3185 ifmedia_add(media, m | IFM_10G_LR, 0, NULL); 3186 ifmedia_set(media, m | IFM_10G_LR); 3187 break; 3188 3189 case FW_PORT_MOD_TYPE_SR: 3190 ifmedia_add(media, m | IFM_10G_SR, 0, NULL); 3191 ifmedia_set(media, m | IFM_10G_SR); 3192 break; 3193 3194 case FW_PORT_MOD_TYPE_LRM: 3195 ifmedia_add(media, m | IFM_10G_LRM, 0, NULL); 3196 ifmedia_set(media, m | IFM_10G_LRM); 3197 break; 3198 3199 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 3200 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 3201 ifmedia_add(media, m | IFM_10G_TWINAX, 0, NULL); 3202 ifmedia_set(media, m | IFM_10G_TWINAX); 3203 break; 3204 3205 case FW_PORT_MOD_TYPE_NONE: 3206 m &= ~IFM_FDX; 3207 ifmedia_add(media, m | IFM_NONE, 0, NULL); 3208 ifmedia_set(media, m | IFM_NONE); 3209 break; 3210 3211 case FW_PORT_MOD_TYPE_NA: 3212 case FW_PORT_MOD_TYPE_ER: 3213 default: 3214 device_printf(pi->dev, 3215 "unknown port_type (%d), mod_type (%d)\n", 3216 pi->port_type, pi->mod_type); 3217 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 3218 ifmedia_set(media, m | IFM_UNKNOWN); 3219 break; 3220 } 3221 break; 3222 3223 case FW_PORT_TYPE_QSFP: 3224 switch (pi->mod_type) { 3225 3226 case FW_PORT_MOD_TYPE_LR: 3227 ifmedia_add(media, m | IFM_40G_LR4, 0, NULL); 3228 ifmedia_set(media, m | IFM_40G_LR4); 3229 break; 3230 3231 case FW_PORT_MOD_TYPE_SR: 3232 ifmedia_add(media, m | IFM_40G_SR4, 0, NULL); 3233 ifmedia_set(media, m | IFM_40G_SR4); 3234 break; 3235 3236 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 3237 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 3238 ifmedia_add(media, m | IFM_40G_CR4, 0, NULL); 3239 ifmedia_set(media, m | IFM_40G_CR4); 3240 break; 3241 3242 case FW_PORT_MOD_TYPE_NONE: 3243 m &= ~IFM_FDX; 3244 ifmedia_add(media, m | IFM_NONE, 0, NULL); 3245 ifmedia_set(media, m | IFM_NONE); 3246 break; 3247 3248 default: 3249 device_printf(pi->dev, 3250 "unknown port_type (%d), mod_type (%d)\n", 3251 pi->port_type, pi->mod_type); 3252 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 3253 ifmedia_set(media, m | IFM_UNKNOWN); 3254 break; 3255 } 3256 break; 3257 3258 default: 3259 device_printf(pi->dev, 3260 "unknown port_type (%d), mod_type (%d)\n", pi->port_type, 3261 pi->mod_type); 3262 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 3263 ifmedia_set(media, m | IFM_UNKNOWN); 3264 break; 3265 } 3266 3267 PORT_UNLOCK(pi); 3268 } 3269 3270 #define FW_MAC_EXACT_CHUNK 7 3271 3272 /* 3273 * Program the port's XGMAC based on parameters in ifnet. The caller also 3274 * indicates which parameters should be programmed (the rest are left alone). 3275 */ 3276 int 3277 update_mac_settings(struct ifnet *ifp, int flags) 3278 { 3279 int rc = 0; 3280 struct vi_info *vi = ifp->if_softc; 3281 struct port_info *pi = vi->pi; 3282 struct adapter *sc = pi->adapter; 3283 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 3284 3285 ASSERT_SYNCHRONIZED_OP(sc); 3286 KASSERT(flags, ("%s: not told what to update.", __func__)); 3287 3288 if (flags & XGMAC_MTU) 3289 mtu = ifp->if_mtu; 3290 3291 if (flags & XGMAC_PROMISC) 3292 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 3293 3294 if (flags & XGMAC_ALLMULTI) 3295 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 3296 3297 if (flags & XGMAC_VLANEX) 3298 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 3299 3300 if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) { 3301 rc = -t4_set_rxmode(sc, sc->mbox, vi->viid, mtu, promisc, 3302 allmulti, 1, vlanex, false); 3303 if (rc) { 3304 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, 3305 rc); 3306 return (rc); 3307 } 3308 } 3309 3310 if (flags & XGMAC_UCADDR) { 3311 uint8_t ucaddr[ETHER_ADDR_LEN]; 3312 3313 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 3314 rc = t4_change_mac(sc, sc->mbox, vi->viid, vi->xact_addr_filt, 3315 ucaddr, true, true); 3316 if (rc < 0) { 3317 rc = -rc; 3318 if_printf(ifp, "change_mac failed: %d\n", rc); 3319 return (rc); 3320 } else { 3321 vi->xact_addr_filt = rc; 3322 rc = 0; 3323 } 3324 } 3325 3326 if (flags & XGMAC_MCADDRS) { 3327 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 3328 int del = 1; 3329 uint64_t hash = 0; 3330 struct ifmultiaddr *ifma; 3331 int i = 0, j; 3332 3333 if_maddr_rlock(ifp); 3334 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3335 if (ifma->ifma_addr->sa_family != AF_LINK) 3336 continue; 3337 mcaddr[i] = 3338 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 3339 MPASS(ETHER_IS_MULTICAST(mcaddr[i])); 3340 i++; 3341 3342 if (i == FW_MAC_EXACT_CHUNK) { 3343 rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid, 3344 del, i, mcaddr, NULL, &hash, 0); 3345 if (rc < 0) { 3346 rc = -rc; 3347 for (j = 0; j < i; j++) { 3348 if_printf(ifp, 3349 "failed to add mc address" 3350 " %02x:%02x:%02x:" 3351 "%02x:%02x:%02x rc=%d\n", 3352 mcaddr[j][0], mcaddr[j][1], 3353 mcaddr[j][2], mcaddr[j][3], 3354 mcaddr[j][4], mcaddr[j][5], 3355 rc); 3356 } 3357 goto mcfail; 3358 } 3359 del = 0; 3360 i = 0; 3361 } 3362 } 3363 if (i > 0) { 3364 rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid, del, i, 3365 mcaddr, NULL, &hash, 0); 3366 if (rc < 0) { 3367 rc = -rc; 3368 for (j = 0; j < i; j++) { 3369 if_printf(ifp, 3370 "failed to add mc address" 3371 " %02x:%02x:%02x:" 3372 "%02x:%02x:%02x rc=%d\n", 3373 mcaddr[j][0], mcaddr[j][1], 3374 mcaddr[j][2], mcaddr[j][3], 3375 mcaddr[j][4], mcaddr[j][5], 3376 rc); 3377 } 3378 goto mcfail; 3379 } 3380 } 3381 3382 rc = -t4_set_addr_hash(sc, sc->mbox, vi->viid, 0, hash, 0); 3383 if (rc != 0) 3384 if_printf(ifp, "failed to set mc address hash: %d", rc); 3385 mcfail: 3386 if_maddr_runlock(ifp); 3387 } 3388 3389 return (rc); 3390 } 3391 3392 /* 3393 * {begin|end}_synchronized_op must be called from the same thread. 3394 */ 3395 int 3396 begin_synchronized_op(struct adapter *sc, struct vi_info *vi, int flags, 3397 char *wmesg) 3398 { 3399 int rc, pri; 3400 3401 #ifdef WITNESS 3402 /* the caller thinks it's ok to sleep, but is it really? */ 3403 if (flags & SLEEP_OK) 3404 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 3405 "begin_synchronized_op"); 3406 #endif 3407 3408 if (INTR_OK) 3409 pri = PCATCH; 3410 else 3411 pri = 0; 3412 3413 ADAPTER_LOCK(sc); 3414 for (;;) { 3415 3416 if (vi && IS_DOOMED(vi)) { 3417 rc = ENXIO; 3418 goto done; 3419 } 3420 3421 if (!IS_BUSY(sc)) { 3422 rc = 0; 3423 break; 3424 } 3425 3426 if (!(flags & SLEEP_OK)) { 3427 rc = EBUSY; 3428 goto done; 3429 } 3430 3431 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 3432 rc = EINTR; 3433 goto done; 3434 } 3435 } 3436 3437 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 3438 SET_BUSY(sc); 3439 #ifdef INVARIANTS 3440 sc->last_op = wmesg; 3441 sc->last_op_thr = curthread; 3442 sc->last_op_flags = flags; 3443 #endif 3444 3445 done: 3446 if (!(flags & HOLD_LOCK) || rc) 3447 ADAPTER_UNLOCK(sc); 3448 3449 return (rc); 3450 } 3451 3452 /* 3453 * Tell if_ioctl and if_init that the VI is going away. This is 3454 * special variant of begin_synchronized_op and must be paired with a 3455 * call to end_synchronized_op. 3456 */ 3457 void 3458 doom_vi(struct adapter *sc, struct vi_info *vi) 3459 { 3460 3461 ADAPTER_LOCK(sc); 3462 SET_DOOMED(vi); 3463 wakeup(&sc->flags); 3464 while (IS_BUSY(sc)) 3465 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 3466 SET_BUSY(sc); 3467 #ifdef INVARIANTS 3468 sc->last_op = "t4detach"; 3469 sc->last_op_thr = curthread; 3470 sc->last_op_flags = 0; 3471 #endif 3472 ADAPTER_UNLOCK(sc); 3473 } 3474 3475 /* 3476 * {begin|end}_synchronized_op must be called from the same thread. 3477 */ 3478 void 3479 end_synchronized_op(struct adapter *sc, int flags) 3480 { 3481 3482 if (flags & LOCK_HELD) 3483 ADAPTER_LOCK_ASSERT_OWNED(sc); 3484 else 3485 ADAPTER_LOCK(sc); 3486 3487 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 3488 CLR_BUSY(sc); 3489 wakeup(&sc->flags); 3490 ADAPTER_UNLOCK(sc); 3491 } 3492 3493 static int 3494 cxgbe_init_synchronized(struct vi_info *vi) 3495 { 3496 struct port_info *pi = vi->pi; 3497 struct adapter *sc = pi->adapter; 3498 struct ifnet *ifp = vi->ifp; 3499 int rc = 0, i; 3500 struct sge_txq *txq; 3501 3502 ASSERT_SYNCHRONIZED_OP(sc); 3503 3504 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 3505 return (0); /* already running */ 3506 3507 if (!(sc->flags & FULL_INIT_DONE) && 3508 ((rc = adapter_full_init(sc)) != 0)) 3509 return (rc); /* error message displayed already */ 3510 3511 if (!(vi->flags & VI_INIT_DONE) && 3512 ((rc = vi_full_init(vi)) != 0)) 3513 return (rc); /* error message displayed already */ 3514 3515 rc = update_mac_settings(ifp, XGMAC_ALL); 3516 if (rc) 3517 goto done; /* error message displayed already */ 3518 3519 rc = -t4_enable_vi(sc, sc->mbox, vi->viid, true, true); 3520 if (rc != 0) { 3521 if_printf(ifp, "enable_vi failed: %d\n", rc); 3522 goto done; 3523 } 3524 3525 /* 3526 * Can't fail from this point onwards. Review cxgbe_uninit_synchronized 3527 * if this changes. 3528 */ 3529 3530 for_each_txq(vi, i, txq) { 3531 TXQ_LOCK(txq); 3532 txq->eq.flags |= EQ_ENABLED; 3533 TXQ_UNLOCK(txq); 3534 } 3535 3536 /* 3537 * The first iq of the first port to come up is used for tracing. 3538 */ 3539 if (sc->traceq < 0 && IS_MAIN_VI(vi)) { 3540 sc->traceq = sc->sge.rxq[vi->first_rxq].iq.abs_id; 3541 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : 3542 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | 3543 V_QUEUENUMBER(sc->traceq)); 3544 pi->flags |= HAS_TRACEQ; 3545 } 3546 3547 /* all ok */ 3548 PORT_LOCK(pi); 3549 ifp->if_drv_flags |= IFF_DRV_RUNNING; 3550 pi->up_vis++; 3551 3552 if (pi->nvi > 1) 3553 callout_reset(&vi->tick, hz, vi_tick, vi); 3554 else 3555 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 3556 PORT_UNLOCK(pi); 3557 done: 3558 if (rc != 0) 3559 cxgbe_uninit_synchronized(vi); 3560 3561 return (rc); 3562 } 3563 3564 /* 3565 * Idempotent. 3566 */ 3567 static int 3568 cxgbe_uninit_synchronized(struct vi_info *vi) 3569 { 3570 struct port_info *pi = vi->pi; 3571 struct adapter *sc = pi->adapter; 3572 struct ifnet *ifp = vi->ifp; 3573 int rc, i; 3574 struct sge_txq *txq; 3575 3576 ASSERT_SYNCHRONIZED_OP(sc); 3577 3578 if (!(vi->flags & VI_INIT_DONE)) { 3579 KASSERT(!(ifp->if_drv_flags & IFF_DRV_RUNNING), 3580 ("uninited VI is running")); 3581 return (0); 3582 } 3583 3584 /* 3585 * Disable the VI so that all its data in either direction is discarded 3586 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 3587 * tick) intact as the TP can deliver negative advice or data that it's 3588 * holding in its RAM (for an offloaded connection) even after the VI is 3589 * disabled. 3590 */ 3591 rc = -t4_enable_vi(sc, sc->mbox, vi->viid, false, false); 3592 if (rc) { 3593 if_printf(ifp, "disable_vi failed: %d\n", rc); 3594 return (rc); 3595 } 3596 3597 for_each_txq(vi, i, txq) { 3598 TXQ_LOCK(txq); 3599 txq->eq.flags &= ~EQ_ENABLED; 3600 TXQ_UNLOCK(txq); 3601 } 3602 3603 PORT_LOCK(pi); 3604 if (pi->nvi == 1) 3605 callout_stop(&pi->tick); 3606 else 3607 callout_stop(&vi->tick); 3608 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 3609 PORT_UNLOCK(pi); 3610 return (0); 3611 } 3612 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3613 pi->up_vis--; 3614 if (pi->up_vis > 0) { 3615 PORT_UNLOCK(pi); 3616 return (0); 3617 } 3618 PORT_UNLOCK(pi); 3619 3620 pi->link_cfg.link_ok = 0; 3621 pi->link_cfg.speed = 0; 3622 pi->linkdnrc = -1; 3623 t4_os_link_changed(sc, pi->port_id, 0, -1); 3624 3625 return (0); 3626 } 3627 3628 /* 3629 * It is ok for this function to fail midway and return right away. t4_detach 3630 * will walk the entire sc->irq list and clean up whatever is valid. 3631 */ 3632 static int 3633 setup_intr_handlers(struct adapter *sc) 3634 { 3635 int rc, rid, p, q, v; 3636 char s[8]; 3637 struct irq *irq; 3638 struct port_info *pi; 3639 struct vi_info *vi; 3640 struct sge_rxq *rxq; 3641 #ifdef TCP_OFFLOAD 3642 struct sge_ofld_rxq *ofld_rxq; 3643 #endif 3644 #ifdef DEV_NETMAP 3645 struct sge_nm_rxq *nm_rxq; 3646 #endif 3647 #ifdef RSS 3648 int nbuckets = rss_getnumbuckets(); 3649 #endif 3650 3651 /* 3652 * Setup interrupts. 3653 */ 3654 irq = &sc->irq[0]; 3655 rid = sc->intr_type == INTR_INTX ? 0 : 1; 3656 if (sc->intr_count == 1) 3657 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all")); 3658 3659 /* Multiple interrupts. */ 3660 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 3661 ("%s: too few intr.", __func__)); 3662 3663 /* The first one is always error intr */ 3664 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 3665 if (rc != 0) 3666 return (rc); 3667 irq++; 3668 rid++; 3669 3670 /* The second one is always the firmware event queue */ 3671 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt"); 3672 if (rc != 0) 3673 return (rc); 3674 irq++; 3675 rid++; 3676 3677 for_each_port(sc, p) { 3678 pi = sc->port[p]; 3679 for_each_vi(pi, v, vi) { 3680 vi->first_intr = rid - 1; 3681 #ifdef DEV_NETMAP 3682 if (vi->flags & VI_NETMAP) { 3683 for_each_nm_rxq(vi, q, nm_rxq) { 3684 snprintf(s, sizeof(s), "%d-%d", p, q); 3685 rc = t4_alloc_irq(sc, irq, rid, 3686 t4_nm_intr, nm_rxq, s); 3687 if (rc != 0) 3688 return (rc); 3689 irq++; 3690 rid++; 3691 vi->nintr++; 3692 } 3693 continue; 3694 } 3695 #endif 3696 if (vi->flags & INTR_RXQ) { 3697 for_each_rxq(vi, q, rxq) { 3698 if (v == 0) 3699 snprintf(s, sizeof(s), "%d.%d", 3700 p, q); 3701 else 3702 snprintf(s, sizeof(s), 3703 "%d(%d).%d", p, v, q); 3704 rc = t4_alloc_irq(sc, irq, rid, 3705 t4_intr, rxq, s); 3706 if (rc != 0) 3707 return (rc); 3708 #ifdef RSS 3709 bus_bind_intr(sc->dev, irq->res, 3710 rss_getcpu(q % nbuckets)); 3711 #endif 3712 irq++; 3713 rid++; 3714 vi->nintr++; 3715 } 3716 } 3717 #ifdef TCP_OFFLOAD 3718 if (vi->flags & INTR_OFLD_RXQ) { 3719 for_each_ofld_rxq(vi, q, ofld_rxq) { 3720 snprintf(s, sizeof(s), "%d,%d", p, q); 3721 rc = t4_alloc_irq(sc, irq, rid, 3722 t4_intr, ofld_rxq, s); 3723 if (rc != 0) 3724 return (rc); 3725 irq++; 3726 rid++; 3727 vi->nintr++; 3728 } 3729 } 3730 #endif 3731 } 3732 } 3733 MPASS(irq == &sc->irq[sc->intr_count]); 3734 3735 return (0); 3736 } 3737 3738 int 3739 adapter_full_init(struct adapter *sc) 3740 { 3741 int rc, i; 3742 3743 ASSERT_SYNCHRONIZED_OP(sc); 3744 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3745 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 3746 ("%s: FULL_INIT_DONE already", __func__)); 3747 3748 /* 3749 * queues that belong to the adapter (not any particular port). 3750 */ 3751 rc = t4_setup_adapter_queues(sc); 3752 if (rc != 0) 3753 goto done; 3754 3755 for (i = 0; i < nitems(sc->tq); i++) { 3756 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 3757 taskqueue_thread_enqueue, &sc->tq[i]); 3758 if (sc->tq[i] == NULL) { 3759 device_printf(sc->dev, 3760 "failed to allocate task queue %d\n", i); 3761 rc = ENOMEM; 3762 goto done; 3763 } 3764 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3765 device_get_nameunit(sc->dev), i); 3766 } 3767 3768 t4_intr_enable(sc); 3769 sc->flags |= FULL_INIT_DONE; 3770 done: 3771 if (rc != 0) 3772 adapter_full_uninit(sc); 3773 3774 return (rc); 3775 } 3776 3777 int 3778 adapter_full_uninit(struct adapter *sc) 3779 { 3780 int i; 3781 3782 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3783 3784 t4_teardown_adapter_queues(sc); 3785 3786 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3787 taskqueue_free(sc->tq[i]); 3788 sc->tq[i] = NULL; 3789 } 3790 3791 sc->flags &= ~FULL_INIT_DONE; 3792 3793 return (0); 3794 } 3795 3796 #ifdef RSS 3797 #define SUPPORTED_RSS_HASHTYPES (RSS_HASHTYPE_RSS_IPV4 | \ 3798 RSS_HASHTYPE_RSS_TCP_IPV4 | RSS_HASHTYPE_RSS_IPV6 | \ 3799 RSS_HASHTYPE_RSS_TCP_IPV6 | RSS_HASHTYPE_RSS_UDP_IPV4 | \ 3800 RSS_HASHTYPE_RSS_UDP_IPV6) 3801 3802 /* Translates kernel hash types to hardware. */ 3803 static int 3804 hashconfig_to_hashen(int hashconfig) 3805 { 3806 int hashen = 0; 3807 3808 if (hashconfig & RSS_HASHTYPE_RSS_IPV4) 3809 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN; 3810 if (hashconfig & RSS_HASHTYPE_RSS_IPV6) 3811 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN; 3812 if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV4) { 3813 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN | 3814 F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN; 3815 } 3816 if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV6) { 3817 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN | 3818 F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN; 3819 } 3820 if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV4) 3821 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN; 3822 if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV6) 3823 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN; 3824 3825 return (hashen); 3826 } 3827 3828 /* Translates hardware hash types to kernel. */ 3829 static int 3830 hashen_to_hashconfig(int hashen) 3831 { 3832 int hashconfig = 0; 3833 3834 if (hashen & F_FW_RSS_VI_CONFIG_CMD_UDPEN) { 3835 /* 3836 * If UDP hashing was enabled it must have been enabled for 3837 * either IPv4 or IPv6 (inclusive or). Enabling UDP without 3838 * enabling any 4-tuple hash is nonsense configuration. 3839 */ 3840 MPASS(hashen & (F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN | 3841 F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)); 3842 3843 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) 3844 hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV4; 3845 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) 3846 hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV6; 3847 } 3848 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) 3849 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV4; 3850 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) 3851 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV6; 3852 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN) 3853 hashconfig |= RSS_HASHTYPE_RSS_IPV4; 3854 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN) 3855 hashconfig |= RSS_HASHTYPE_RSS_IPV6; 3856 3857 return (hashconfig); 3858 } 3859 #endif 3860 3861 int 3862 vi_full_init(struct vi_info *vi) 3863 { 3864 struct adapter *sc = vi->pi->adapter; 3865 struct ifnet *ifp = vi->ifp; 3866 uint16_t *rss; 3867 struct sge_rxq *rxq; 3868 int rc, i, j, hashen; 3869 #ifdef RSS 3870 int nbuckets = rss_getnumbuckets(); 3871 int hashconfig = rss_gethashconfig(); 3872 int extra; 3873 uint32_t raw_rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; 3874 uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; 3875 #endif 3876 3877 ASSERT_SYNCHRONIZED_OP(sc); 3878 KASSERT((vi->flags & VI_INIT_DONE) == 0, 3879 ("%s: VI_INIT_DONE already", __func__)); 3880 3881 sysctl_ctx_init(&vi->ctx); 3882 vi->flags |= VI_SYSCTL_CTX; 3883 3884 /* 3885 * Allocate tx/rx/fl queues for this VI. 3886 */ 3887 rc = t4_setup_vi_queues(vi); 3888 if (rc != 0) 3889 goto done; /* error message displayed already */ 3890 3891 #ifdef DEV_NETMAP 3892 /* Netmap VIs configure RSS when netmap is enabled. */ 3893 if (vi->flags & VI_NETMAP) { 3894 vi->flags |= VI_INIT_DONE; 3895 return (0); 3896 } 3897 #endif 3898 3899 /* 3900 * Setup RSS for this VI. Save a copy of the RSS table for later use. 3901 */ 3902 if (vi->nrxq > vi->rss_size) { 3903 if_printf(ifp, "nrxq (%d) > hw RSS table size (%d); " 3904 "some queues will never receive traffic.\n", vi->nrxq, 3905 vi->rss_size); 3906 } else if (vi->rss_size % vi->nrxq) { 3907 if_printf(ifp, "nrxq (%d), hw RSS table size (%d); " 3908 "expect uneven traffic distribution.\n", vi->nrxq, 3909 vi->rss_size); 3910 } 3911 #ifdef RSS 3912 MPASS(RSS_KEYSIZE == 40); 3913 if (vi->nrxq != nbuckets) { 3914 if_printf(ifp, "nrxq (%d) != kernel RSS buckets (%d);" 3915 "performance will be impacted.\n", vi->nrxq, nbuckets); 3916 } 3917 3918 rss_getkey((void *)&raw_rss_key[0]); 3919 for (i = 0; i < nitems(rss_key); i++) { 3920 rss_key[i] = htobe32(raw_rss_key[nitems(rss_key) - 1 - i]); 3921 } 3922 t4_write_rss_key(sc, (void *)&rss_key[0], -1); 3923 #endif 3924 rss = malloc(vi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); 3925 for (i = 0; i < vi->rss_size;) { 3926 #ifdef RSS 3927 j = rss_get_indirection_to_bucket(i); 3928 j %= vi->nrxq; 3929 rxq = &sc->sge.rxq[vi->first_rxq + j]; 3930 rss[i++] = rxq->iq.abs_id; 3931 #else 3932 for_each_rxq(vi, j, rxq) { 3933 rss[i++] = rxq->iq.abs_id; 3934 if (i == vi->rss_size) 3935 break; 3936 } 3937 #endif 3938 } 3939 3940 rc = -t4_config_rss_range(sc, sc->mbox, vi->viid, 0, vi->rss_size, rss, 3941 vi->rss_size); 3942 if (rc != 0) { 3943 if_printf(ifp, "rss_config failed: %d\n", rc); 3944 goto done; 3945 } 3946 3947 #ifdef RSS 3948 hashen = hashconfig_to_hashen(hashconfig); 3949 3950 /* 3951 * We may have had to enable some hashes even though the global config 3952 * wants them disabled. This is a potential problem that must be 3953 * reported to the user. 3954 */ 3955 extra = hashen_to_hashconfig(hashen) ^ hashconfig; 3956 3957 /* 3958 * If we consider only the supported hash types, then the enabled hashes 3959 * are a superset of the requested hashes. In other words, there cannot 3960 * be any supported hash that was requested but not enabled, but there 3961 * can be hashes that were not requested but had to be enabled. 3962 */ 3963 extra &= SUPPORTED_RSS_HASHTYPES; 3964 MPASS((extra & hashconfig) == 0); 3965 3966 if (extra) { 3967 if_printf(ifp, 3968 "global RSS config (0x%x) cannot be accomodated.\n", 3969 hashconfig); 3970 } 3971 if (extra & RSS_HASHTYPE_RSS_IPV4) 3972 if_printf(ifp, "IPv4 2-tuple hashing forced on.\n"); 3973 if (extra & RSS_HASHTYPE_RSS_TCP_IPV4) 3974 if_printf(ifp, "TCP/IPv4 4-tuple hashing forced on.\n"); 3975 if (extra & RSS_HASHTYPE_RSS_IPV6) 3976 if_printf(ifp, "IPv6 2-tuple hashing forced on.\n"); 3977 if (extra & RSS_HASHTYPE_RSS_TCP_IPV6) 3978 if_printf(ifp, "TCP/IPv6 4-tuple hashing forced on.\n"); 3979 if (extra & RSS_HASHTYPE_RSS_UDP_IPV4) 3980 if_printf(ifp, "UDP/IPv4 4-tuple hashing forced on.\n"); 3981 if (extra & RSS_HASHTYPE_RSS_UDP_IPV6) 3982 if_printf(ifp, "UDP/IPv6 4-tuple hashing forced on.\n"); 3983 #else 3984 hashen = F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN | 3985 F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN | 3986 F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN | 3987 F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN | F_FW_RSS_VI_CONFIG_CMD_UDPEN; 3988 #endif 3989 rc = -t4_config_vi_rss(sc, sc->mbox, vi->viid, hashen, rss[0]); 3990 if (rc != 0) { 3991 if_printf(ifp, "rss hash/defaultq config failed: %d\n", rc); 3992 goto done; 3993 } 3994 3995 vi->rss = rss; 3996 vi->flags |= VI_INIT_DONE; 3997 done: 3998 if (rc != 0) 3999 vi_full_uninit(vi); 4000 4001 return (rc); 4002 } 4003 4004 /* 4005 * Idempotent. 4006 */ 4007 int 4008 vi_full_uninit(struct vi_info *vi) 4009 { 4010 struct port_info *pi = vi->pi; 4011 struct adapter *sc = pi->adapter; 4012 int i; 4013 struct sge_rxq *rxq; 4014 struct sge_txq *txq; 4015 #ifdef TCP_OFFLOAD 4016 struct sge_ofld_rxq *ofld_rxq; 4017 struct sge_wrq *ofld_txq; 4018 #endif 4019 4020 if (vi->flags & VI_INIT_DONE) { 4021 4022 /* Need to quiesce queues. */ 4023 #ifdef DEV_NETMAP 4024 if (vi->flags & VI_NETMAP) 4025 goto skip; 4026 #endif 4027 4028 /* XXX: Only for the first VI? */ 4029 if (IS_MAIN_VI(vi)) 4030 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]); 4031 4032 for_each_txq(vi, i, txq) { 4033 quiesce_txq(sc, txq); 4034 } 4035 4036 #ifdef TCP_OFFLOAD 4037 for_each_ofld_txq(vi, i, ofld_txq) { 4038 quiesce_wrq(sc, ofld_txq); 4039 } 4040 #endif 4041 4042 for_each_rxq(vi, i, rxq) { 4043 quiesce_iq(sc, &rxq->iq); 4044 quiesce_fl(sc, &rxq->fl); 4045 } 4046 4047 #ifdef TCP_OFFLOAD 4048 for_each_ofld_rxq(vi, i, ofld_rxq) { 4049 quiesce_iq(sc, &ofld_rxq->iq); 4050 quiesce_fl(sc, &ofld_rxq->fl); 4051 } 4052 #endif 4053 free(vi->rss, M_CXGBE); 4054 } 4055 #ifdef DEV_NETMAP 4056 skip: 4057 #endif 4058 4059 t4_teardown_vi_queues(vi); 4060 vi->flags &= ~VI_INIT_DONE; 4061 4062 return (0); 4063 } 4064 4065 static void 4066 quiesce_txq(struct adapter *sc, struct sge_txq *txq) 4067 { 4068 struct sge_eq *eq = &txq->eq; 4069 struct sge_qstat *spg = (void *)&eq->desc[eq->sidx]; 4070 4071 (void) sc; /* unused */ 4072 4073 #ifdef INVARIANTS 4074 TXQ_LOCK(txq); 4075 MPASS((eq->flags & EQ_ENABLED) == 0); 4076 TXQ_UNLOCK(txq); 4077 #endif 4078 4079 /* Wait for the mp_ring to empty. */ 4080 while (!mp_ring_is_idle(txq->r)) { 4081 mp_ring_check_drainage(txq->r, 0); 4082 pause("rquiesce", 1); 4083 } 4084 4085 /* Then wait for the hardware to finish. */ 4086 while (spg->cidx != htobe16(eq->pidx)) 4087 pause("equiesce", 1); 4088 4089 /* Finally, wait for the driver to reclaim all descriptors. */ 4090 while (eq->cidx != eq->pidx) 4091 pause("dquiesce", 1); 4092 } 4093 4094 static void 4095 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq) 4096 { 4097 4098 /* XXXTX */ 4099 } 4100 4101 static void 4102 quiesce_iq(struct adapter *sc, struct sge_iq *iq) 4103 { 4104 (void) sc; /* unused */ 4105 4106 /* Synchronize with the interrupt handler */ 4107 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 4108 pause("iqfree", 1); 4109 } 4110 4111 static void 4112 quiesce_fl(struct adapter *sc, struct sge_fl *fl) 4113 { 4114 mtx_lock(&sc->sfl_lock); 4115 FL_LOCK(fl); 4116 fl->flags |= FL_DOOMED; 4117 FL_UNLOCK(fl); 4118 callout_stop(&sc->sfl_callout); 4119 mtx_unlock(&sc->sfl_lock); 4120 4121 KASSERT((fl->flags & FL_STARVING) == 0, 4122 ("%s: still starving", __func__)); 4123 } 4124 4125 static int 4126 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 4127 driver_intr_t *handler, void *arg, char *name) 4128 { 4129 int rc; 4130 4131 irq->rid = rid; 4132 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 4133 RF_SHAREABLE | RF_ACTIVE); 4134 if (irq->res == NULL) { 4135 device_printf(sc->dev, 4136 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 4137 return (ENOMEM); 4138 } 4139 4140 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 4141 NULL, handler, arg, &irq->tag); 4142 if (rc != 0) { 4143 device_printf(sc->dev, 4144 "failed to setup interrupt for rid %d, name %s: %d\n", 4145 rid, name, rc); 4146 } else if (name) 4147 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 4148 4149 return (rc); 4150 } 4151 4152 static int 4153 t4_free_irq(struct adapter *sc, struct irq *irq) 4154 { 4155 if (irq->tag) 4156 bus_teardown_intr(sc->dev, irq->res, irq->tag); 4157 if (irq->res) 4158 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 4159 4160 bzero(irq, sizeof(*irq)); 4161 4162 return (0); 4163 } 4164 4165 static void 4166 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 4167 unsigned int end) 4168 { 4169 uint32_t *p = (uint32_t *)(buf + start); 4170 4171 for ( ; start <= end; start += sizeof(uint32_t)) 4172 *p++ = t4_read_reg(sc, start); 4173 } 4174 4175 static void 4176 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 4177 { 4178 int i, n; 4179 const unsigned int *reg_ranges; 4180 static const unsigned int t4_reg_ranges[] = { 4181 0x1008, 0x1108, 4182 0x1180, 0x11b4, 4183 0x11fc, 0x123c, 4184 0x1300, 0x173c, 4185 0x1800, 0x18fc, 4186 0x3000, 0x30d8, 4187 0x30e0, 0x5924, 4188 0x5960, 0x59d4, 4189 0x5a00, 0x5af8, 4190 0x6000, 0x6098, 4191 0x6100, 0x6150, 4192 0x6200, 0x6208, 4193 0x6240, 0x6248, 4194 0x6280, 0x6338, 4195 0x6370, 0x638c, 4196 0x6400, 0x643c, 4197 0x6500, 0x6524, 4198 0x6a00, 0x6a38, 4199 0x6a60, 0x6a78, 4200 0x6b00, 0x6b84, 4201 0x6bf0, 0x6c84, 4202 0x6cf0, 0x6d84, 4203 0x6df0, 0x6e84, 4204 0x6ef0, 0x6f84, 4205 0x6ff0, 0x7084, 4206 0x70f0, 0x7184, 4207 0x71f0, 0x7284, 4208 0x72f0, 0x7384, 4209 0x73f0, 0x7450, 4210 0x7500, 0x7530, 4211 0x7600, 0x761c, 4212 0x7680, 0x76cc, 4213 0x7700, 0x7798, 4214 0x77c0, 0x77fc, 4215 0x7900, 0x79fc, 4216 0x7b00, 0x7c38, 4217 0x7d00, 0x7efc, 4218 0x8dc0, 0x8e1c, 4219 0x8e30, 0x8e78, 4220 0x8ea0, 0x8f6c, 4221 0x8fc0, 0x9074, 4222 0x90fc, 0x90fc, 4223 0x9400, 0x9458, 4224 0x9600, 0x96bc, 4225 0x9800, 0x9808, 4226 0x9820, 0x983c, 4227 0x9850, 0x9864, 4228 0x9c00, 0x9c6c, 4229 0x9c80, 0x9cec, 4230 0x9d00, 0x9d6c, 4231 0x9d80, 0x9dec, 4232 0x9e00, 0x9e6c, 4233 0x9e80, 0x9eec, 4234 0x9f00, 0x9f6c, 4235 0x9f80, 0x9fec, 4236 0xd004, 0xd03c, 4237 0xdfc0, 0xdfe0, 4238 0xe000, 0xea7c, 4239 0xf000, 0x11110, 4240 0x11118, 0x11190, 4241 0x19040, 0x1906c, 4242 0x19078, 0x19080, 4243 0x1908c, 0x19124, 4244 0x19150, 0x191b0, 4245 0x191d0, 0x191e8, 4246 0x19238, 0x1924c, 4247 0x193f8, 0x19474, 4248 0x19490, 0x194f8, 4249 0x19800, 0x19f30, 4250 0x1a000, 0x1a06c, 4251 0x1a0b0, 0x1a120, 4252 0x1a128, 0x1a138, 4253 0x1a190, 0x1a1c4, 4254 0x1a1fc, 0x1a1fc, 4255 0x1e040, 0x1e04c, 4256 0x1e284, 0x1e28c, 4257 0x1e2c0, 0x1e2c0, 4258 0x1e2e0, 0x1e2e0, 4259 0x1e300, 0x1e384, 4260 0x1e3c0, 0x1e3c8, 4261 0x1e440, 0x1e44c, 4262 0x1e684, 0x1e68c, 4263 0x1e6c0, 0x1e6c0, 4264 0x1e6e0, 0x1e6e0, 4265 0x1e700, 0x1e784, 4266 0x1e7c0, 0x1e7c8, 4267 0x1e840, 0x1e84c, 4268 0x1ea84, 0x1ea8c, 4269 0x1eac0, 0x1eac0, 4270 0x1eae0, 0x1eae0, 4271 0x1eb00, 0x1eb84, 4272 0x1ebc0, 0x1ebc8, 4273 0x1ec40, 0x1ec4c, 4274 0x1ee84, 0x1ee8c, 4275 0x1eec0, 0x1eec0, 4276 0x1eee0, 0x1eee0, 4277 0x1ef00, 0x1ef84, 4278 0x1efc0, 0x1efc8, 4279 0x1f040, 0x1f04c, 4280 0x1f284, 0x1f28c, 4281 0x1f2c0, 0x1f2c0, 4282 0x1f2e0, 0x1f2e0, 4283 0x1f300, 0x1f384, 4284 0x1f3c0, 0x1f3c8, 4285 0x1f440, 0x1f44c, 4286 0x1f684, 0x1f68c, 4287 0x1f6c0, 0x1f6c0, 4288 0x1f6e0, 0x1f6e0, 4289 0x1f700, 0x1f784, 4290 0x1f7c0, 0x1f7c8, 4291 0x1f840, 0x1f84c, 4292 0x1fa84, 0x1fa8c, 4293 0x1fac0, 0x1fac0, 4294 0x1fae0, 0x1fae0, 4295 0x1fb00, 0x1fb84, 4296 0x1fbc0, 0x1fbc8, 4297 0x1fc40, 0x1fc4c, 4298 0x1fe84, 0x1fe8c, 4299 0x1fec0, 0x1fec0, 4300 0x1fee0, 0x1fee0, 4301 0x1ff00, 0x1ff84, 4302 0x1ffc0, 0x1ffc8, 4303 0x20000, 0x2002c, 4304 0x20100, 0x2013c, 4305 0x20190, 0x201c8, 4306 0x20200, 0x20318, 4307 0x20400, 0x20528, 4308 0x20540, 0x20614, 4309 0x21000, 0x21040, 4310 0x2104c, 0x21060, 4311 0x210c0, 0x210ec, 4312 0x21200, 0x21268, 4313 0x21270, 0x21284, 4314 0x212fc, 0x21388, 4315 0x21400, 0x21404, 4316 0x21500, 0x21518, 4317 0x2152c, 0x2153c, 4318 0x21550, 0x21554, 4319 0x21600, 0x21600, 4320 0x21608, 0x21628, 4321 0x21630, 0x2163c, 4322 0x21700, 0x2171c, 4323 0x21780, 0x2178c, 4324 0x21800, 0x21c38, 4325 0x21c80, 0x21d7c, 4326 0x21e00, 0x21e04, 4327 0x22000, 0x2202c, 4328 0x22100, 0x2213c, 4329 0x22190, 0x221c8, 4330 0x22200, 0x22318, 4331 0x22400, 0x22528, 4332 0x22540, 0x22614, 4333 0x23000, 0x23040, 4334 0x2304c, 0x23060, 4335 0x230c0, 0x230ec, 4336 0x23200, 0x23268, 4337 0x23270, 0x23284, 4338 0x232fc, 0x23388, 4339 0x23400, 0x23404, 4340 0x23500, 0x23518, 4341 0x2352c, 0x2353c, 4342 0x23550, 0x23554, 4343 0x23600, 0x23600, 4344 0x23608, 0x23628, 4345 0x23630, 0x2363c, 4346 0x23700, 0x2371c, 4347 0x23780, 0x2378c, 4348 0x23800, 0x23c38, 4349 0x23c80, 0x23d7c, 4350 0x23e00, 0x23e04, 4351 0x24000, 0x2402c, 4352 0x24100, 0x2413c, 4353 0x24190, 0x241c8, 4354 0x24200, 0x24318, 4355 0x24400, 0x24528, 4356 0x24540, 0x24614, 4357 0x25000, 0x25040, 4358 0x2504c, 0x25060, 4359 0x250c0, 0x250ec, 4360 0x25200, 0x25268, 4361 0x25270, 0x25284, 4362 0x252fc, 0x25388, 4363 0x25400, 0x25404, 4364 0x25500, 0x25518, 4365 0x2552c, 0x2553c, 4366 0x25550, 0x25554, 4367 0x25600, 0x25600, 4368 0x25608, 0x25628, 4369 0x25630, 0x2563c, 4370 0x25700, 0x2571c, 4371 0x25780, 0x2578c, 4372 0x25800, 0x25c38, 4373 0x25c80, 0x25d7c, 4374 0x25e00, 0x25e04, 4375 0x26000, 0x2602c, 4376 0x26100, 0x2613c, 4377 0x26190, 0x261c8, 4378 0x26200, 0x26318, 4379 0x26400, 0x26528, 4380 0x26540, 0x26614, 4381 0x27000, 0x27040, 4382 0x2704c, 0x27060, 4383 0x270c0, 0x270ec, 4384 0x27200, 0x27268, 4385 0x27270, 0x27284, 4386 0x272fc, 0x27388, 4387 0x27400, 0x27404, 4388 0x27500, 0x27518, 4389 0x2752c, 0x2753c, 4390 0x27550, 0x27554, 4391 0x27600, 0x27600, 4392 0x27608, 0x27628, 4393 0x27630, 0x2763c, 4394 0x27700, 0x2771c, 4395 0x27780, 0x2778c, 4396 0x27800, 0x27c38, 4397 0x27c80, 0x27d7c, 4398 0x27e00, 0x27e04 4399 }; 4400 static const unsigned int t5_reg_ranges[] = { 4401 0x1008, 0x1148, 4402 0x1180, 0x11b4, 4403 0x11fc, 0x123c, 4404 0x1280, 0x173c, 4405 0x1800, 0x18fc, 4406 0x3000, 0x3028, 4407 0x3060, 0x30d8, 4408 0x30e0, 0x30fc, 4409 0x3140, 0x357c, 4410 0x35a8, 0x35cc, 4411 0x35ec, 0x35ec, 4412 0x3600, 0x5624, 4413 0x56cc, 0x575c, 4414 0x580c, 0x5814, 4415 0x5890, 0x58bc, 4416 0x5940, 0x59dc, 4417 0x59fc, 0x5a18, 4418 0x5a60, 0x5a9c, 4419 0x5b94, 0x5bfc, 4420 0x6000, 0x6040, 4421 0x6058, 0x614c, 4422 0x7700, 0x7798, 4423 0x77c0, 0x78fc, 4424 0x7b00, 0x7c54, 4425 0x7d00, 0x7efc, 4426 0x8dc0, 0x8de0, 4427 0x8df8, 0x8e84, 4428 0x8ea0, 0x8f84, 4429 0x8fc0, 0x90f8, 4430 0x9400, 0x9470, 4431 0x9600, 0x96f4, 4432 0x9800, 0x9808, 4433 0x9820, 0x983c, 4434 0x9850, 0x9864, 4435 0x9c00, 0x9c6c, 4436 0x9c80, 0x9cec, 4437 0x9d00, 0x9d6c, 4438 0x9d80, 0x9dec, 4439 0x9e00, 0x9e6c, 4440 0x9e80, 0x9eec, 4441 0x9f00, 0x9f6c, 4442 0x9f80, 0xa020, 4443 0xd004, 0xd03c, 4444 0xdfc0, 0xdfe0, 4445 0xe000, 0x11088, 4446 0x1109c, 0x11110, 4447 0x11118, 0x1117c, 4448 0x11190, 0x11204, 4449 0x19040, 0x1906c, 4450 0x19078, 0x19080, 4451 0x1908c, 0x19124, 4452 0x19150, 0x191b0, 4453 0x191d0, 0x191e8, 4454 0x19238, 0x19290, 4455 0x193f8, 0x19474, 4456 0x19490, 0x194cc, 4457 0x194f0, 0x194f8, 4458 0x19c00, 0x19c60, 4459 0x19c94, 0x19e10, 4460 0x19e50, 0x19f34, 4461 0x19f40, 0x19f50, 4462 0x19f90, 0x19fe4, 4463 0x1a000, 0x1a06c, 4464 0x1a0b0, 0x1a120, 4465 0x1a128, 0x1a138, 4466 0x1a190, 0x1a1c4, 4467 0x1a1fc, 0x1a1fc, 4468 0x1e008, 0x1e00c, 4469 0x1e040, 0x1e04c, 4470 0x1e284, 0x1e290, 4471 0x1e2c0, 0x1e2c0, 4472 0x1e2e0, 0x1e2e0, 4473 0x1e300, 0x1e384, 4474 0x1e3c0, 0x1e3c8, 4475 0x1e408, 0x1e40c, 4476 0x1e440, 0x1e44c, 4477 0x1e684, 0x1e690, 4478 0x1e6c0, 0x1e6c0, 4479 0x1e6e0, 0x1e6e0, 4480 0x1e700, 0x1e784, 4481 0x1e7c0, 0x1e7c8, 4482 0x1e808, 0x1e80c, 4483 0x1e840, 0x1e84c, 4484 0x1ea84, 0x1ea90, 4485 0x1eac0, 0x1eac0, 4486 0x1eae0, 0x1eae0, 4487 0x1eb00, 0x1eb84, 4488 0x1ebc0, 0x1ebc8, 4489 0x1ec08, 0x1ec0c, 4490 0x1ec40, 0x1ec4c, 4491 0x1ee84, 0x1ee90, 4492 0x1eec0, 0x1eec0, 4493 0x1eee0, 0x1eee0, 4494 0x1ef00, 0x1ef84, 4495 0x1efc0, 0x1efc8, 4496 0x1f008, 0x1f00c, 4497 0x1f040, 0x1f04c, 4498 0x1f284, 0x1f290, 4499 0x1f2c0, 0x1f2c0, 4500 0x1f2e0, 0x1f2e0, 4501 0x1f300, 0x1f384, 4502 0x1f3c0, 0x1f3c8, 4503 0x1f408, 0x1f40c, 4504 0x1f440, 0x1f44c, 4505 0x1f684, 0x1f690, 4506 0x1f6c0, 0x1f6c0, 4507 0x1f6e0, 0x1f6e0, 4508 0x1f700, 0x1f784, 4509 0x1f7c0, 0x1f7c8, 4510 0x1f808, 0x1f80c, 4511 0x1f840, 0x1f84c, 4512 0x1fa84, 0x1fa90, 4513 0x1fac0, 0x1fac0, 4514 0x1fae0, 0x1fae0, 4515 0x1fb00, 0x1fb84, 4516 0x1fbc0, 0x1fbc8, 4517 0x1fc08, 0x1fc0c, 4518 0x1fc40, 0x1fc4c, 4519 0x1fe84, 0x1fe90, 4520 0x1fec0, 0x1fec0, 4521 0x1fee0, 0x1fee0, 4522 0x1ff00, 0x1ff84, 4523 0x1ffc0, 0x1ffc8, 4524 0x30000, 0x30030, 4525 0x30100, 0x30144, 4526 0x30190, 0x301d0, 4527 0x30200, 0x30318, 4528 0x30400, 0x3052c, 4529 0x30540, 0x3061c, 4530 0x30800, 0x30834, 4531 0x308c0, 0x30908, 4532 0x30910, 0x309ac, 4533 0x30a00, 0x30a2c, 4534 0x30a44, 0x30a50, 4535 0x30a74, 0x30c24, 4536 0x30d00, 0x30d00, 4537 0x30d08, 0x30d14, 4538 0x30d1c, 0x30d20, 4539 0x30d3c, 0x30d50, 4540 0x31200, 0x3120c, 4541 0x31220, 0x31220, 4542 0x31240, 0x31240, 4543 0x31600, 0x3160c, 4544 0x31a00, 0x31a1c, 4545 0x31e00, 0x31e20, 4546 0x31e38, 0x31e3c, 4547 0x31e80, 0x31e80, 4548 0x31e88, 0x31ea8, 4549 0x31eb0, 0x31eb4, 4550 0x31ec8, 0x31ed4, 4551 0x31fb8, 0x32004, 4552 0x32200, 0x32200, 4553 0x32208, 0x32240, 4554 0x32248, 0x32280, 4555 0x32288, 0x322c0, 4556 0x322c8, 0x322fc, 4557 0x32600, 0x32630, 4558 0x32a00, 0x32abc, 4559 0x32b00, 0x32b70, 4560 0x33000, 0x33048, 4561 0x33060, 0x3309c, 4562 0x330f0, 0x33148, 4563 0x33160, 0x3319c, 4564 0x331f0, 0x332e4, 4565 0x332f8, 0x333e4, 4566 0x333f8, 0x33448, 4567 0x33460, 0x3349c, 4568 0x334f0, 0x33548, 4569 0x33560, 0x3359c, 4570 0x335f0, 0x336e4, 4571 0x336f8, 0x337e4, 4572 0x337f8, 0x337fc, 4573 0x33814, 0x33814, 4574 0x3382c, 0x3382c, 4575 0x33880, 0x3388c, 4576 0x338e8, 0x338ec, 4577 0x33900, 0x33948, 4578 0x33960, 0x3399c, 4579 0x339f0, 0x33ae4, 4580 0x33af8, 0x33b10, 4581 0x33b28, 0x33b28, 4582 0x33b3c, 0x33b50, 4583 0x33bf0, 0x33c10, 4584 0x33c28, 0x33c28, 4585 0x33c3c, 0x33c50, 4586 0x33cf0, 0x33cfc, 4587 0x34000, 0x34030, 4588 0x34100, 0x34144, 4589 0x34190, 0x341d0, 4590 0x34200, 0x34318, 4591 0x34400, 0x3452c, 4592 0x34540, 0x3461c, 4593 0x34800, 0x34834, 4594 0x348c0, 0x34908, 4595 0x34910, 0x349ac, 4596 0x34a00, 0x34a2c, 4597 0x34a44, 0x34a50, 4598 0x34a74, 0x34c24, 4599 0x34d00, 0x34d00, 4600 0x34d08, 0x34d14, 4601 0x34d1c, 0x34d20, 4602 0x34d3c, 0x34d50, 4603 0x35200, 0x3520c, 4604 0x35220, 0x35220, 4605 0x35240, 0x35240, 4606 0x35600, 0x3560c, 4607 0x35a00, 0x35a1c, 4608 0x35e00, 0x35e20, 4609 0x35e38, 0x35e3c, 4610 0x35e80, 0x35e80, 4611 0x35e88, 0x35ea8, 4612 0x35eb0, 0x35eb4, 4613 0x35ec8, 0x35ed4, 4614 0x35fb8, 0x36004, 4615 0x36200, 0x36200, 4616 0x36208, 0x36240, 4617 0x36248, 0x36280, 4618 0x36288, 0x362c0, 4619 0x362c8, 0x362fc, 4620 0x36600, 0x36630, 4621 0x36a00, 0x36abc, 4622 0x36b00, 0x36b70, 4623 0x37000, 0x37048, 4624 0x37060, 0x3709c, 4625 0x370f0, 0x37148, 4626 0x37160, 0x3719c, 4627 0x371f0, 0x372e4, 4628 0x372f8, 0x373e4, 4629 0x373f8, 0x37448, 4630 0x37460, 0x3749c, 4631 0x374f0, 0x37548, 4632 0x37560, 0x3759c, 4633 0x375f0, 0x376e4, 4634 0x376f8, 0x377e4, 4635 0x377f8, 0x377fc, 4636 0x37814, 0x37814, 4637 0x3782c, 0x3782c, 4638 0x37880, 0x3788c, 4639 0x378e8, 0x378ec, 4640 0x37900, 0x37948, 4641 0x37960, 0x3799c, 4642 0x379f0, 0x37ae4, 4643 0x37af8, 0x37b10, 4644 0x37b28, 0x37b28, 4645 0x37b3c, 0x37b50, 4646 0x37bf0, 0x37c10, 4647 0x37c28, 0x37c28, 4648 0x37c3c, 0x37c50, 4649 0x37cf0, 0x37cfc, 4650 0x38000, 0x38030, 4651 0x38100, 0x38144, 4652 0x38190, 0x381d0, 4653 0x38200, 0x38318, 4654 0x38400, 0x3852c, 4655 0x38540, 0x3861c, 4656 0x38800, 0x38834, 4657 0x388c0, 0x38908, 4658 0x38910, 0x389ac, 4659 0x38a00, 0x38a2c, 4660 0x38a44, 0x38a50, 4661 0x38a74, 0x38c24, 4662 0x38d00, 0x38d00, 4663 0x38d08, 0x38d14, 4664 0x38d1c, 0x38d20, 4665 0x38d3c, 0x38d50, 4666 0x39200, 0x3920c, 4667 0x39220, 0x39220, 4668 0x39240, 0x39240, 4669 0x39600, 0x3960c, 4670 0x39a00, 0x39a1c, 4671 0x39e00, 0x39e20, 4672 0x39e38, 0x39e3c, 4673 0x39e80, 0x39e80, 4674 0x39e88, 0x39ea8, 4675 0x39eb0, 0x39eb4, 4676 0x39ec8, 0x39ed4, 4677 0x39fb8, 0x3a004, 4678 0x3a200, 0x3a200, 4679 0x3a208, 0x3a240, 4680 0x3a248, 0x3a280, 4681 0x3a288, 0x3a2c0, 4682 0x3a2c8, 0x3a2fc, 4683 0x3a600, 0x3a630, 4684 0x3aa00, 0x3aabc, 4685 0x3ab00, 0x3ab70, 4686 0x3b000, 0x3b048, 4687 0x3b060, 0x3b09c, 4688 0x3b0f0, 0x3b148, 4689 0x3b160, 0x3b19c, 4690 0x3b1f0, 0x3b2e4, 4691 0x3b2f8, 0x3b3e4, 4692 0x3b3f8, 0x3b448, 4693 0x3b460, 0x3b49c, 4694 0x3b4f0, 0x3b548, 4695 0x3b560, 0x3b59c, 4696 0x3b5f0, 0x3b6e4, 4697 0x3b6f8, 0x3b7e4, 4698 0x3b7f8, 0x3b7fc, 4699 0x3b814, 0x3b814, 4700 0x3b82c, 0x3b82c, 4701 0x3b880, 0x3b88c, 4702 0x3b8e8, 0x3b8ec, 4703 0x3b900, 0x3b948, 4704 0x3b960, 0x3b99c, 4705 0x3b9f0, 0x3bae4, 4706 0x3baf8, 0x3bb10, 4707 0x3bb28, 0x3bb28, 4708 0x3bb3c, 0x3bb50, 4709 0x3bbf0, 0x3bc10, 4710 0x3bc28, 0x3bc28, 4711 0x3bc3c, 0x3bc50, 4712 0x3bcf0, 0x3bcfc, 4713 0x3c000, 0x3c030, 4714 0x3c100, 0x3c144, 4715 0x3c190, 0x3c1d0, 4716 0x3c200, 0x3c318, 4717 0x3c400, 0x3c52c, 4718 0x3c540, 0x3c61c, 4719 0x3c800, 0x3c834, 4720 0x3c8c0, 0x3c908, 4721 0x3c910, 0x3c9ac, 4722 0x3ca00, 0x3ca2c, 4723 0x3ca44, 0x3ca50, 4724 0x3ca74, 0x3cc24, 4725 0x3cd00, 0x3cd00, 4726 0x3cd08, 0x3cd14, 4727 0x3cd1c, 0x3cd20, 4728 0x3cd3c, 0x3cd50, 4729 0x3d200, 0x3d20c, 4730 0x3d220, 0x3d220, 4731 0x3d240, 0x3d240, 4732 0x3d600, 0x3d60c, 4733 0x3da00, 0x3da1c, 4734 0x3de00, 0x3de20, 4735 0x3de38, 0x3de3c, 4736 0x3de80, 0x3de80, 4737 0x3de88, 0x3dea8, 4738 0x3deb0, 0x3deb4, 4739 0x3dec8, 0x3ded4, 4740 0x3dfb8, 0x3e004, 4741 0x3e200, 0x3e200, 4742 0x3e208, 0x3e240, 4743 0x3e248, 0x3e280, 4744 0x3e288, 0x3e2c0, 4745 0x3e2c8, 0x3e2fc, 4746 0x3e600, 0x3e630, 4747 0x3ea00, 0x3eabc, 4748 0x3eb00, 0x3eb70, 4749 0x3f000, 0x3f048, 4750 0x3f060, 0x3f09c, 4751 0x3f0f0, 0x3f148, 4752 0x3f160, 0x3f19c, 4753 0x3f1f0, 0x3f2e4, 4754 0x3f2f8, 0x3f3e4, 4755 0x3f3f8, 0x3f448, 4756 0x3f460, 0x3f49c, 4757 0x3f4f0, 0x3f548, 4758 0x3f560, 0x3f59c, 4759 0x3f5f0, 0x3f6e4, 4760 0x3f6f8, 0x3f7e4, 4761 0x3f7f8, 0x3f7fc, 4762 0x3f814, 0x3f814, 4763 0x3f82c, 0x3f82c, 4764 0x3f880, 0x3f88c, 4765 0x3f8e8, 0x3f8ec, 4766 0x3f900, 0x3f948, 4767 0x3f960, 0x3f99c, 4768 0x3f9f0, 0x3fae4, 4769 0x3faf8, 0x3fb10, 4770 0x3fb28, 0x3fb28, 4771 0x3fb3c, 0x3fb50, 4772 0x3fbf0, 0x3fc10, 4773 0x3fc28, 0x3fc28, 4774 0x3fc3c, 0x3fc50, 4775 0x3fcf0, 0x3fcfc, 4776 0x40000, 0x4000c, 4777 0x40040, 0x40068, 4778 0x4007c, 0x40144, 4779 0x40180, 0x4018c, 4780 0x40200, 0x40298, 4781 0x402ac, 0x4033c, 4782 0x403f8, 0x403fc, 4783 0x41304, 0x413c4, 4784 0x41400, 0x4141c, 4785 0x41480, 0x414d0, 4786 0x44000, 0x44078, 4787 0x440c0, 0x44278, 4788 0x442c0, 0x44478, 4789 0x444c0, 0x44678, 4790 0x446c0, 0x44878, 4791 0x448c0, 0x449fc, 4792 0x45000, 0x45068, 4793 0x45080, 0x45084, 4794 0x450a0, 0x450b0, 4795 0x45200, 0x45268, 4796 0x45280, 0x45284, 4797 0x452a0, 0x452b0, 4798 0x460c0, 0x460e4, 4799 0x47000, 0x4708c, 4800 0x47200, 0x47250, 4801 0x47400, 0x47420, 4802 0x47600, 0x47618, 4803 0x47800, 0x47814, 4804 0x48000, 0x4800c, 4805 0x48040, 0x48068, 4806 0x4807c, 0x48144, 4807 0x48180, 0x4818c, 4808 0x48200, 0x48298, 4809 0x482ac, 0x4833c, 4810 0x483f8, 0x483fc, 4811 0x49304, 0x493c4, 4812 0x49400, 0x4941c, 4813 0x49480, 0x494d0, 4814 0x4c000, 0x4c078, 4815 0x4c0c0, 0x4c278, 4816 0x4c2c0, 0x4c478, 4817 0x4c4c0, 0x4c678, 4818 0x4c6c0, 0x4c878, 4819 0x4c8c0, 0x4c9fc, 4820 0x4d000, 0x4d068, 4821 0x4d080, 0x4d084, 4822 0x4d0a0, 0x4d0b0, 4823 0x4d200, 0x4d268, 4824 0x4d280, 0x4d284, 4825 0x4d2a0, 0x4d2b0, 4826 0x4e0c0, 0x4e0e4, 4827 0x4f000, 0x4f08c, 4828 0x4f200, 0x4f250, 4829 0x4f400, 0x4f420, 4830 0x4f600, 0x4f618, 4831 0x4f800, 0x4f814, 4832 0x50000, 0x500cc, 4833 0x50400, 0x50400, 4834 0x50800, 0x508cc, 4835 0x50c00, 0x50c00, 4836 0x51000, 0x5101c, 4837 0x51300, 0x51308, 4838 }; 4839 4840 if (is_t4(sc)) { 4841 reg_ranges = &t4_reg_ranges[0]; 4842 n = nitems(t4_reg_ranges); 4843 } else { 4844 reg_ranges = &t5_reg_ranges[0]; 4845 n = nitems(t5_reg_ranges); 4846 } 4847 4848 regs->version = chip_id(sc) | chip_rev(sc) << 10; 4849 for (i = 0; i < n; i += 2) 4850 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 4851 } 4852 4853 #define A_PL_INDIR_CMD 0x1f8 4854 4855 #define S_PL_AUTOINC 31 4856 #define M_PL_AUTOINC 0x1U 4857 #define V_PL_AUTOINC(x) ((x) << S_PL_AUTOINC) 4858 #define G_PL_AUTOINC(x) (((x) >> S_PL_AUTOINC) & M_PL_AUTOINC) 4859 4860 #define S_PL_VFID 20 4861 #define M_PL_VFID 0xffU 4862 #define V_PL_VFID(x) ((x) << S_PL_VFID) 4863 #define G_PL_VFID(x) (((x) >> S_PL_VFID) & M_PL_VFID) 4864 4865 #define S_PL_ADDR 0 4866 #define M_PL_ADDR 0xfffffU 4867 #define V_PL_ADDR(x) ((x) << S_PL_ADDR) 4868 #define G_PL_ADDR(x) (((x) >> S_PL_ADDR) & M_PL_ADDR) 4869 4870 #define A_PL_INDIR_DATA 0x1fc 4871 4872 static uint64_t 4873 read_vf_stat(struct adapter *sc, unsigned int viid, int reg) 4874 { 4875 u32 stats[2]; 4876 4877 mtx_assert(&sc->regwin_lock, MA_OWNED); 4878 t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) | 4879 V_PL_VFID(G_FW_VIID_VIN(viid)) | V_PL_ADDR(VF_MPS_REG(reg))); 4880 stats[0] = t4_read_reg(sc, A_PL_INDIR_DATA); 4881 stats[1] = t4_read_reg(sc, A_PL_INDIR_DATA); 4882 return (((uint64_t)stats[1]) << 32 | stats[0]); 4883 } 4884 4885 static void 4886 t4_get_vi_stats(struct adapter *sc, unsigned int viid, 4887 struct fw_vi_stats_vf *stats) 4888 { 4889 4890 #define GET_STAT(name) \ 4891 read_vf_stat(sc, viid, A_MPS_VF_STAT_##name##_L) 4892 4893 stats->tx_bcast_bytes = GET_STAT(TX_VF_BCAST_BYTES); 4894 stats->tx_bcast_frames = GET_STAT(TX_VF_BCAST_FRAMES); 4895 stats->tx_mcast_bytes = GET_STAT(TX_VF_MCAST_BYTES); 4896 stats->tx_mcast_frames = GET_STAT(TX_VF_MCAST_FRAMES); 4897 stats->tx_ucast_bytes = GET_STAT(TX_VF_UCAST_BYTES); 4898 stats->tx_ucast_frames = GET_STAT(TX_VF_UCAST_FRAMES); 4899 stats->tx_drop_frames = GET_STAT(TX_VF_DROP_FRAMES); 4900 stats->tx_offload_bytes = GET_STAT(TX_VF_OFFLOAD_BYTES); 4901 stats->tx_offload_frames = GET_STAT(TX_VF_OFFLOAD_FRAMES); 4902 stats->rx_bcast_bytes = GET_STAT(RX_VF_BCAST_BYTES); 4903 stats->rx_bcast_frames = GET_STAT(RX_VF_BCAST_FRAMES); 4904 stats->rx_mcast_bytes = GET_STAT(RX_VF_MCAST_BYTES); 4905 stats->rx_mcast_frames = GET_STAT(RX_VF_MCAST_FRAMES); 4906 stats->rx_ucast_bytes = GET_STAT(RX_VF_UCAST_BYTES); 4907 stats->rx_ucast_frames = GET_STAT(RX_VF_UCAST_FRAMES); 4908 stats->rx_err_frames = GET_STAT(RX_VF_ERR_FRAMES); 4909 4910 #undef GET_STAT 4911 } 4912 4913 static void 4914 t4_clr_vi_stats(struct adapter *sc, unsigned int viid) 4915 { 4916 int reg; 4917 4918 t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) | 4919 V_PL_VFID(G_FW_VIID_VIN(viid)) | 4920 V_PL_ADDR(VF_MPS_REG(A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L))); 4921 for (reg = A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L; 4922 reg <= A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H; reg += 4) 4923 t4_write_reg(sc, A_PL_INDIR_DATA, 0); 4924 } 4925 4926 static void 4927 vi_refresh_stats(struct adapter *sc, struct vi_info *vi) 4928 { 4929 struct timeval tv; 4930 const struct timeval interval = {0, 250000}; /* 250ms */ 4931 4932 if (!(vi->flags & VI_INIT_DONE)) 4933 return; 4934 4935 getmicrotime(&tv); 4936 timevalsub(&tv, &interval); 4937 if (timevalcmp(&tv, &vi->last_refreshed, <)) 4938 return; 4939 4940 mtx_lock(&sc->regwin_lock); 4941 t4_get_vi_stats(sc, vi->viid, &vi->stats); 4942 getmicrotime(&vi->last_refreshed); 4943 mtx_unlock(&sc->regwin_lock); 4944 } 4945 4946 static void 4947 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi) 4948 { 4949 int i; 4950 u_int v, tnl_cong_drops; 4951 struct timeval tv; 4952 const struct timeval interval = {0, 250000}; /* 250ms */ 4953 4954 getmicrotime(&tv); 4955 timevalsub(&tv, &interval); 4956 if (timevalcmp(&tv, &pi->last_refreshed, <)) 4957 return; 4958 4959 tnl_cong_drops = 0; 4960 t4_get_port_stats(sc, pi->tx_chan, &pi->stats); 4961 for (i = 0; i < sc->chip_params->nchan; i++) { 4962 if (pi->rx_chan_map & (1 << i)) { 4963 mtx_lock(&sc->regwin_lock); 4964 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 4965 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 4966 mtx_unlock(&sc->regwin_lock); 4967 tnl_cong_drops += v; 4968 } 4969 } 4970 pi->tnl_cong_drops = tnl_cong_drops; 4971 getmicrotime(&pi->last_refreshed); 4972 } 4973 4974 static void 4975 cxgbe_tick(void *arg) 4976 { 4977 struct port_info *pi = arg; 4978 struct adapter *sc = pi->adapter; 4979 4980 PORT_LOCK_ASSERT_OWNED(pi); 4981 cxgbe_refresh_stats(sc, pi); 4982 4983 callout_schedule(&pi->tick, hz); 4984 } 4985 4986 void 4987 vi_tick(void *arg) 4988 { 4989 struct vi_info *vi = arg; 4990 struct adapter *sc = vi->pi->adapter; 4991 4992 vi_refresh_stats(sc, vi); 4993 4994 callout_schedule(&vi->tick, hz); 4995 } 4996 4997 static void 4998 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 4999 { 5000 struct ifnet *vlan; 5001 5002 if (arg != ifp || ifp->if_type != IFT_ETHER) 5003 return; 5004 5005 vlan = VLAN_DEVAT(ifp, vid); 5006 VLAN_SETCOOKIE(vlan, ifp); 5007 } 5008 5009 static int 5010 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 5011 { 5012 5013 #ifdef INVARIANTS 5014 panic("%s: opcode 0x%02x on iq %p with payload %p", 5015 __func__, rss->opcode, iq, m); 5016 #else 5017 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 5018 __func__, rss->opcode, iq, m); 5019 m_freem(m); 5020 #endif 5021 return (EDOOFUS); 5022 } 5023 5024 int 5025 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 5026 { 5027 uintptr_t *loc, new; 5028 5029 if (opcode >= nitems(sc->cpl_handler)) 5030 return (EINVAL); 5031 5032 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 5033 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 5034 atomic_store_rel_ptr(loc, new); 5035 5036 return (0); 5037 } 5038 5039 static int 5040 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 5041 { 5042 5043 #ifdef INVARIANTS 5044 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 5045 #else 5046 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 5047 __func__, iq, ctrl); 5048 #endif 5049 return (EDOOFUS); 5050 } 5051 5052 int 5053 t4_register_an_handler(struct adapter *sc, an_handler_t h) 5054 { 5055 uintptr_t *loc, new; 5056 5057 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 5058 loc = (uintptr_t *) &sc->an_handler; 5059 atomic_store_rel_ptr(loc, new); 5060 5061 return (0); 5062 } 5063 5064 static int 5065 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 5066 { 5067 const struct cpl_fw6_msg *cpl = 5068 __containerof(rpl, struct cpl_fw6_msg, data[0]); 5069 5070 #ifdef INVARIANTS 5071 panic("%s: fw_msg type %d", __func__, cpl->type); 5072 #else 5073 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 5074 #endif 5075 return (EDOOFUS); 5076 } 5077 5078 int 5079 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 5080 { 5081 uintptr_t *loc, new; 5082 5083 if (type >= nitems(sc->fw_msg_handler)) 5084 return (EINVAL); 5085 5086 /* 5087 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 5088 * handler dispatch table. Reject any attempt to install a handler for 5089 * this subtype. 5090 */ 5091 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 5092 return (EINVAL); 5093 5094 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 5095 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 5096 atomic_store_rel_ptr(loc, new); 5097 5098 return (0); 5099 } 5100 5101 static void 5102 t4_sysctls(struct adapter *sc) 5103 { 5104 struct sysctl_ctx_list *ctx; 5105 struct sysctl_oid *oid; 5106 struct sysctl_oid_list *children, *c0; 5107 static char *caps[] = { 5108 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 5109 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL" /* caps[1] niccaps */ 5110 "\6HASHFILTER\7ETHOFLD", 5111 "\20\1TOE", /* caps[2] toecaps */ 5112 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 5113 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 5114 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 5115 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 5116 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 5117 "\4PO_INITIAOR\5PO_TARGET" 5118 }; 5119 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 5120 5121 ctx = device_get_sysctl_ctx(sc->dev); 5122 5123 /* 5124 * dev.t4nex.X. 5125 */ 5126 oid = device_get_sysctl_tree(sc->dev); 5127 c0 = children = SYSCTL_CHILDREN(oid); 5128 5129 sc->sc_do_rxcopy = 1; 5130 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, 5131 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); 5132 5133 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 5134 sc->params.nports, "# of ports"); 5135 5136 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 5137 NULL, chip_rev(sc), "chip hardware revision"); 5138 5139 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 5140 CTLFLAG_RD, sc->fw_version, 0, "firmware version"); 5141 5142 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 5143 CTLFLAG_RD, sc->cfg_file, 0, "configuration file"); 5144 5145 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 5146 sc->cfcsum, "config file checksum"); 5147 5148 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 5149 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 5150 sysctl_bitfield, "A", "available doorbells"); 5151 5152 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 5153 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 5154 sysctl_bitfield, "A", "available link capabilities"); 5155 5156 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 5157 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 5158 sysctl_bitfield, "A", "available NIC capabilities"); 5159 5160 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 5161 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 5162 sysctl_bitfield, "A", "available TCP offload capabilities"); 5163 5164 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 5165 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 5166 sysctl_bitfield, "A", "available RDMA capabilities"); 5167 5168 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 5169 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 5170 sysctl_bitfield, "A", "available iSCSI capabilities"); 5171 5172 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 5173 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 5174 sysctl_bitfield, "A", "available FCoE capabilities"); 5175 5176 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 5177 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 5178 5179 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 5180 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 5181 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 5182 "interrupt holdoff timer values (us)"); 5183 5184 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 5185 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 5186 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 5187 "interrupt holdoff packet counter values"); 5188 5189 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, 5190 NULL, sc->tids.nftids, "number of filters"); 5191 5192 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | 5193 CTLFLAG_RD, sc, 0, sysctl_temperature, "I", 5194 "chip temperature (in Celsius)"); 5195 5196 t4_sge_sysctls(sc, ctx, children); 5197 5198 sc->lro_timeout = 100; 5199 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, 5200 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); 5201 5202 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "debug_flags", CTLFLAG_RW, 5203 &sc->debug_flags, 0, "flags to enable runtime debugging"); 5204 5205 #ifdef SBUF_DRAIN 5206 /* 5207 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 5208 */ 5209 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 5210 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 5211 "logs and miscellaneous information"); 5212 children = SYSCTL_CHILDREN(oid); 5213 5214 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 5215 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5216 sysctl_cctrl, "A", "congestion control"); 5217 5218 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 5219 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5220 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 5221 5222 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 5223 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 5224 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 5225 5226 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 5227 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 5228 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 5229 5230 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 5231 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 5232 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 5233 5234 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 5235 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 5236 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 5237 5238 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 5239 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 5240 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 5241 5242 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 5243 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5244 chip_id(sc) <= CHELSIO_T5 ? sysctl_cim_la : sysctl_cim_la_t6, 5245 "A", "CIM logic analyzer"); 5246 5247 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", 5248 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5249 sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); 5250 5251 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 5252 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 5253 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 5254 5255 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 5256 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 5257 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 5258 5259 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 5260 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 5261 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 5262 5263 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 5264 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 5265 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 5266 5267 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 5268 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 5269 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 5270 5271 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 5272 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 5273 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 5274 5275 if (chip_id(sc) > CHELSIO_T4) { 5276 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 5277 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 5278 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 5279 5280 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 5281 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 5282 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 5283 } 5284 5285 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", 5286 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5287 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); 5288 5289 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 5290 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5291 sysctl_cim_qcfg, "A", "CIM queue configuration"); 5292 5293 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 5294 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5295 sysctl_cpl_stats, "A", "CPL statistics"); 5296 5297 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 5298 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5299 sysctl_ddp_stats, "A", "non-TCP DDP statistics"); 5300 5301 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 5302 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5303 sysctl_devlog, "A", "firmware's device log"); 5304 5305 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 5306 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5307 sysctl_fcoe_stats, "A", "FCoE statistics"); 5308 5309 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 5310 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5311 sysctl_hw_sched, "A", "hardware scheduler "); 5312 5313 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 5314 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5315 sysctl_l2t, "A", "hardware L2 table"); 5316 5317 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 5318 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5319 sysctl_lb_stats, "A", "loopback statistics"); 5320 5321 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 5322 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5323 sysctl_meminfo, "A", "memory regions"); 5324 5325 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", 5326 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5327 chip_id(sc) <= CHELSIO_T5 ? sysctl_mps_tcam : sysctl_mps_tcam_t6, 5328 "A", "MPS TCAM entries"); 5329 5330 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 5331 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5332 sysctl_path_mtus, "A", "path MTUs"); 5333 5334 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 5335 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5336 sysctl_pm_stats, "A", "PM statistics"); 5337 5338 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 5339 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5340 sysctl_rdma_stats, "A", "RDMA statistics"); 5341 5342 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 5343 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5344 sysctl_tcp_stats, "A", "TCP statistics"); 5345 5346 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 5347 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5348 sysctl_tids, "A", "TID information"); 5349 5350 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 5351 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5352 sysctl_tp_err_stats, "A", "TP error statistics"); 5353 5354 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", 5355 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5356 sysctl_tp_la, "A", "TP logic analyzer"); 5357 5358 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 5359 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5360 sysctl_tx_rate, "A", "Tx rate"); 5361 5362 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", 5363 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5364 sysctl_ulprx_la, "A", "ULPRX logic analyzer"); 5365 5366 if (is_t5(sc)) { 5367 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 5368 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5369 sysctl_wcwr_stats, "A", "write combined work requests"); 5370 } 5371 #endif 5372 5373 #ifdef TCP_OFFLOAD 5374 if (is_offload(sc)) { 5375 /* 5376 * dev.t4nex.X.toe. 5377 */ 5378 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 5379 NULL, "TOE parameters"); 5380 children = SYSCTL_CHILDREN(oid); 5381 5382 sc->tt.sndbuf = 256 * 1024; 5383 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 5384 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 5385 5386 sc->tt.ddp = 0; 5387 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 5388 &sc->tt.ddp, 0, "DDP allowed"); 5389 5390 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 5391 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 5392 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 5393 5394 sc->tt.ddp_thres = 5395 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 5396 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 5397 &sc->tt.ddp_thres, 0, "DDP threshold"); 5398 5399 sc->tt.rx_coalesce = 1; 5400 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", 5401 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); 5402 5403 sc->tt.tx_align = 1; 5404 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align", 5405 CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload"); 5406 } 5407 #endif 5408 } 5409 5410 void 5411 vi_sysctls(struct vi_info *vi) 5412 { 5413 struct sysctl_ctx_list *ctx; 5414 struct sysctl_oid *oid; 5415 struct sysctl_oid_list *children; 5416 5417 ctx = device_get_sysctl_ctx(vi->dev); 5418 5419 /* 5420 * dev.[nv](cxgbe|cxl).X. 5421 */ 5422 oid = device_get_sysctl_tree(vi->dev); 5423 children = SYSCTL_CHILDREN(oid); 5424 5425 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "viid", CTLFLAG_RD, NULL, 5426 vi->viid, "VI identifer"); 5427 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 5428 &vi->nrxq, 0, "# of rx queues"); 5429 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 5430 &vi->ntxq, 0, "# of tx queues"); 5431 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 5432 &vi->first_rxq, 0, "index of first rx queue"); 5433 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 5434 &vi->first_txq, 0, "index of first tx queue"); 5435 5436 if (vi->flags & VI_NETMAP) 5437 return; 5438 5439 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT | 5440 CTLFLAG_RW, vi, 0, sysctl_noflowq, "IU", 5441 "Reserve queue 0 for non-flowid packets"); 5442 5443 #ifdef TCP_OFFLOAD 5444 if (vi->nofldrxq != 0) { 5445 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 5446 &vi->nofldrxq, 0, 5447 "# of rx queues for offloaded TCP connections"); 5448 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 5449 &vi->nofldtxq, 0, 5450 "# of tx queues for offloaded TCP connections"); 5451 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 5452 CTLFLAG_RD, &vi->first_ofld_rxq, 0, 5453 "index of first TOE rx queue"); 5454 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 5455 CTLFLAG_RD, &vi->first_ofld_txq, 0, 5456 "index of first TOE tx queue"); 5457 } 5458 #endif 5459 5460 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 5461 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_holdoff_tmr_idx, "I", 5462 "holdoff timer index"); 5463 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 5464 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_holdoff_pktc_idx, "I", 5465 "holdoff packet counter index"); 5466 5467 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 5468 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_qsize_rxq, "I", 5469 "rx queue size"); 5470 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 5471 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_qsize_txq, "I", 5472 "tx queue size"); 5473 } 5474 5475 static void 5476 cxgbe_sysctls(struct port_info *pi) 5477 { 5478 struct sysctl_ctx_list *ctx; 5479 struct sysctl_oid *oid; 5480 struct sysctl_oid_list *children; 5481 struct adapter *sc = pi->adapter; 5482 5483 ctx = device_get_sysctl_ctx(pi->dev); 5484 5485 /* 5486 * dev.cxgbe.X. 5487 */ 5488 oid = device_get_sysctl_tree(pi->dev); 5489 children = SYSCTL_CHILDREN(oid); 5490 5491 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | 5492 CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); 5493 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { 5494 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 5495 CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I", 5496 "PHY temperature (in Celsius)"); 5497 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", 5498 CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I", 5499 "PHY firmware version"); 5500 } 5501 5502 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings", 5503 CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings, 5504 "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)"); 5505 5506 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "max_speed", CTLFLAG_RD, NULL, 5507 port_top_speed(pi), "max speed (in Gbps)"); 5508 5509 /* 5510 * dev.cxgbe.X.stats. 5511 */ 5512 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 5513 NULL, "port statistics"); 5514 children = SYSCTL_CHILDREN(oid); 5515 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD, 5516 &pi->tx_parse_error, 0, 5517 "# of tx packets with invalid length or # of segments"); 5518 5519 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 5520 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 5521 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ 5522 sysctl_handle_t4_reg64, "QU", desc) 5523 5524 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 5525 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 5526 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 5527 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 5528 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 5529 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 5530 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 5531 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 5532 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 5533 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 5534 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 5535 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 5536 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 5537 "# of tx frames in this range", 5538 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 5539 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 5540 "# of tx frames in this range", 5541 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 5542 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 5543 "# of tx frames in this range", 5544 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 5545 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 5546 "# of tx frames in this range", 5547 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 5548 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 5549 "# of tx frames in this range", 5550 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 5551 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 5552 "# of tx frames in this range", 5553 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 5554 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 5555 "# of tx frames in this range", 5556 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 5557 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 5558 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 5559 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 5560 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 5561 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 5562 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 5563 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 5564 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 5565 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 5566 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 5567 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 5568 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 5569 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 5570 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 5571 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 5572 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 5573 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 5574 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 5575 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 5576 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 5577 5578 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 5579 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 5580 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 5581 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 5582 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 5583 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 5584 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 5585 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 5586 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 5587 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 5588 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 5589 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 5590 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 5591 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 5592 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 5593 "# of frames received with bad FCS", 5594 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 5595 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 5596 "# of frames received with length error", 5597 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 5598 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 5599 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 5600 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 5601 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 5602 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 5603 "# of rx frames in this range", 5604 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 5605 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 5606 "# of rx frames in this range", 5607 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 5608 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 5609 "# of rx frames in this range", 5610 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 5611 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 5612 "# of rx frames in this range", 5613 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 5614 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 5615 "# of rx frames in this range", 5616 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 5617 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 5618 "# of rx frames in this range", 5619 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 5620 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 5621 "# of rx frames in this range", 5622 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 5623 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 5624 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 5625 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 5626 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 5627 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 5628 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 5629 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 5630 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 5631 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 5632 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 5633 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 5634 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 5635 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 5636 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 5637 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 5638 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 5639 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 5640 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 5641 5642 #undef SYSCTL_ADD_T4_REG64 5643 5644 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 5645 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 5646 &pi->stats.name, desc) 5647 5648 /* We get these from port_stats and they may be stale by upto 1s */ 5649 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 5650 "# drops due to buffer-group 0 overflows"); 5651 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 5652 "# drops due to buffer-group 1 overflows"); 5653 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 5654 "# drops due to buffer-group 2 overflows"); 5655 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 5656 "# drops due to buffer-group 3 overflows"); 5657 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 5658 "# of buffer-group 0 truncated packets"); 5659 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 5660 "# of buffer-group 1 truncated packets"); 5661 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 5662 "# of buffer-group 2 truncated packets"); 5663 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 5664 "# of buffer-group 3 truncated packets"); 5665 5666 #undef SYSCTL_ADD_T4_PORTSTAT 5667 } 5668 5669 static int 5670 sysctl_int_array(SYSCTL_HANDLER_ARGS) 5671 { 5672 int rc, *i, space = 0; 5673 struct sbuf sb; 5674 5675 sbuf_new_for_sysctl(&sb, NULL, 64, req); 5676 for (i = arg1; arg2; arg2 -= sizeof(int), i++) { 5677 if (space) 5678 sbuf_printf(&sb, " "); 5679 sbuf_printf(&sb, "%d", *i); 5680 space = 1; 5681 } 5682 rc = sbuf_finish(&sb); 5683 sbuf_delete(&sb); 5684 return (rc); 5685 } 5686 5687 static int 5688 sysctl_bitfield(SYSCTL_HANDLER_ARGS) 5689 { 5690 int rc; 5691 struct sbuf *sb; 5692 5693 rc = sysctl_wire_old_buffer(req, 0); 5694 if (rc != 0) 5695 return(rc); 5696 5697 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5698 if (sb == NULL) 5699 return (ENOMEM); 5700 5701 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 5702 rc = sbuf_finish(sb); 5703 sbuf_delete(sb); 5704 5705 return (rc); 5706 } 5707 5708 static int 5709 sysctl_btphy(SYSCTL_HANDLER_ARGS) 5710 { 5711 struct port_info *pi = arg1; 5712 int op = arg2; 5713 struct adapter *sc = pi->adapter; 5714 u_int v; 5715 int rc; 5716 5717 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, "t4btt"); 5718 if (rc) 5719 return (rc); 5720 /* XXX: magic numbers */ 5721 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, 5722 &v); 5723 end_synchronized_op(sc, 0); 5724 if (rc) 5725 return (rc); 5726 if (op == 0) 5727 v /= 256; 5728 5729 rc = sysctl_handle_int(oidp, &v, 0, req); 5730 return (rc); 5731 } 5732 5733 static int 5734 sysctl_noflowq(SYSCTL_HANDLER_ARGS) 5735 { 5736 struct vi_info *vi = arg1; 5737 int rc, val; 5738 5739 val = vi->rsrv_noflowq; 5740 rc = sysctl_handle_int(oidp, &val, 0, req); 5741 if (rc != 0 || req->newptr == NULL) 5742 return (rc); 5743 5744 if ((val >= 1) && (vi->ntxq > 1)) 5745 vi->rsrv_noflowq = 1; 5746 else 5747 vi->rsrv_noflowq = 0; 5748 5749 return (rc); 5750 } 5751 5752 static int 5753 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 5754 { 5755 struct vi_info *vi = arg1; 5756 struct adapter *sc = vi->pi->adapter; 5757 int idx, rc, i; 5758 struct sge_rxq *rxq; 5759 #ifdef TCP_OFFLOAD 5760 struct sge_ofld_rxq *ofld_rxq; 5761 #endif 5762 uint8_t v; 5763 5764 idx = vi->tmr_idx; 5765 5766 rc = sysctl_handle_int(oidp, &idx, 0, req); 5767 if (rc != 0 || req->newptr == NULL) 5768 return (rc); 5769 5770 if (idx < 0 || idx >= SGE_NTIMERS) 5771 return (EINVAL); 5772 5773 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5774 "t4tmr"); 5775 if (rc) 5776 return (rc); 5777 5778 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(vi->pktc_idx != -1); 5779 for_each_rxq(vi, i, rxq) { 5780 #ifdef atomic_store_rel_8 5781 atomic_store_rel_8(&rxq->iq.intr_params, v); 5782 #else 5783 rxq->iq.intr_params = v; 5784 #endif 5785 } 5786 #ifdef TCP_OFFLOAD 5787 for_each_ofld_rxq(vi, i, ofld_rxq) { 5788 #ifdef atomic_store_rel_8 5789 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); 5790 #else 5791 ofld_rxq->iq.intr_params = v; 5792 #endif 5793 } 5794 #endif 5795 vi->tmr_idx = idx; 5796 5797 end_synchronized_op(sc, LOCK_HELD); 5798 return (0); 5799 } 5800 5801 static int 5802 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 5803 { 5804 struct vi_info *vi = arg1; 5805 struct adapter *sc = vi->pi->adapter; 5806 int idx, rc; 5807 5808 idx = vi->pktc_idx; 5809 5810 rc = sysctl_handle_int(oidp, &idx, 0, req); 5811 if (rc != 0 || req->newptr == NULL) 5812 return (rc); 5813 5814 if (idx < -1 || idx >= SGE_NCOUNTERS) 5815 return (EINVAL); 5816 5817 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5818 "t4pktc"); 5819 if (rc) 5820 return (rc); 5821 5822 if (vi->flags & VI_INIT_DONE) 5823 rc = EBUSY; /* cannot be changed once the queues are created */ 5824 else 5825 vi->pktc_idx = idx; 5826 5827 end_synchronized_op(sc, LOCK_HELD); 5828 return (rc); 5829 } 5830 5831 static int 5832 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 5833 { 5834 struct vi_info *vi = arg1; 5835 struct adapter *sc = vi->pi->adapter; 5836 int qsize, rc; 5837 5838 qsize = vi->qsize_rxq; 5839 5840 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5841 if (rc != 0 || req->newptr == NULL) 5842 return (rc); 5843 5844 if (qsize < 128 || (qsize & 7)) 5845 return (EINVAL); 5846 5847 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5848 "t4rxqs"); 5849 if (rc) 5850 return (rc); 5851 5852 if (vi->flags & VI_INIT_DONE) 5853 rc = EBUSY; /* cannot be changed once the queues are created */ 5854 else 5855 vi->qsize_rxq = qsize; 5856 5857 end_synchronized_op(sc, LOCK_HELD); 5858 return (rc); 5859 } 5860 5861 static int 5862 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 5863 { 5864 struct vi_info *vi = arg1; 5865 struct adapter *sc = vi->pi->adapter; 5866 int qsize, rc; 5867 5868 qsize = vi->qsize_txq; 5869 5870 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5871 if (rc != 0 || req->newptr == NULL) 5872 return (rc); 5873 5874 if (qsize < 128 || qsize > 65536) 5875 return (EINVAL); 5876 5877 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5878 "t4txqs"); 5879 if (rc) 5880 return (rc); 5881 5882 if (vi->flags & VI_INIT_DONE) 5883 rc = EBUSY; /* cannot be changed once the queues are created */ 5884 else 5885 vi->qsize_txq = qsize; 5886 5887 end_synchronized_op(sc, LOCK_HELD); 5888 return (rc); 5889 } 5890 5891 static int 5892 sysctl_pause_settings(SYSCTL_HANDLER_ARGS) 5893 { 5894 struct port_info *pi = arg1; 5895 struct adapter *sc = pi->adapter; 5896 struct link_config *lc = &pi->link_cfg; 5897 int rc; 5898 5899 if (req->newptr == NULL) { 5900 struct sbuf *sb; 5901 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX"; 5902 5903 rc = sysctl_wire_old_buffer(req, 0); 5904 if (rc != 0) 5905 return(rc); 5906 5907 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5908 if (sb == NULL) 5909 return (ENOMEM); 5910 5911 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits); 5912 rc = sbuf_finish(sb); 5913 sbuf_delete(sb); 5914 } else { 5915 char s[2]; 5916 int n; 5917 5918 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX)); 5919 s[1] = 0; 5920 5921 rc = sysctl_handle_string(oidp, s, sizeof(s), req); 5922 if (rc != 0) 5923 return(rc); 5924 5925 if (s[1] != 0) 5926 return (EINVAL); 5927 if (s[0] < '0' || s[0] > '9') 5928 return (EINVAL); /* not a number */ 5929 n = s[0] - '0'; 5930 if (n & ~(PAUSE_TX | PAUSE_RX)) 5931 return (EINVAL); /* some other bit is set too */ 5932 5933 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, 5934 "t4PAUSE"); 5935 if (rc) 5936 return (rc); 5937 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) { 5938 int link_ok = lc->link_ok; 5939 5940 lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX); 5941 lc->requested_fc |= n; 5942 rc = -t4_link_l1cfg(sc, sc->mbox, pi->tx_chan, lc); 5943 lc->link_ok = link_ok; /* restore */ 5944 } 5945 end_synchronized_op(sc, 0); 5946 } 5947 5948 return (rc); 5949 } 5950 5951 static int 5952 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 5953 { 5954 struct adapter *sc = arg1; 5955 int reg = arg2; 5956 uint64_t val; 5957 5958 val = t4_read_reg64(sc, reg); 5959 5960 return (sysctl_handle_64(oidp, &val, 0, req)); 5961 } 5962 5963 static int 5964 sysctl_temperature(SYSCTL_HANDLER_ARGS) 5965 { 5966 struct adapter *sc = arg1; 5967 int rc, t; 5968 uint32_t param, val; 5969 5970 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); 5971 if (rc) 5972 return (rc); 5973 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 5974 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 5975 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); 5976 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 5977 end_synchronized_op(sc, 0); 5978 if (rc) 5979 return (rc); 5980 5981 /* unknown is returned as 0 but we display -1 in that case */ 5982 t = val == 0 ? -1 : val; 5983 5984 rc = sysctl_handle_int(oidp, &t, 0, req); 5985 return (rc); 5986 } 5987 5988 #ifdef SBUF_DRAIN 5989 static int 5990 sysctl_cctrl(SYSCTL_HANDLER_ARGS) 5991 { 5992 struct adapter *sc = arg1; 5993 struct sbuf *sb; 5994 int rc, i; 5995 uint16_t incr[NMTUS][NCCTRL_WIN]; 5996 static const char *dec_fac[] = { 5997 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 5998 "0.9375" 5999 }; 6000 6001 rc = sysctl_wire_old_buffer(req, 0); 6002 if (rc != 0) 6003 return (rc); 6004 6005 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6006 if (sb == NULL) 6007 return (ENOMEM); 6008 6009 t4_read_cong_tbl(sc, incr); 6010 6011 for (i = 0; i < NCCTRL_WIN; ++i) { 6012 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 6013 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 6014 incr[5][i], incr[6][i], incr[7][i]); 6015 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 6016 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 6017 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 6018 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 6019 } 6020 6021 rc = sbuf_finish(sb); 6022 sbuf_delete(sb); 6023 6024 return (rc); 6025 } 6026 6027 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 6028 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 6029 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 6030 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 6031 }; 6032 6033 static int 6034 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 6035 { 6036 struct adapter *sc = arg1; 6037 struct sbuf *sb; 6038 int rc, i, n, qid = arg2; 6039 uint32_t *buf, *p; 6040 char *qtype; 6041 u_int cim_num_obq = sc->chip_params->cim_num_obq; 6042 6043 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 6044 ("%s: bad qid %d\n", __func__, qid)); 6045 6046 if (qid < CIM_NUM_IBQ) { 6047 /* inbound queue */ 6048 qtype = "IBQ"; 6049 n = 4 * CIM_IBQ_SIZE; 6050 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 6051 rc = t4_read_cim_ibq(sc, qid, buf, n); 6052 } else { 6053 /* outbound queue */ 6054 qtype = "OBQ"; 6055 qid -= CIM_NUM_IBQ; 6056 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 6057 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 6058 rc = t4_read_cim_obq(sc, qid, buf, n); 6059 } 6060 6061 if (rc < 0) { 6062 rc = -rc; 6063 goto done; 6064 } 6065 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 6066 6067 rc = sysctl_wire_old_buffer(req, 0); 6068 if (rc != 0) 6069 goto done; 6070 6071 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 6072 if (sb == NULL) { 6073 rc = ENOMEM; 6074 goto done; 6075 } 6076 6077 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 6078 for (i = 0, p = buf; i < n; i += 16, p += 4) 6079 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 6080 p[2], p[3]); 6081 6082 rc = sbuf_finish(sb); 6083 sbuf_delete(sb); 6084 done: 6085 free(buf, M_CXGBE); 6086 return (rc); 6087 } 6088 6089 static int 6090 sysctl_cim_la(SYSCTL_HANDLER_ARGS) 6091 { 6092 struct adapter *sc = arg1; 6093 u_int cfg; 6094 struct sbuf *sb; 6095 uint32_t *buf, *p; 6096 int rc; 6097 6098 MPASS(chip_id(sc) <= CHELSIO_T5); 6099 6100 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 6101 if (rc != 0) 6102 return (rc); 6103 6104 rc = sysctl_wire_old_buffer(req, 0); 6105 if (rc != 0) 6106 return (rc); 6107 6108 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6109 if (sb == NULL) 6110 return (ENOMEM); 6111 6112 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 6113 M_ZERO | M_WAITOK); 6114 6115 rc = -t4_cim_read_la(sc, buf, NULL); 6116 if (rc != 0) 6117 goto done; 6118 6119 sbuf_printf(sb, "Status Data PC%s", 6120 cfg & F_UPDBGLACAPTPCONLY ? "" : 6121 " LS0Stat LS0Addr LS0Data"); 6122 6123 for (p = buf; p <= &buf[sc->params.cim_la_size - 8]; p += 8) { 6124 if (cfg & F_UPDBGLACAPTPCONLY) { 6125 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 6126 p[6], p[7]); 6127 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 6128 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 6129 p[4] & 0xff, p[5] >> 8); 6130 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 6131 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 6132 p[1] & 0xf, p[2] >> 4); 6133 } else { 6134 sbuf_printf(sb, 6135 "\n %02x %x%07x %x%07x %08x %08x " 6136 "%08x%08x%08x%08x", 6137 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 6138 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 6139 p[6], p[7]); 6140 } 6141 } 6142 6143 rc = sbuf_finish(sb); 6144 sbuf_delete(sb); 6145 done: 6146 free(buf, M_CXGBE); 6147 return (rc); 6148 } 6149 6150 static int 6151 sysctl_cim_la_t6(SYSCTL_HANDLER_ARGS) 6152 { 6153 struct adapter *sc = arg1; 6154 u_int cfg; 6155 struct sbuf *sb; 6156 uint32_t *buf, *p; 6157 int rc; 6158 6159 MPASS(chip_id(sc) > CHELSIO_T5); 6160 6161 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 6162 if (rc != 0) 6163 return (rc); 6164 6165 rc = sysctl_wire_old_buffer(req, 0); 6166 if (rc != 0) 6167 return (rc); 6168 6169 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6170 if (sb == NULL) 6171 return (ENOMEM); 6172 6173 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 6174 M_ZERO | M_WAITOK); 6175 6176 rc = -t4_cim_read_la(sc, buf, NULL); 6177 if (rc != 0) 6178 goto done; 6179 6180 sbuf_printf(sb, "Status Inst Data PC%s", 6181 cfg & F_UPDBGLACAPTPCONLY ? "" : 6182 " LS0Stat LS0Addr LS0Data LS1Stat LS1Addr LS1Data"); 6183 6184 for (p = buf; p <= &buf[sc->params.cim_la_size - 10]; p += 10) { 6185 if (cfg & F_UPDBGLACAPTPCONLY) { 6186 sbuf_printf(sb, "\n %02x %08x %08x %08x", 6187 p[3] & 0xff, p[2], p[1], p[0]); 6188 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x %02x%06x", 6189 (p[6] >> 8) & 0xff, p[6] & 0xff, p[5] >> 8, 6190 p[5] & 0xff, p[4] >> 8, p[4] & 0xff, p[3] >> 8); 6191 sbuf_printf(sb, "\n %02x %04x%04x %04x%04x %04x%04x", 6192 (p[9] >> 16) & 0xff, p[9] & 0xffff, p[8] >> 16, 6193 p[8] & 0xffff, p[7] >> 16, p[7] & 0xffff, 6194 p[6] >> 16); 6195 } else { 6196 sbuf_printf(sb, "\n %02x %04x%04x %04x%04x %04x%04x " 6197 "%08x %08x %08x %08x %08x %08x", 6198 (p[9] >> 16) & 0xff, 6199 p[9] & 0xffff, p[8] >> 16, 6200 p[8] & 0xffff, p[7] >> 16, 6201 p[7] & 0xffff, p[6] >> 16, 6202 p[2], p[1], p[0], p[5], p[4], p[3]); 6203 } 6204 } 6205 6206 rc = sbuf_finish(sb); 6207 sbuf_delete(sb); 6208 done: 6209 free(buf, M_CXGBE); 6210 return (rc); 6211 } 6212 6213 static int 6214 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) 6215 { 6216 struct adapter *sc = arg1; 6217 u_int i; 6218 struct sbuf *sb; 6219 uint32_t *buf, *p; 6220 int rc; 6221 6222 rc = sysctl_wire_old_buffer(req, 0); 6223 if (rc != 0) 6224 return (rc); 6225 6226 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6227 if (sb == NULL) 6228 return (ENOMEM); 6229 6230 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, 6231 M_ZERO | M_WAITOK); 6232 6233 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); 6234 p = buf; 6235 6236 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 6237 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], 6238 p[1], p[0]); 6239 } 6240 6241 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); 6242 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 6243 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", 6244 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, 6245 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, 6246 (p[1] >> 2) | ((p[2] & 3) << 30), 6247 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, 6248 p[0] & 1); 6249 } 6250 6251 rc = sbuf_finish(sb); 6252 sbuf_delete(sb); 6253 free(buf, M_CXGBE); 6254 return (rc); 6255 } 6256 6257 static int 6258 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) 6259 { 6260 struct adapter *sc = arg1; 6261 u_int i; 6262 struct sbuf *sb; 6263 uint32_t *buf, *p; 6264 int rc; 6265 6266 rc = sysctl_wire_old_buffer(req, 0); 6267 if (rc != 0) 6268 return (rc); 6269 6270 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6271 if (sb == NULL) 6272 return (ENOMEM); 6273 6274 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, 6275 M_ZERO | M_WAITOK); 6276 6277 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); 6278 p = buf; 6279 6280 sbuf_printf(sb, "Cntl ID DataBE Addr Data"); 6281 for (i = 0; i < CIM_PIFLA_SIZE; i++, p += 6) { 6282 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", 6283 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, 6284 p[4], p[3], p[2], p[1], p[0]); 6285 } 6286 6287 sbuf_printf(sb, "\n\nCntl ID Data"); 6288 for (i = 0; i < CIM_PIFLA_SIZE; i++, p += 6) { 6289 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", 6290 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); 6291 } 6292 6293 rc = sbuf_finish(sb); 6294 sbuf_delete(sb); 6295 free(buf, M_CXGBE); 6296 return (rc); 6297 } 6298 6299 static int 6300 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 6301 { 6302 struct adapter *sc = arg1; 6303 struct sbuf *sb; 6304 int rc, i; 6305 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 6306 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 6307 uint16_t thres[CIM_NUM_IBQ]; 6308 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 6309 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 6310 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 6311 6312 cim_num_obq = sc->chip_params->cim_num_obq; 6313 if (is_t4(sc)) { 6314 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 6315 obq_rdaddr = A_UP_OBQ_0_REALADDR; 6316 } else { 6317 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 6318 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 6319 } 6320 nq = CIM_NUM_IBQ + cim_num_obq; 6321 6322 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 6323 if (rc == 0) 6324 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 6325 if (rc != 0) 6326 return (rc); 6327 6328 t4_read_cimq_cfg(sc, base, size, thres); 6329 6330 rc = sysctl_wire_old_buffer(req, 0); 6331 if (rc != 0) 6332 return (rc); 6333 6334 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 6335 if (sb == NULL) 6336 return (ENOMEM); 6337 6338 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 6339 6340 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 6341 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 6342 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 6343 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 6344 G_QUEREMFLITS(p[2]) * 16); 6345 for ( ; i < nq; i++, p += 4, wr += 2) 6346 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 6347 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 6348 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 6349 G_QUEREMFLITS(p[2]) * 16); 6350 6351 rc = sbuf_finish(sb); 6352 sbuf_delete(sb); 6353 6354 return (rc); 6355 } 6356 6357 static int 6358 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 6359 { 6360 struct adapter *sc = arg1; 6361 struct sbuf *sb; 6362 int rc; 6363 struct tp_cpl_stats stats; 6364 6365 rc = sysctl_wire_old_buffer(req, 0); 6366 if (rc != 0) 6367 return (rc); 6368 6369 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6370 if (sb == NULL) 6371 return (ENOMEM); 6372 6373 mtx_lock(&sc->regwin_lock); 6374 t4_tp_get_cpl_stats(sc, &stats); 6375 mtx_unlock(&sc->regwin_lock); 6376 6377 if (sc->chip_params->nchan > 2) { 6378 sbuf_printf(sb, " channel 0 channel 1" 6379 " channel 2 channel 3"); 6380 sbuf_printf(sb, "\nCPL requests: %10u %10u %10u %10u", 6381 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 6382 sbuf_printf(sb, "\nCPL responses: %10u %10u %10u %10u", 6383 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 6384 } else { 6385 sbuf_printf(sb, " channel 0 channel 1"); 6386 sbuf_printf(sb, "\nCPL requests: %10u %10u", 6387 stats.req[0], stats.req[1]); 6388 sbuf_printf(sb, "\nCPL responses: %10u %10u", 6389 stats.rsp[0], stats.rsp[1]); 6390 } 6391 6392 rc = sbuf_finish(sb); 6393 sbuf_delete(sb); 6394 6395 return (rc); 6396 } 6397 6398 static int 6399 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 6400 { 6401 struct adapter *sc = arg1; 6402 struct sbuf *sb; 6403 int rc; 6404 struct tp_usm_stats stats; 6405 6406 rc = sysctl_wire_old_buffer(req, 0); 6407 if (rc != 0) 6408 return(rc); 6409 6410 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6411 if (sb == NULL) 6412 return (ENOMEM); 6413 6414 t4_get_usm_stats(sc, &stats); 6415 6416 sbuf_printf(sb, "Frames: %u\n", stats.frames); 6417 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 6418 sbuf_printf(sb, "Drops: %u", stats.drops); 6419 6420 rc = sbuf_finish(sb); 6421 sbuf_delete(sb); 6422 6423 return (rc); 6424 } 6425 6426 const char *devlog_level_strings[] = { 6427 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 6428 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 6429 [FW_DEVLOG_LEVEL_ERR] = "ERR", 6430 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 6431 [FW_DEVLOG_LEVEL_INFO] = "INFO", 6432 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 6433 }; 6434 6435 const char *devlog_facility_strings[] = { 6436 [FW_DEVLOG_FACILITY_CORE] = "CORE", 6437 [FW_DEVLOG_FACILITY_CF] = "CF", 6438 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 6439 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 6440 [FW_DEVLOG_FACILITY_RES] = "RES", 6441 [FW_DEVLOG_FACILITY_HW] = "HW", 6442 [FW_DEVLOG_FACILITY_FLR] = "FLR", 6443 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 6444 [FW_DEVLOG_FACILITY_PHY] = "PHY", 6445 [FW_DEVLOG_FACILITY_MAC] = "MAC", 6446 [FW_DEVLOG_FACILITY_PORT] = "PORT", 6447 [FW_DEVLOG_FACILITY_VI] = "VI", 6448 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 6449 [FW_DEVLOG_FACILITY_ACL] = "ACL", 6450 [FW_DEVLOG_FACILITY_TM] = "TM", 6451 [FW_DEVLOG_FACILITY_QFC] = "QFC", 6452 [FW_DEVLOG_FACILITY_DCB] = "DCB", 6453 [FW_DEVLOG_FACILITY_ETH] = "ETH", 6454 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 6455 [FW_DEVLOG_FACILITY_RI] = "RI", 6456 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 6457 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 6458 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 6459 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 6460 }; 6461 6462 static int 6463 sysctl_devlog(SYSCTL_HANDLER_ARGS) 6464 { 6465 struct adapter *sc = arg1; 6466 struct devlog_params *dparams = &sc->params.devlog; 6467 struct fw_devlog_e *buf, *e; 6468 int i, j, rc, nentries, first = 0, m; 6469 struct sbuf *sb; 6470 uint64_t ftstamp = UINT64_MAX; 6471 6472 if (dparams->start == 0) { 6473 dparams->memtype = FW_MEMTYPE_EDC0; 6474 dparams->start = 0x84000; 6475 dparams->size = 32768; 6476 } 6477 6478 nentries = dparams->size / sizeof(struct fw_devlog_e); 6479 6480 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 6481 if (buf == NULL) 6482 return (ENOMEM); 6483 6484 m = fwmtype_to_hwmtype(dparams->memtype); 6485 rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf); 6486 if (rc != 0) 6487 goto done; 6488 6489 for (i = 0; i < nentries; i++) { 6490 e = &buf[i]; 6491 6492 if (e->timestamp == 0) 6493 break; /* end */ 6494 6495 e->timestamp = be64toh(e->timestamp); 6496 e->seqno = be32toh(e->seqno); 6497 for (j = 0; j < 8; j++) 6498 e->params[j] = be32toh(e->params[j]); 6499 6500 if (e->timestamp < ftstamp) { 6501 ftstamp = e->timestamp; 6502 first = i; 6503 } 6504 } 6505 6506 if (buf[first].timestamp == 0) 6507 goto done; /* nothing in the log */ 6508 6509 rc = sysctl_wire_old_buffer(req, 0); 6510 if (rc != 0) 6511 goto done; 6512 6513 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6514 if (sb == NULL) { 6515 rc = ENOMEM; 6516 goto done; 6517 } 6518 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 6519 "Seq#", "Tstamp", "Level", "Facility", "Message"); 6520 6521 i = first; 6522 do { 6523 e = &buf[i]; 6524 if (e->timestamp == 0) 6525 break; /* end */ 6526 6527 sbuf_printf(sb, "%10d %15ju %8s %8s ", 6528 e->seqno, e->timestamp, 6529 (e->level < nitems(devlog_level_strings) ? 6530 devlog_level_strings[e->level] : "UNKNOWN"), 6531 (e->facility < nitems(devlog_facility_strings) ? 6532 devlog_facility_strings[e->facility] : "UNKNOWN")); 6533 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 6534 e->params[2], e->params[3], e->params[4], 6535 e->params[5], e->params[6], e->params[7]); 6536 6537 if (++i == nentries) 6538 i = 0; 6539 } while (i != first); 6540 6541 rc = sbuf_finish(sb); 6542 sbuf_delete(sb); 6543 done: 6544 free(buf, M_CXGBE); 6545 return (rc); 6546 } 6547 6548 static int 6549 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 6550 { 6551 struct adapter *sc = arg1; 6552 struct sbuf *sb; 6553 int rc; 6554 struct tp_fcoe_stats stats[MAX_NCHAN]; 6555 int i, nchan = sc->chip_params->nchan; 6556 6557 rc = sysctl_wire_old_buffer(req, 0); 6558 if (rc != 0) 6559 return (rc); 6560 6561 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6562 if (sb == NULL) 6563 return (ENOMEM); 6564 6565 for (i = 0; i < nchan; i++) 6566 t4_get_fcoe_stats(sc, i, &stats[i]); 6567 6568 if (nchan > 2) { 6569 sbuf_printf(sb, " channel 0 channel 1" 6570 " channel 2 channel 3"); 6571 sbuf_printf(sb, "\noctetsDDP: %16ju %16ju %16ju %16ju", 6572 stats[0].octets_ddp, stats[1].octets_ddp, 6573 stats[2].octets_ddp, stats[3].octets_ddp); 6574 sbuf_printf(sb, "\nframesDDP: %16u %16u %16u %16u", 6575 stats[0].frames_ddp, stats[1].frames_ddp, 6576 stats[2].frames_ddp, stats[3].frames_ddp); 6577 sbuf_printf(sb, "\nframesDrop: %16u %16u %16u %16u", 6578 stats[0].frames_drop, stats[1].frames_drop, 6579 stats[2].frames_drop, stats[3].frames_drop); 6580 } else { 6581 sbuf_printf(sb, " channel 0 channel 1"); 6582 sbuf_printf(sb, "\noctetsDDP: %16ju %16ju", 6583 stats[0].octets_ddp, stats[1].octets_ddp); 6584 sbuf_printf(sb, "\nframesDDP: %16u %16u", 6585 stats[0].frames_ddp, stats[1].frames_ddp); 6586 sbuf_printf(sb, "\nframesDrop: %16u %16u", 6587 stats[0].frames_drop, stats[1].frames_drop); 6588 } 6589 6590 rc = sbuf_finish(sb); 6591 sbuf_delete(sb); 6592 6593 return (rc); 6594 } 6595 6596 static int 6597 sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 6598 { 6599 struct adapter *sc = arg1; 6600 struct sbuf *sb; 6601 int rc, i; 6602 unsigned int map, kbps, ipg, mode; 6603 unsigned int pace_tab[NTX_SCHED]; 6604 6605 rc = sysctl_wire_old_buffer(req, 0); 6606 if (rc != 0) 6607 return (rc); 6608 6609 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6610 if (sb == NULL) 6611 return (ENOMEM); 6612 6613 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 6614 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 6615 t4_read_pace_tbl(sc, pace_tab); 6616 6617 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 6618 "Class IPG (0.1 ns) Flow IPG (us)"); 6619 6620 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 6621 t4_get_tx_sched(sc, i, &kbps, &ipg); 6622 sbuf_printf(sb, "\n %u %-5s %u ", i, 6623 (mode & (1 << i)) ? "flow" : "class", map & 3); 6624 if (kbps) 6625 sbuf_printf(sb, "%9u ", kbps); 6626 else 6627 sbuf_printf(sb, " disabled "); 6628 6629 if (ipg) 6630 sbuf_printf(sb, "%13u ", ipg); 6631 else 6632 sbuf_printf(sb, " disabled "); 6633 6634 if (pace_tab[i]) 6635 sbuf_printf(sb, "%10u", pace_tab[i]); 6636 else 6637 sbuf_printf(sb, " disabled"); 6638 } 6639 6640 rc = sbuf_finish(sb); 6641 sbuf_delete(sb); 6642 6643 return (rc); 6644 } 6645 6646 static int 6647 sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 6648 { 6649 struct adapter *sc = arg1; 6650 struct sbuf *sb; 6651 int rc, i, j; 6652 uint64_t *p0, *p1; 6653 struct lb_port_stats s[2]; 6654 static const char *stat_name[] = { 6655 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 6656 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 6657 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 6658 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 6659 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 6660 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 6661 "BG2FramesTrunc:", "BG3FramesTrunc:" 6662 }; 6663 6664 rc = sysctl_wire_old_buffer(req, 0); 6665 if (rc != 0) 6666 return (rc); 6667 6668 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6669 if (sb == NULL) 6670 return (ENOMEM); 6671 6672 memset(s, 0, sizeof(s)); 6673 6674 for (i = 0; i < sc->chip_params->nchan; i += 2) { 6675 t4_get_lb_stats(sc, i, &s[0]); 6676 t4_get_lb_stats(sc, i + 1, &s[1]); 6677 6678 p0 = &s[0].octets; 6679 p1 = &s[1].octets; 6680 sbuf_printf(sb, "%s Loopback %u" 6681 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 6682 6683 for (j = 0; j < nitems(stat_name); j++) 6684 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 6685 *p0++, *p1++); 6686 } 6687 6688 rc = sbuf_finish(sb); 6689 sbuf_delete(sb); 6690 6691 return (rc); 6692 } 6693 6694 static int 6695 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) 6696 { 6697 int rc = 0; 6698 struct port_info *pi = arg1; 6699 struct sbuf *sb; 6700 static const char *linkdnreasons[] = { 6701 "non-specific", "remote fault", "autoneg failed", "reserved3", 6702 "PHY overheated", "unknown", "rx los", "reserved7" 6703 }; 6704 6705 rc = sysctl_wire_old_buffer(req, 0); 6706 if (rc != 0) 6707 return(rc); 6708 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); 6709 if (sb == NULL) 6710 return (ENOMEM); 6711 6712 if (pi->linkdnrc < 0) 6713 sbuf_printf(sb, "n/a"); 6714 else if (pi->linkdnrc < nitems(linkdnreasons)) 6715 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]); 6716 else 6717 sbuf_printf(sb, "%d", pi->linkdnrc); 6718 6719 rc = sbuf_finish(sb); 6720 sbuf_delete(sb); 6721 6722 return (rc); 6723 } 6724 6725 struct mem_desc { 6726 unsigned int base; 6727 unsigned int limit; 6728 unsigned int idx; 6729 }; 6730 6731 static int 6732 mem_desc_cmp(const void *a, const void *b) 6733 { 6734 return ((const struct mem_desc *)a)->base - 6735 ((const struct mem_desc *)b)->base; 6736 } 6737 6738 static void 6739 mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 6740 unsigned int to) 6741 { 6742 unsigned int size; 6743 6744 size = to - from + 1; 6745 if (size == 0) 6746 return; 6747 6748 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 6749 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 6750 } 6751 6752 static int 6753 sysctl_meminfo(SYSCTL_HANDLER_ARGS) 6754 { 6755 struct adapter *sc = arg1; 6756 struct sbuf *sb; 6757 int rc, i, n; 6758 uint32_t lo, hi, used, alloc; 6759 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 6760 static const char *region[] = { 6761 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 6762 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 6763 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 6764 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 6765 "RQUDP region:", "PBL region:", "TXPBL region:", 6766 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 6767 "On-chip queues:" 6768 }; 6769 struct mem_desc avail[4]; 6770 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 6771 struct mem_desc *md = mem; 6772 6773 rc = sysctl_wire_old_buffer(req, 0); 6774 if (rc != 0) 6775 return (rc); 6776 6777 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6778 if (sb == NULL) 6779 return (ENOMEM); 6780 6781 for (i = 0; i < nitems(mem); i++) { 6782 mem[i].limit = 0; 6783 mem[i].idx = i; 6784 } 6785 6786 /* Find and sort the populated memory ranges */ 6787 i = 0; 6788 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 6789 if (lo & F_EDRAM0_ENABLE) { 6790 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 6791 avail[i].base = G_EDRAM0_BASE(hi) << 20; 6792 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 6793 avail[i].idx = 0; 6794 i++; 6795 } 6796 if (lo & F_EDRAM1_ENABLE) { 6797 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 6798 avail[i].base = G_EDRAM1_BASE(hi) << 20; 6799 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 6800 avail[i].idx = 1; 6801 i++; 6802 } 6803 if (lo & F_EXT_MEM_ENABLE) { 6804 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 6805 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 6806 avail[i].limit = avail[i].base + 6807 (G_EXT_MEM_SIZE(hi) << 20); 6808 avail[i].idx = is_t5(sc) ? 3 : 2; /* Call it MC0 for T5 */ 6809 i++; 6810 } 6811 if (is_t5(sc) && lo & F_EXT_MEM1_ENABLE) { 6812 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 6813 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 6814 avail[i].limit = avail[i].base + 6815 (G_EXT_MEM1_SIZE(hi) << 20); 6816 avail[i].idx = 4; 6817 i++; 6818 } 6819 if (!i) /* no memory available */ 6820 return 0; 6821 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 6822 6823 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 6824 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 6825 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 6826 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 6827 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 6828 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 6829 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 6830 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 6831 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 6832 6833 /* the next few have explicit upper bounds */ 6834 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 6835 md->limit = md->base - 1 + 6836 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 6837 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 6838 md++; 6839 6840 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 6841 md->limit = md->base - 1 + 6842 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 6843 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 6844 md++; 6845 6846 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6847 if (chip_id(sc) <= CHELSIO_T5) { 6848 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 6849 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 6850 } else { 6851 hi = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 6852 md->base = t4_read_reg(sc, A_LE_DB_HASH_TBL_BASE_ADDR); 6853 } 6854 md->limit = 0; 6855 } else { 6856 md->base = 0; 6857 md->idx = nitems(region); /* hide it */ 6858 } 6859 md++; 6860 6861 #define ulp_region(reg) \ 6862 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 6863 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 6864 6865 ulp_region(RX_ISCSI); 6866 ulp_region(RX_TDDP); 6867 ulp_region(TX_TPT); 6868 ulp_region(RX_STAG); 6869 ulp_region(RX_RQ); 6870 ulp_region(RX_RQUDP); 6871 ulp_region(RX_PBL); 6872 ulp_region(TX_PBL); 6873 #undef ulp_region 6874 6875 md->base = 0; 6876 md->idx = nitems(region); 6877 if (!is_t4(sc)) { 6878 uint32_t size = 0; 6879 uint32_t sge_ctrl = t4_read_reg(sc, A_SGE_CONTROL2); 6880 uint32_t fifo_size = t4_read_reg(sc, A_SGE_DBVFIFO_SIZE); 6881 6882 if (is_t5(sc)) { 6883 if (sge_ctrl & F_VFIFO_ENABLE) 6884 size = G_DBVFIFO_SIZE(fifo_size); 6885 } else 6886 size = G_T6_DBVFIFO_SIZE(fifo_size); 6887 6888 if (size) { 6889 md->base = G_BASEADDR(t4_read_reg(sc, 6890 A_SGE_DBVFIFO_BADDR)); 6891 md->limit = md->base + (size << 2) - 1; 6892 } 6893 } 6894 md++; 6895 6896 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 6897 md->limit = 0; 6898 md++; 6899 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 6900 md->limit = 0; 6901 md++; 6902 6903 md->base = sc->vres.ocq.start; 6904 if (sc->vres.ocq.size) 6905 md->limit = md->base + sc->vres.ocq.size - 1; 6906 else 6907 md->idx = nitems(region); /* hide it */ 6908 md++; 6909 6910 /* add any address-space holes, there can be up to 3 */ 6911 for (n = 0; n < i - 1; n++) 6912 if (avail[n].limit < avail[n + 1].base) 6913 (md++)->base = avail[n].limit; 6914 if (avail[n].limit) 6915 (md++)->base = avail[n].limit; 6916 6917 n = md - mem; 6918 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 6919 6920 for (lo = 0; lo < i; lo++) 6921 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 6922 avail[lo].limit - 1); 6923 6924 sbuf_printf(sb, "\n"); 6925 for (i = 0; i < n; i++) { 6926 if (mem[i].idx >= nitems(region)) 6927 continue; /* skip holes */ 6928 if (!mem[i].limit) 6929 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 6930 mem_region_show(sb, region[mem[i].idx], mem[i].base, 6931 mem[i].limit); 6932 } 6933 6934 sbuf_printf(sb, "\n"); 6935 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 6936 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 6937 mem_region_show(sb, "uP RAM:", lo, hi); 6938 6939 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 6940 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 6941 mem_region_show(sb, "uP Extmem2:", lo, hi); 6942 6943 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 6944 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 6945 G_PMRXMAXPAGE(lo), 6946 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 6947 (lo & F_PMRXNUMCHN) ? 2 : 1); 6948 6949 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 6950 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 6951 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 6952 G_PMTXMAXPAGE(lo), 6953 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 6954 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 6955 sbuf_printf(sb, "%u p-structs\n", 6956 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 6957 6958 for (i = 0; i < 4; i++) { 6959 if (chip_id(sc) > CHELSIO_T5) 6960 lo = t4_read_reg(sc, A_MPS_RX_MAC_BG_PG_CNT0 + i * 4); 6961 else 6962 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 6963 if (is_t5(sc)) { 6964 used = G_T5_USED(lo); 6965 alloc = G_T5_ALLOC(lo); 6966 } else { 6967 used = G_USED(lo); 6968 alloc = G_ALLOC(lo); 6969 } 6970 /* For T6 these are MAC buffer groups */ 6971 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 6972 i, used, alloc); 6973 } 6974 for (i = 0; i < sc->chip_params->nchan; i++) { 6975 if (chip_id(sc) > CHELSIO_T5) 6976 lo = t4_read_reg(sc, A_MPS_RX_LPBK_BG_PG_CNT0 + i * 4); 6977 else 6978 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 6979 if (is_t5(sc)) { 6980 used = G_T5_USED(lo); 6981 alloc = G_T5_ALLOC(lo); 6982 } else { 6983 used = G_USED(lo); 6984 alloc = G_ALLOC(lo); 6985 } 6986 /* For T6 these are MAC buffer groups */ 6987 sbuf_printf(sb, 6988 "\nLoopback %d using %u pages out of %u allocated", 6989 i, used, alloc); 6990 } 6991 6992 rc = sbuf_finish(sb); 6993 sbuf_delete(sb); 6994 6995 return (rc); 6996 } 6997 6998 static inline void 6999 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) 7000 { 7001 *mask = x | y; 7002 y = htobe64(y); 7003 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); 7004 } 7005 7006 static int 7007 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) 7008 { 7009 struct adapter *sc = arg1; 7010 struct sbuf *sb; 7011 int rc, i; 7012 7013 MPASS(chip_id(sc) <= CHELSIO_T5); 7014 7015 rc = sysctl_wire_old_buffer(req, 0); 7016 if (rc != 0) 7017 return (rc); 7018 7019 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7020 if (sb == NULL) 7021 return (ENOMEM); 7022 7023 sbuf_printf(sb, 7024 "Idx Ethernet address Mask Vld Ports PF" 7025 " VF Replication P0 P1 P2 P3 ML"); 7026 for (i = 0; i < sc->chip_params->mps_tcam_size; i++) { 7027 uint64_t tcamx, tcamy, mask; 7028 uint32_t cls_lo, cls_hi; 7029 uint8_t addr[ETHER_ADDR_LEN]; 7030 7031 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); 7032 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); 7033 if (tcamx & tcamy) 7034 continue; 7035 tcamxy2valmask(tcamx, tcamy, addr, &mask); 7036 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 7037 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 7038 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" 7039 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], 7040 addr[3], addr[4], addr[5], (uintmax_t)mask, 7041 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', 7042 G_PORTMAP(cls_hi), G_PF(cls_lo), 7043 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); 7044 7045 if (cls_lo & F_REPLICATE) { 7046 struct fw_ldst_cmd ldst_cmd; 7047 7048 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 7049 ldst_cmd.op_to_addrspace = 7050 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 7051 F_FW_CMD_REQUEST | F_FW_CMD_READ | 7052 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 7053 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 7054 ldst_cmd.u.mps.rplc.fid_idx = 7055 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 7056 V_FW_LDST_CMD_IDX(i)); 7057 7058 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 7059 "t4mps"); 7060 if (rc) 7061 break; 7062 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 7063 sizeof(ldst_cmd), &ldst_cmd); 7064 end_synchronized_op(sc, 0); 7065 7066 if (rc != 0) { 7067 sbuf_printf(sb, "%36d", rc); 7068 rc = 0; 7069 } else { 7070 sbuf_printf(sb, " %08x %08x %08x %08x", 7071 be32toh(ldst_cmd.u.mps.rplc.rplc127_96), 7072 be32toh(ldst_cmd.u.mps.rplc.rplc95_64), 7073 be32toh(ldst_cmd.u.mps.rplc.rplc63_32), 7074 be32toh(ldst_cmd.u.mps.rplc.rplc31_0)); 7075 } 7076 } else 7077 sbuf_printf(sb, "%36s", ""); 7078 7079 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), 7080 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), 7081 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); 7082 } 7083 7084 if (rc) 7085 (void) sbuf_finish(sb); 7086 else 7087 rc = sbuf_finish(sb); 7088 sbuf_delete(sb); 7089 7090 return (rc); 7091 } 7092 7093 static int 7094 sysctl_mps_tcam_t6(SYSCTL_HANDLER_ARGS) 7095 { 7096 struct adapter *sc = arg1; 7097 struct sbuf *sb; 7098 int rc, i; 7099 7100 MPASS(chip_id(sc) > CHELSIO_T5); 7101 7102 rc = sysctl_wire_old_buffer(req, 0); 7103 if (rc != 0) 7104 return (rc); 7105 7106 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7107 if (sb == NULL) 7108 return (ENOMEM); 7109 7110 sbuf_printf(sb, "Idx Ethernet address Mask VNI Mask" 7111 " IVLAN Vld DIP_Hit Lookup Port Vld Ports PF VF" 7112 " Replication" 7113 " P0 P1 P2 P3 ML\n"); 7114 7115 for (i = 0; i < sc->chip_params->mps_tcam_size; i++) { 7116 uint8_t dip_hit, vlan_vld, lookup_type, port_num; 7117 uint16_t ivlan; 7118 uint64_t tcamx, tcamy, val, mask; 7119 uint32_t cls_lo, cls_hi, ctl, data2, vnix, vniy; 7120 uint8_t addr[ETHER_ADDR_LEN]; 7121 7122 ctl = V_CTLREQID(1) | V_CTLCMDTYPE(0) | V_CTLXYBITSEL(0); 7123 if (i < 256) 7124 ctl |= V_CTLTCAMINDEX(i) | V_CTLTCAMSEL(0); 7125 else 7126 ctl |= V_CTLTCAMINDEX(i - 256) | V_CTLTCAMSEL(1); 7127 t4_write_reg(sc, A_MPS_CLS_TCAM_DATA2_CTL, ctl); 7128 val = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA1_REQ_ID1); 7129 tcamy = G_DMACH(val) << 32; 7130 tcamy |= t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA0_REQ_ID1); 7131 data2 = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA2_REQ_ID1); 7132 lookup_type = G_DATALKPTYPE(data2); 7133 port_num = G_DATAPORTNUM(data2); 7134 if (lookup_type && lookup_type != M_DATALKPTYPE) { 7135 /* Inner header VNI */ 7136 vniy = ((data2 & F_DATAVIDH2) << 23) | 7137 (G_DATAVIDH1(data2) << 16) | G_VIDL(val); 7138 dip_hit = data2 & F_DATADIPHIT; 7139 vlan_vld = 0; 7140 } else { 7141 vniy = 0; 7142 dip_hit = 0; 7143 vlan_vld = data2 & F_DATAVIDH2; 7144 ivlan = G_VIDL(val); 7145 } 7146 7147 ctl |= V_CTLXYBITSEL(1); 7148 t4_write_reg(sc, A_MPS_CLS_TCAM_DATA2_CTL, ctl); 7149 val = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA1_REQ_ID1); 7150 tcamx = G_DMACH(val) << 32; 7151 tcamx |= t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA0_REQ_ID1); 7152 data2 = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA2_REQ_ID1); 7153 if (lookup_type && lookup_type != M_DATALKPTYPE) { 7154 /* Inner header VNI mask */ 7155 vnix = ((data2 & F_DATAVIDH2) << 23) | 7156 (G_DATAVIDH1(data2) << 16) | G_VIDL(val); 7157 } else 7158 vnix = 0; 7159 7160 if (tcamx & tcamy) 7161 continue; 7162 tcamxy2valmask(tcamx, tcamy, addr, &mask); 7163 7164 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 7165 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 7166 7167 if (lookup_type && lookup_type != M_DATALKPTYPE) { 7168 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x " 7169 "%012jx %06x %06x - - %3c" 7170 " 'I' %4x %3c %#x%4u%4d", i, addr[0], 7171 addr[1], addr[2], addr[3], addr[4], addr[5], 7172 (uintmax_t)mask, vniy, vnix, dip_hit ? 'Y' : 'N', 7173 port_num, cls_lo & F_T6_SRAM_VLD ? 'Y' : 'N', 7174 G_PORTMAP(cls_hi), G_T6_PF(cls_lo), 7175 cls_lo & F_T6_VF_VALID ? G_T6_VF(cls_lo) : -1); 7176 } else { 7177 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x " 7178 "%012jx - - ", i, addr[0], addr[1], 7179 addr[2], addr[3], addr[4], addr[5], 7180 (uintmax_t)mask); 7181 7182 if (vlan_vld) 7183 sbuf_printf(sb, "%4u Y ", ivlan); 7184 else 7185 sbuf_printf(sb, " - N "); 7186 7187 sbuf_printf(sb, "- %3c %4x %3c %#x%4u%4d", 7188 lookup_type ? 'I' : 'O', port_num, 7189 cls_lo & F_T6_SRAM_VLD ? 'Y' : 'N', 7190 G_PORTMAP(cls_hi), G_T6_PF(cls_lo), 7191 cls_lo & F_T6_VF_VALID ? G_T6_VF(cls_lo) : -1); 7192 } 7193 7194 7195 if (cls_lo & F_T6_REPLICATE) { 7196 struct fw_ldst_cmd ldst_cmd; 7197 7198 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 7199 ldst_cmd.op_to_addrspace = 7200 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 7201 F_FW_CMD_REQUEST | F_FW_CMD_READ | 7202 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 7203 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 7204 ldst_cmd.u.mps.rplc.fid_idx = 7205 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 7206 V_FW_LDST_CMD_IDX(i)); 7207 7208 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 7209 "t6mps"); 7210 if (rc) 7211 break; 7212 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 7213 sizeof(ldst_cmd), &ldst_cmd); 7214 end_synchronized_op(sc, 0); 7215 7216 if (rc != 0) { 7217 sbuf_printf(sb, "%72d", rc); 7218 rc = 0; 7219 } else { 7220 sbuf_printf(sb, " %08x %08x %08x %08x" 7221 " %08x %08x %08x %08x", 7222 be32toh(ldst_cmd.u.mps.rplc.rplc255_224), 7223 be32toh(ldst_cmd.u.mps.rplc.rplc223_192), 7224 be32toh(ldst_cmd.u.mps.rplc.rplc191_160), 7225 be32toh(ldst_cmd.u.mps.rplc.rplc159_128), 7226 be32toh(ldst_cmd.u.mps.rplc.rplc127_96), 7227 be32toh(ldst_cmd.u.mps.rplc.rplc95_64), 7228 be32toh(ldst_cmd.u.mps.rplc.rplc63_32), 7229 be32toh(ldst_cmd.u.mps.rplc.rplc31_0)); 7230 } 7231 } else 7232 sbuf_printf(sb, "%72s", ""); 7233 7234 sbuf_printf(sb, "%4u%3u%3u%3u %#x", 7235 G_T6_SRAM_PRIO0(cls_lo), G_T6_SRAM_PRIO1(cls_lo), 7236 G_T6_SRAM_PRIO2(cls_lo), G_T6_SRAM_PRIO3(cls_lo), 7237 (cls_lo >> S_T6_MULTILISTEN0) & 0xf); 7238 } 7239 7240 if (rc) 7241 (void) sbuf_finish(sb); 7242 else 7243 rc = sbuf_finish(sb); 7244 sbuf_delete(sb); 7245 7246 return (rc); 7247 } 7248 7249 static int 7250 sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 7251 { 7252 struct adapter *sc = arg1; 7253 struct sbuf *sb; 7254 int rc; 7255 uint16_t mtus[NMTUS]; 7256 7257 rc = sysctl_wire_old_buffer(req, 0); 7258 if (rc != 0) 7259 return (rc); 7260 7261 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7262 if (sb == NULL) 7263 return (ENOMEM); 7264 7265 t4_read_mtu_tbl(sc, mtus, NULL); 7266 7267 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 7268 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 7269 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 7270 mtus[14], mtus[15]); 7271 7272 rc = sbuf_finish(sb); 7273 sbuf_delete(sb); 7274 7275 return (rc); 7276 } 7277 7278 static int 7279 sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 7280 { 7281 struct adapter *sc = arg1; 7282 struct sbuf *sb; 7283 int rc, i; 7284 uint32_t tx_cnt[MAX_PM_NSTATS], rx_cnt[MAX_PM_NSTATS]; 7285 uint64_t tx_cyc[MAX_PM_NSTATS], rx_cyc[MAX_PM_NSTATS]; 7286 static const char *tx_stats[MAX_PM_NSTATS] = { 7287 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:", 7288 "Tx FIFO wait", NULL, "Tx latency" 7289 }; 7290 static const char *rx_stats[MAX_PM_NSTATS] = { 7291 "Read:", "Write bypass:", "Write mem:", "Flush:", 7292 " Rx FIFO wait", NULL, "Rx latency" 7293 }; 7294 7295 rc = sysctl_wire_old_buffer(req, 0); 7296 if (rc != 0) 7297 return (rc); 7298 7299 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7300 if (sb == NULL) 7301 return (ENOMEM); 7302 7303 t4_pmtx_get_stats(sc, tx_cnt, tx_cyc); 7304 t4_pmrx_get_stats(sc, rx_cnt, rx_cyc); 7305 7306 sbuf_printf(sb, " Tx pcmds Tx bytes"); 7307 for (i = 0; i < 4; i++) { 7308 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i], 7309 tx_cyc[i]); 7310 } 7311 7312 sbuf_printf(sb, "\n Rx pcmds Rx bytes"); 7313 for (i = 0; i < 4; i++) { 7314 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i], 7315 rx_cyc[i]); 7316 } 7317 7318 if (chip_id(sc) > CHELSIO_T5) { 7319 sbuf_printf(sb, 7320 "\n Total wait Total occupancy"); 7321 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i], 7322 tx_cyc[i]); 7323 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i], 7324 rx_cyc[i]); 7325 7326 i += 2; 7327 MPASS(i < nitems(tx_stats)); 7328 7329 sbuf_printf(sb, 7330 "\n Reads Total wait"); 7331 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i], 7332 tx_cyc[i]); 7333 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i], 7334 rx_cyc[i]); 7335 } 7336 7337 rc = sbuf_finish(sb); 7338 sbuf_delete(sb); 7339 7340 return (rc); 7341 } 7342 7343 static int 7344 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 7345 { 7346 struct adapter *sc = arg1; 7347 struct sbuf *sb; 7348 int rc; 7349 struct tp_rdma_stats stats; 7350 7351 rc = sysctl_wire_old_buffer(req, 0); 7352 if (rc != 0) 7353 return (rc); 7354 7355 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7356 if (sb == NULL) 7357 return (ENOMEM); 7358 7359 mtx_lock(&sc->regwin_lock); 7360 t4_tp_get_rdma_stats(sc, &stats); 7361 mtx_unlock(&sc->regwin_lock); 7362 7363 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 7364 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 7365 7366 rc = sbuf_finish(sb); 7367 sbuf_delete(sb); 7368 7369 return (rc); 7370 } 7371 7372 static int 7373 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 7374 { 7375 struct adapter *sc = arg1; 7376 struct sbuf *sb; 7377 int rc; 7378 struct tp_tcp_stats v4, v6; 7379 7380 rc = sysctl_wire_old_buffer(req, 0); 7381 if (rc != 0) 7382 return (rc); 7383 7384 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7385 if (sb == NULL) 7386 return (ENOMEM); 7387 7388 mtx_lock(&sc->regwin_lock); 7389 t4_tp_get_tcp_stats(sc, &v4, &v6); 7390 mtx_unlock(&sc->regwin_lock); 7391 7392 sbuf_printf(sb, 7393 " IP IPv6\n"); 7394 sbuf_printf(sb, "OutRsts: %20u %20u\n", 7395 v4.tcp_out_rsts, v6.tcp_out_rsts); 7396 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 7397 v4.tcp_in_segs, v6.tcp_in_segs); 7398 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 7399 v4.tcp_out_segs, v6.tcp_out_segs); 7400 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 7401 v4.tcp_retrans_segs, v6.tcp_retrans_segs); 7402 7403 rc = sbuf_finish(sb); 7404 sbuf_delete(sb); 7405 7406 return (rc); 7407 } 7408 7409 static int 7410 sysctl_tids(SYSCTL_HANDLER_ARGS) 7411 { 7412 struct adapter *sc = arg1; 7413 struct sbuf *sb; 7414 int rc; 7415 struct tid_info *t = &sc->tids; 7416 7417 rc = sysctl_wire_old_buffer(req, 0); 7418 if (rc != 0) 7419 return (rc); 7420 7421 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7422 if (sb == NULL) 7423 return (ENOMEM); 7424 7425 if (t->natids) { 7426 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 7427 t->atids_in_use); 7428 } 7429 7430 if (t->ntids) { 7431 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 7432 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 7433 7434 if (b) { 7435 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 7436 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 7437 t->ntids - 1); 7438 } else { 7439 sbuf_printf(sb, "TID range: %u-%u", 7440 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 7441 t->ntids - 1); 7442 } 7443 } else 7444 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 7445 sbuf_printf(sb, ", in use: %u\n", 7446 atomic_load_acq_int(&t->tids_in_use)); 7447 } 7448 7449 if (t->nstids) { 7450 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 7451 t->stid_base + t->nstids - 1, t->stids_in_use); 7452 } 7453 7454 if (t->nftids) { 7455 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 7456 t->ftid_base + t->nftids - 1); 7457 } 7458 7459 if (t->netids) { 7460 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base, 7461 t->etid_base + t->netids - 1); 7462 } 7463 7464 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 7465 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 7466 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 7467 7468 rc = sbuf_finish(sb); 7469 sbuf_delete(sb); 7470 7471 return (rc); 7472 } 7473 7474 static int 7475 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 7476 { 7477 struct adapter *sc = arg1; 7478 struct sbuf *sb; 7479 int rc; 7480 struct tp_err_stats stats; 7481 7482 rc = sysctl_wire_old_buffer(req, 0); 7483 if (rc != 0) 7484 return (rc); 7485 7486 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7487 if (sb == NULL) 7488 return (ENOMEM); 7489 7490 mtx_lock(&sc->regwin_lock); 7491 t4_tp_get_err_stats(sc, &stats); 7492 mtx_unlock(&sc->regwin_lock); 7493 7494 if (sc->chip_params->nchan > 2) { 7495 sbuf_printf(sb, " channel 0 channel 1" 7496 " channel 2 channel 3\n"); 7497 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 7498 stats.mac_in_errs[0], stats.mac_in_errs[1], 7499 stats.mac_in_errs[2], stats.mac_in_errs[3]); 7500 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 7501 stats.hdr_in_errs[0], stats.hdr_in_errs[1], 7502 stats.hdr_in_errs[2], stats.hdr_in_errs[3]); 7503 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 7504 stats.tcp_in_errs[0], stats.tcp_in_errs[1], 7505 stats.tcp_in_errs[2], stats.tcp_in_errs[3]); 7506 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 7507 stats.tcp6_in_errs[0], stats.tcp6_in_errs[1], 7508 stats.tcp6_in_errs[2], stats.tcp6_in_errs[3]); 7509 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 7510 stats.tnl_cong_drops[0], stats.tnl_cong_drops[1], 7511 stats.tnl_cong_drops[2], stats.tnl_cong_drops[3]); 7512 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 7513 stats.tnl_tx_drops[0], stats.tnl_tx_drops[1], 7514 stats.tnl_tx_drops[2], stats.tnl_tx_drops[3]); 7515 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 7516 stats.ofld_vlan_drops[0], stats.ofld_vlan_drops[1], 7517 stats.ofld_vlan_drops[2], stats.ofld_vlan_drops[3]); 7518 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 7519 stats.ofld_chan_drops[0], stats.ofld_chan_drops[1], 7520 stats.ofld_chan_drops[2], stats.ofld_chan_drops[3]); 7521 } else { 7522 sbuf_printf(sb, " channel 0 channel 1\n"); 7523 sbuf_printf(sb, "macInErrs: %10u %10u\n", 7524 stats.mac_in_errs[0], stats.mac_in_errs[1]); 7525 sbuf_printf(sb, "hdrInErrs: %10u %10u\n", 7526 stats.hdr_in_errs[0], stats.hdr_in_errs[1]); 7527 sbuf_printf(sb, "tcpInErrs: %10u %10u\n", 7528 stats.tcp_in_errs[0], stats.tcp_in_errs[1]); 7529 sbuf_printf(sb, "tcp6InErrs: %10u %10u\n", 7530 stats.tcp6_in_errs[0], stats.tcp6_in_errs[1]); 7531 sbuf_printf(sb, "tnlCongDrops: %10u %10u\n", 7532 stats.tnl_cong_drops[0], stats.tnl_cong_drops[1]); 7533 sbuf_printf(sb, "tnlTxDrops: %10u %10u\n", 7534 stats.tnl_tx_drops[0], stats.tnl_tx_drops[1]); 7535 sbuf_printf(sb, "ofldVlanDrops: %10u %10u\n", 7536 stats.ofld_vlan_drops[0], stats.ofld_vlan_drops[1]); 7537 sbuf_printf(sb, "ofldChanDrops: %10u %10u\n\n", 7538 stats.ofld_chan_drops[0], stats.ofld_chan_drops[1]); 7539 } 7540 7541 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 7542 stats.ofld_no_neigh, stats.ofld_cong_defer); 7543 7544 rc = sbuf_finish(sb); 7545 sbuf_delete(sb); 7546 7547 return (rc); 7548 } 7549 7550 struct field_desc { 7551 const char *name; 7552 u_int start; 7553 u_int width; 7554 }; 7555 7556 static void 7557 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) 7558 { 7559 char buf[32]; 7560 int line_size = 0; 7561 7562 while (f->name) { 7563 uint64_t mask = (1ULL << f->width) - 1; 7564 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, 7565 ((uintmax_t)v >> f->start) & mask); 7566 7567 if (line_size + len >= 79) { 7568 line_size = 8; 7569 sbuf_printf(sb, "\n "); 7570 } 7571 sbuf_printf(sb, "%s ", buf); 7572 line_size += len + 1; 7573 f++; 7574 } 7575 sbuf_printf(sb, "\n"); 7576 } 7577 7578 static const struct field_desc tp_la0[] = { 7579 { "RcfOpCodeOut", 60, 4 }, 7580 { "State", 56, 4 }, 7581 { "WcfState", 52, 4 }, 7582 { "RcfOpcSrcOut", 50, 2 }, 7583 { "CRxError", 49, 1 }, 7584 { "ERxError", 48, 1 }, 7585 { "SanityFailed", 47, 1 }, 7586 { "SpuriousMsg", 46, 1 }, 7587 { "FlushInputMsg", 45, 1 }, 7588 { "FlushInputCpl", 44, 1 }, 7589 { "RssUpBit", 43, 1 }, 7590 { "RssFilterHit", 42, 1 }, 7591 { "Tid", 32, 10 }, 7592 { "InitTcb", 31, 1 }, 7593 { "LineNumber", 24, 7 }, 7594 { "Emsg", 23, 1 }, 7595 { "EdataOut", 22, 1 }, 7596 { "Cmsg", 21, 1 }, 7597 { "CdataOut", 20, 1 }, 7598 { "EreadPdu", 19, 1 }, 7599 { "CreadPdu", 18, 1 }, 7600 { "TunnelPkt", 17, 1 }, 7601 { "RcfPeerFin", 16, 1 }, 7602 { "RcfReasonOut", 12, 4 }, 7603 { "TxCchannel", 10, 2 }, 7604 { "RcfTxChannel", 8, 2 }, 7605 { "RxEchannel", 6, 2 }, 7606 { "RcfRxChannel", 5, 1 }, 7607 { "RcfDataOutSrdy", 4, 1 }, 7608 { "RxDvld", 3, 1 }, 7609 { "RxOoDvld", 2, 1 }, 7610 { "RxCongestion", 1, 1 }, 7611 { "TxCongestion", 0, 1 }, 7612 { NULL } 7613 }; 7614 7615 static const struct field_desc tp_la1[] = { 7616 { "CplCmdIn", 56, 8 }, 7617 { "CplCmdOut", 48, 8 }, 7618 { "ESynOut", 47, 1 }, 7619 { "EAckOut", 46, 1 }, 7620 { "EFinOut", 45, 1 }, 7621 { "ERstOut", 44, 1 }, 7622 { "SynIn", 43, 1 }, 7623 { "AckIn", 42, 1 }, 7624 { "FinIn", 41, 1 }, 7625 { "RstIn", 40, 1 }, 7626 { "DataIn", 39, 1 }, 7627 { "DataInVld", 38, 1 }, 7628 { "PadIn", 37, 1 }, 7629 { "RxBufEmpty", 36, 1 }, 7630 { "RxDdp", 35, 1 }, 7631 { "RxFbCongestion", 34, 1 }, 7632 { "TxFbCongestion", 33, 1 }, 7633 { "TxPktSumSrdy", 32, 1 }, 7634 { "RcfUlpType", 28, 4 }, 7635 { "Eread", 27, 1 }, 7636 { "Ebypass", 26, 1 }, 7637 { "Esave", 25, 1 }, 7638 { "Static0", 24, 1 }, 7639 { "Cread", 23, 1 }, 7640 { "Cbypass", 22, 1 }, 7641 { "Csave", 21, 1 }, 7642 { "CPktOut", 20, 1 }, 7643 { "RxPagePoolFull", 18, 2 }, 7644 { "RxLpbkPkt", 17, 1 }, 7645 { "TxLpbkPkt", 16, 1 }, 7646 { "RxVfValid", 15, 1 }, 7647 { "SynLearned", 14, 1 }, 7648 { "SetDelEntry", 13, 1 }, 7649 { "SetInvEntry", 12, 1 }, 7650 { "CpcmdDvld", 11, 1 }, 7651 { "CpcmdSave", 10, 1 }, 7652 { "RxPstructsFull", 8, 2 }, 7653 { "EpcmdDvld", 7, 1 }, 7654 { "EpcmdFlush", 6, 1 }, 7655 { "EpcmdTrimPrefix", 5, 1 }, 7656 { "EpcmdTrimPostfix", 4, 1 }, 7657 { "ERssIp4Pkt", 3, 1 }, 7658 { "ERssIp6Pkt", 2, 1 }, 7659 { "ERssTcpUdpPkt", 1, 1 }, 7660 { "ERssFceFipPkt", 0, 1 }, 7661 { NULL } 7662 }; 7663 7664 static const struct field_desc tp_la2[] = { 7665 { "CplCmdIn", 56, 8 }, 7666 { "MpsVfVld", 55, 1 }, 7667 { "MpsPf", 52, 3 }, 7668 { "MpsVf", 44, 8 }, 7669 { "SynIn", 43, 1 }, 7670 { "AckIn", 42, 1 }, 7671 { "FinIn", 41, 1 }, 7672 { "RstIn", 40, 1 }, 7673 { "DataIn", 39, 1 }, 7674 { "DataInVld", 38, 1 }, 7675 { "PadIn", 37, 1 }, 7676 { "RxBufEmpty", 36, 1 }, 7677 { "RxDdp", 35, 1 }, 7678 { "RxFbCongestion", 34, 1 }, 7679 { "TxFbCongestion", 33, 1 }, 7680 { "TxPktSumSrdy", 32, 1 }, 7681 { "RcfUlpType", 28, 4 }, 7682 { "Eread", 27, 1 }, 7683 { "Ebypass", 26, 1 }, 7684 { "Esave", 25, 1 }, 7685 { "Static0", 24, 1 }, 7686 { "Cread", 23, 1 }, 7687 { "Cbypass", 22, 1 }, 7688 { "Csave", 21, 1 }, 7689 { "CPktOut", 20, 1 }, 7690 { "RxPagePoolFull", 18, 2 }, 7691 { "RxLpbkPkt", 17, 1 }, 7692 { "TxLpbkPkt", 16, 1 }, 7693 { "RxVfValid", 15, 1 }, 7694 { "SynLearned", 14, 1 }, 7695 { "SetDelEntry", 13, 1 }, 7696 { "SetInvEntry", 12, 1 }, 7697 { "CpcmdDvld", 11, 1 }, 7698 { "CpcmdSave", 10, 1 }, 7699 { "RxPstructsFull", 8, 2 }, 7700 { "EpcmdDvld", 7, 1 }, 7701 { "EpcmdFlush", 6, 1 }, 7702 { "EpcmdTrimPrefix", 5, 1 }, 7703 { "EpcmdTrimPostfix", 4, 1 }, 7704 { "ERssIp4Pkt", 3, 1 }, 7705 { "ERssIp6Pkt", 2, 1 }, 7706 { "ERssTcpUdpPkt", 1, 1 }, 7707 { "ERssFceFipPkt", 0, 1 }, 7708 { NULL } 7709 }; 7710 7711 static void 7712 tp_la_show(struct sbuf *sb, uint64_t *p, int idx) 7713 { 7714 7715 field_desc_show(sb, *p, tp_la0); 7716 } 7717 7718 static void 7719 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) 7720 { 7721 7722 if (idx) 7723 sbuf_printf(sb, "\n"); 7724 field_desc_show(sb, p[0], tp_la0); 7725 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 7726 field_desc_show(sb, p[1], tp_la0); 7727 } 7728 7729 static void 7730 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) 7731 { 7732 7733 if (idx) 7734 sbuf_printf(sb, "\n"); 7735 field_desc_show(sb, p[0], tp_la0); 7736 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 7737 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); 7738 } 7739 7740 static int 7741 sysctl_tp_la(SYSCTL_HANDLER_ARGS) 7742 { 7743 struct adapter *sc = arg1; 7744 struct sbuf *sb; 7745 uint64_t *buf, *p; 7746 int rc; 7747 u_int i, inc; 7748 void (*show_func)(struct sbuf *, uint64_t *, int); 7749 7750 rc = sysctl_wire_old_buffer(req, 0); 7751 if (rc != 0) 7752 return (rc); 7753 7754 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7755 if (sb == NULL) 7756 return (ENOMEM); 7757 7758 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); 7759 7760 t4_tp_read_la(sc, buf, NULL); 7761 p = buf; 7762 7763 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { 7764 case 2: 7765 inc = 2; 7766 show_func = tp_la_show2; 7767 break; 7768 case 3: 7769 inc = 2; 7770 show_func = tp_la_show3; 7771 break; 7772 default: 7773 inc = 1; 7774 show_func = tp_la_show; 7775 } 7776 7777 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) 7778 (*show_func)(sb, p, i); 7779 7780 rc = sbuf_finish(sb); 7781 sbuf_delete(sb); 7782 free(buf, M_CXGBE); 7783 return (rc); 7784 } 7785 7786 static int 7787 sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 7788 { 7789 struct adapter *sc = arg1; 7790 struct sbuf *sb; 7791 int rc; 7792 u64 nrate[MAX_NCHAN], orate[MAX_NCHAN]; 7793 7794 rc = sysctl_wire_old_buffer(req, 0); 7795 if (rc != 0) 7796 return (rc); 7797 7798 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7799 if (sb == NULL) 7800 return (ENOMEM); 7801 7802 t4_get_chan_txrate(sc, nrate, orate); 7803 7804 if (sc->chip_params->nchan > 2) { 7805 sbuf_printf(sb, " channel 0 channel 1" 7806 " channel 2 channel 3\n"); 7807 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 7808 nrate[0], nrate[1], nrate[2], nrate[3]); 7809 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 7810 orate[0], orate[1], orate[2], orate[3]); 7811 } else { 7812 sbuf_printf(sb, " channel 0 channel 1\n"); 7813 sbuf_printf(sb, "NIC B/s: %10ju %10ju\n", 7814 nrate[0], nrate[1]); 7815 sbuf_printf(sb, "Offload B/s: %10ju %10ju", 7816 orate[0], orate[1]); 7817 } 7818 7819 rc = sbuf_finish(sb); 7820 sbuf_delete(sb); 7821 7822 return (rc); 7823 } 7824 7825 static int 7826 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) 7827 { 7828 struct adapter *sc = arg1; 7829 struct sbuf *sb; 7830 uint32_t *buf, *p; 7831 int rc, i; 7832 7833 rc = sysctl_wire_old_buffer(req, 0); 7834 if (rc != 0) 7835 return (rc); 7836 7837 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7838 if (sb == NULL) 7839 return (ENOMEM); 7840 7841 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, 7842 M_ZERO | M_WAITOK); 7843 7844 t4_ulprx_read_la(sc, buf); 7845 p = buf; 7846 7847 sbuf_printf(sb, " Pcmd Type Message" 7848 " Data"); 7849 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { 7850 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", 7851 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); 7852 } 7853 7854 rc = sbuf_finish(sb); 7855 sbuf_delete(sb); 7856 free(buf, M_CXGBE); 7857 return (rc); 7858 } 7859 7860 static int 7861 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 7862 { 7863 struct adapter *sc = arg1; 7864 struct sbuf *sb; 7865 int rc, v; 7866 7867 rc = sysctl_wire_old_buffer(req, 0); 7868 if (rc != 0) 7869 return (rc); 7870 7871 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7872 if (sb == NULL) 7873 return (ENOMEM); 7874 7875 v = t4_read_reg(sc, A_SGE_STAT_CFG); 7876 if (G_STATSOURCE_T5(v) == 7) { 7877 if (G_STATMODE(v) == 0) { 7878 sbuf_printf(sb, "total %d, incomplete %d", 7879 t4_read_reg(sc, A_SGE_STAT_TOTAL), 7880 t4_read_reg(sc, A_SGE_STAT_MATCH)); 7881 } else if (G_STATMODE(v) == 1) { 7882 sbuf_printf(sb, "total %d, data overflow %d", 7883 t4_read_reg(sc, A_SGE_STAT_TOTAL), 7884 t4_read_reg(sc, A_SGE_STAT_MATCH)); 7885 } 7886 } 7887 rc = sbuf_finish(sb); 7888 sbuf_delete(sb); 7889 7890 return (rc); 7891 } 7892 #endif 7893 7894 static uint32_t 7895 fconf_to_mode(uint32_t fconf) 7896 { 7897 uint32_t mode; 7898 7899 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 7900 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 7901 7902 if (fconf & F_FRAGMENTATION) 7903 mode |= T4_FILTER_IP_FRAGMENT; 7904 7905 if (fconf & F_MPSHITTYPE) 7906 mode |= T4_FILTER_MPS_HIT_TYPE; 7907 7908 if (fconf & F_MACMATCH) 7909 mode |= T4_FILTER_MAC_IDX; 7910 7911 if (fconf & F_ETHERTYPE) 7912 mode |= T4_FILTER_ETH_TYPE; 7913 7914 if (fconf & F_PROTOCOL) 7915 mode |= T4_FILTER_IP_PROTO; 7916 7917 if (fconf & F_TOS) 7918 mode |= T4_FILTER_IP_TOS; 7919 7920 if (fconf & F_VLAN) 7921 mode |= T4_FILTER_VLAN; 7922 7923 if (fconf & F_VNIC_ID) 7924 mode |= T4_FILTER_VNIC; 7925 7926 if (fconf & F_PORT) 7927 mode |= T4_FILTER_PORT; 7928 7929 if (fconf & F_FCOE) 7930 mode |= T4_FILTER_FCoE; 7931 7932 return (mode); 7933 } 7934 7935 static uint32_t 7936 mode_to_fconf(uint32_t mode) 7937 { 7938 uint32_t fconf = 0; 7939 7940 if (mode & T4_FILTER_IP_FRAGMENT) 7941 fconf |= F_FRAGMENTATION; 7942 7943 if (mode & T4_FILTER_MPS_HIT_TYPE) 7944 fconf |= F_MPSHITTYPE; 7945 7946 if (mode & T4_FILTER_MAC_IDX) 7947 fconf |= F_MACMATCH; 7948 7949 if (mode & T4_FILTER_ETH_TYPE) 7950 fconf |= F_ETHERTYPE; 7951 7952 if (mode & T4_FILTER_IP_PROTO) 7953 fconf |= F_PROTOCOL; 7954 7955 if (mode & T4_FILTER_IP_TOS) 7956 fconf |= F_TOS; 7957 7958 if (mode & T4_FILTER_VLAN) 7959 fconf |= F_VLAN; 7960 7961 if (mode & T4_FILTER_VNIC) 7962 fconf |= F_VNIC_ID; 7963 7964 if (mode & T4_FILTER_PORT) 7965 fconf |= F_PORT; 7966 7967 if (mode & T4_FILTER_FCoE) 7968 fconf |= F_FCOE; 7969 7970 return (fconf); 7971 } 7972 7973 static uint32_t 7974 fspec_to_fconf(struct t4_filter_specification *fs) 7975 { 7976 uint32_t fconf = 0; 7977 7978 if (fs->val.frag || fs->mask.frag) 7979 fconf |= F_FRAGMENTATION; 7980 7981 if (fs->val.matchtype || fs->mask.matchtype) 7982 fconf |= F_MPSHITTYPE; 7983 7984 if (fs->val.macidx || fs->mask.macidx) 7985 fconf |= F_MACMATCH; 7986 7987 if (fs->val.ethtype || fs->mask.ethtype) 7988 fconf |= F_ETHERTYPE; 7989 7990 if (fs->val.proto || fs->mask.proto) 7991 fconf |= F_PROTOCOL; 7992 7993 if (fs->val.tos || fs->mask.tos) 7994 fconf |= F_TOS; 7995 7996 if (fs->val.vlan_vld || fs->mask.vlan_vld) 7997 fconf |= F_VLAN; 7998 7999 if (fs->val.vnic_vld || fs->mask.vnic_vld) 8000 fconf |= F_VNIC_ID; 8001 8002 if (fs->val.iport || fs->mask.iport) 8003 fconf |= F_PORT; 8004 8005 if (fs->val.fcoe || fs->mask.fcoe) 8006 fconf |= F_FCOE; 8007 8008 return (fconf); 8009 } 8010 8011 static int 8012 get_filter_mode(struct adapter *sc, uint32_t *mode) 8013 { 8014 int rc; 8015 uint32_t fconf; 8016 8017 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 8018 "t4getfm"); 8019 if (rc) 8020 return (rc); 8021 8022 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 8023 A_TP_VLAN_PRI_MAP); 8024 8025 if (sc->params.tp.vlan_pri_map != fconf) { 8026 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 8027 device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map, 8028 fconf); 8029 } 8030 8031 *mode = fconf_to_mode(fconf); 8032 8033 end_synchronized_op(sc, LOCK_HELD); 8034 return (0); 8035 } 8036 8037 static int 8038 set_filter_mode(struct adapter *sc, uint32_t mode) 8039 { 8040 uint32_t fconf; 8041 int rc; 8042 8043 fconf = mode_to_fconf(mode); 8044 8045 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 8046 "t4setfm"); 8047 if (rc) 8048 return (rc); 8049 8050 if (sc->tids.ftids_in_use > 0) { 8051 rc = EBUSY; 8052 goto done; 8053 } 8054 8055 #ifdef TCP_OFFLOAD 8056 if (uld_active(sc, ULD_TOM)) { 8057 rc = EBUSY; 8058 goto done; 8059 } 8060 #endif 8061 8062 rc = -t4_set_filter_mode(sc, fconf); 8063 done: 8064 end_synchronized_op(sc, LOCK_HELD); 8065 return (rc); 8066 } 8067 8068 static inline uint64_t 8069 get_filter_hits(struct adapter *sc, uint32_t fid) 8070 { 8071 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 8072 uint64_t hits; 8073 8074 memwin_info(sc, 0, &mw_base, NULL); 8075 off = position_memwin(sc, 0, 8076 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 8077 if (is_t4(sc)) { 8078 hits = t4_read_reg64(sc, mw_base + off + 16); 8079 hits = be64toh(hits); 8080 } else { 8081 hits = t4_read_reg(sc, mw_base + off + 24); 8082 hits = be32toh(hits); 8083 } 8084 8085 return (hits); 8086 } 8087 8088 static int 8089 get_filter(struct adapter *sc, struct t4_filter *t) 8090 { 8091 int i, rc, nfilters = sc->tids.nftids; 8092 struct filter_entry *f; 8093 8094 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 8095 "t4getf"); 8096 if (rc) 8097 return (rc); 8098 8099 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 8100 t->idx >= nfilters) { 8101 t->idx = 0xffffffff; 8102 goto done; 8103 } 8104 8105 f = &sc->tids.ftid_tab[t->idx]; 8106 for (i = t->idx; i < nfilters; i++, f++) { 8107 if (f->valid) { 8108 t->idx = i; 8109 t->l2tidx = f->l2t ? f->l2t->idx : 0; 8110 t->smtidx = f->smtidx; 8111 if (f->fs.hitcnts) 8112 t->hits = get_filter_hits(sc, t->idx); 8113 else 8114 t->hits = UINT64_MAX; 8115 t->fs = f->fs; 8116 8117 goto done; 8118 } 8119 } 8120 8121 t->idx = 0xffffffff; 8122 done: 8123 end_synchronized_op(sc, LOCK_HELD); 8124 return (0); 8125 } 8126 8127 static int 8128 set_filter(struct adapter *sc, struct t4_filter *t) 8129 { 8130 unsigned int nfilters, nports; 8131 struct filter_entry *f; 8132 int i, rc; 8133 8134 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 8135 if (rc) 8136 return (rc); 8137 8138 nfilters = sc->tids.nftids; 8139 nports = sc->params.nports; 8140 8141 if (nfilters == 0) { 8142 rc = ENOTSUP; 8143 goto done; 8144 } 8145 8146 if (!(sc->flags & FULL_INIT_DONE)) { 8147 rc = EAGAIN; 8148 goto done; 8149 } 8150 8151 if (t->idx >= nfilters) { 8152 rc = EINVAL; 8153 goto done; 8154 } 8155 8156 /* Validate against the global filter mode */ 8157 if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) != 8158 sc->params.tp.vlan_pri_map) { 8159 rc = E2BIG; 8160 goto done; 8161 } 8162 8163 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 8164 rc = EINVAL; 8165 goto done; 8166 } 8167 8168 if (t->fs.val.iport >= nports) { 8169 rc = EINVAL; 8170 goto done; 8171 } 8172 8173 /* Can't specify an iq if not steering to it */ 8174 if (!t->fs.dirsteer && t->fs.iq) { 8175 rc = EINVAL; 8176 goto done; 8177 } 8178 8179 /* IPv6 filter idx must be 4 aligned */ 8180 if (t->fs.type == 1 && 8181 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 8182 rc = EINVAL; 8183 goto done; 8184 } 8185 8186 if (sc->tids.ftid_tab == NULL) { 8187 KASSERT(sc->tids.ftids_in_use == 0, 8188 ("%s: no memory allocated but filters_in_use > 0", 8189 __func__)); 8190 8191 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 8192 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 8193 if (sc->tids.ftid_tab == NULL) { 8194 rc = ENOMEM; 8195 goto done; 8196 } 8197 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 8198 } 8199 8200 for (i = 0; i < 4; i++) { 8201 f = &sc->tids.ftid_tab[t->idx + i]; 8202 8203 if (f->pending || f->valid) { 8204 rc = EBUSY; 8205 goto done; 8206 } 8207 if (f->locked) { 8208 rc = EPERM; 8209 goto done; 8210 } 8211 8212 if (t->fs.type == 0) 8213 break; 8214 } 8215 8216 f = &sc->tids.ftid_tab[t->idx]; 8217 f->fs = t->fs; 8218 8219 rc = set_filter_wr(sc, t->idx); 8220 done: 8221 end_synchronized_op(sc, 0); 8222 8223 if (rc == 0) { 8224 mtx_lock(&sc->tids.ftid_lock); 8225 for (;;) { 8226 if (f->pending == 0) { 8227 rc = f->valid ? 0 : EIO; 8228 break; 8229 } 8230 8231 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 8232 PCATCH, "t4setfw", 0)) { 8233 rc = EINPROGRESS; 8234 break; 8235 } 8236 } 8237 mtx_unlock(&sc->tids.ftid_lock); 8238 } 8239 return (rc); 8240 } 8241 8242 static int 8243 del_filter(struct adapter *sc, struct t4_filter *t) 8244 { 8245 unsigned int nfilters; 8246 struct filter_entry *f; 8247 int rc; 8248 8249 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 8250 if (rc) 8251 return (rc); 8252 8253 nfilters = sc->tids.nftids; 8254 8255 if (nfilters == 0) { 8256 rc = ENOTSUP; 8257 goto done; 8258 } 8259 8260 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 8261 t->idx >= nfilters) { 8262 rc = EINVAL; 8263 goto done; 8264 } 8265 8266 if (!(sc->flags & FULL_INIT_DONE)) { 8267 rc = EAGAIN; 8268 goto done; 8269 } 8270 8271 f = &sc->tids.ftid_tab[t->idx]; 8272 8273 if (f->pending) { 8274 rc = EBUSY; 8275 goto done; 8276 } 8277 if (f->locked) { 8278 rc = EPERM; 8279 goto done; 8280 } 8281 8282 if (f->valid) { 8283 t->fs = f->fs; /* extra info for the caller */ 8284 rc = del_filter_wr(sc, t->idx); 8285 } 8286 8287 done: 8288 end_synchronized_op(sc, 0); 8289 8290 if (rc == 0) { 8291 mtx_lock(&sc->tids.ftid_lock); 8292 for (;;) { 8293 if (f->pending == 0) { 8294 rc = f->valid ? EIO : 0; 8295 break; 8296 } 8297 8298 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 8299 PCATCH, "t4delfw", 0)) { 8300 rc = EINPROGRESS; 8301 break; 8302 } 8303 } 8304 mtx_unlock(&sc->tids.ftid_lock); 8305 } 8306 8307 return (rc); 8308 } 8309 8310 static void 8311 clear_filter(struct filter_entry *f) 8312 { 8313 if (f->l2t) 8314 t4_l2t_release(f->l2t); 8315 8316 bzero(f, sizeof (*f)); 8317 } 8318 8319 static int 8320 set_filter_wr(struct adapter *sc, int fidx) 8321 { 8322 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 8323 struct fw_filter_wr *fwr; 8324 unsigned int ftid; 8325 struct wrq_cookie cookie; 8326 8327 ASSERT_SYNCHRONIZED_OP(sc); 8328 8329 if (f->fs.newdmac || f->fs.newvlan) { 8330 /* This filter needs an L2T entry; allocate one. */ 8331 f->l2t = t4_l2t_alloc_switching(sc->l2t); 8332 if (f->l2t == NULL) 8333 return (EAGAIN); 8334 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 8335 f->fs.dmac)) { 8336 t4_l2t_release(f->l2t); 8337 f->l2t = NULL; 8338 return (ENOMEM); 8339 } 8340 } 8341 8342 ftid = sc->tids.ftid_base + fidx; 8343 8344 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 8345 if (fwr == NULL) 8346 return (ENOMEM); 8347 bzero(fwr, sizeof(*fwr)); 8348 8349 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 8350 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 8351 fwr->tid_to_iq = 8352 htobe32(V_FW_FILTER_WR_TID(ftid) | 8353 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 8354 V_FW_FILTER_WR_NOREPLY(0) | 8355 V_FW_FILTER_WR_IQ(f->fs.iq)); 8356 fwr->del_filter_to_l2tix = 8357 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 8358 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 8359 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 8360 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 8361 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 8362 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 8363 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 8364 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 8365 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 8366 f->fs.newvlan == VLAN_REWRITE) | 8367 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 8368 f->fs.newvlan == VLAN_REWRITE) | 8369 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 8370 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 8371 V_FW_FILTER_WR_PRIO(f->fs.prio) | 8372 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 8373 fwr->ethtype = htobe16(f->fs.val.ethtype); 8374 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 8375 fwr->frag_to_ovlan_vldm = 8376 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 8377 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 8378 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 8379 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 8380 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 8381 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 8382 fwr->smac_sel = 0; 8383 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 8384 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 8385 fwr->maci_to_matchtypem = 8386 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 8387 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 8388 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 8389 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 8390 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 8391 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 8392 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 8393 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 8394 fwr->ptcl = f->fs.val.proto; 8395 fwr->ptclm = f->fs.mask.proto; 8396 fwr->ttyp = f->fs.val.tos; 8397 fwr->ttypm = f->fs.mask.tos; 8398 fwr->ivlan = htobe16(f->fs.val.vlan); 8399 fwr->ivlanm = htobe16(f->fs.mask.vlan); 8400 fwr->ovlan = htobe16(f->fs.val.vnic); 8401 fwr->ovlanm = htobe16(f->fs.mask.vnic); 8402 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 8403 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 8404 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 8405 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 8406 fwr->lp = htobe16(f->fs.val.dport); 8407 fwr->lpm = htobe16(f->fs.mask.dport); 8408 fwr->fp = htobe16(f->fs.val.sport); 8409 fwr->fpm = htobe16(f->fs.mask.sport); 8410 if (f->fs.newsmac) 8411 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 8412 8413 f->pending = 1; 8414 sc->tids.ftids_in_use++; 8415 8416 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 8417 return (0); 8418 } 8419 8420 static int 8421 del_filter_wr(struct adapter *sc, int fidx) 8422 { 8423 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 8424 struct fw_filter_wr *fwr; 8425 unsigned int ftid; 8426 struct wrq_cookie cookie; 8427 8428 ftid = sc->tids.ftid_base + fidx; 8429 8430 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 8431 if (fwr == NULL) 8432 return (ENOMEM); 8433 bzero(fwr, sizeof (*fwr)); 8434 8435 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 8436 8437 f->pending = 1; 8438 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 8439 return (0); 8440 } 8441 8442 int 8443 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 8444 { 8445 struct adapter *sc = iq->adapter; 8446 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 8447 unsigned int idx = GET_TID(rpl); 8448 unsigned int rc; 8449 struct filter_entry *f; 8450 8451 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 8452 rss->opcode)); 8453 8454 if (is_ftid(sc, idx)) { 8455 8456 idx -= sc->tids.ftid_base; 8457 f = &sc->tids.ftid_tab[idx]; 8458 rc = G_COOKIE(rpl->cookie); 8459 8460 mtx_lock(&sc->tids.ftid_lock); 8461 if (rc == FW_FILTER_WR_FLT_ADDED) { 8462 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 8463 __func__, idx)); 8464 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 8465 f->pending = 0; /* asynchronous setup completed */ 8466 f->valid = 1; 8467 } else { 8468 if (rc != FW_FILTER_WR_FLT_DELETED) { 8469 /* Add or delete failed, display an error */ 8470 log(LOG_ERR, 8471 "filter %u setup failed with error %u\n", 8472 idx, rc); 8473 } 8474 8475 clear_filter(f); 8476 sc->tids.ftids_in_use--; 8477 } 8478 wakeup(&sc->tids.ftid_tab); 8479 mtx_unlock(&sc->tids.ftid_lock); 8480 } 8481 8482 return (0); 8483 } 8484 8485 static int 8486 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 8487 { 8488 int rc; 8489 8490 if (cntxt->cid > M_CTXTQID) 8491 return (EINVAL); 8492 8493 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 8494 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 8495 return (EINVAL); 8496 8497 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 8498 if (rc) 8499 return (rc); 8500 8501 if (sc->flags & FW_OK) { 8502 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 8503 &cntxt->data[0]); 8504 if (rc == 0) 8505 goto done; 8506 } 8507 8508 /* 8509 * Read via firmware failed or wasn't even attempted. Read directly via 8510 * the backdoor. 8511 */ 8512 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 8513 done: 8514 end_synchronized_op(sc, 0); 8515 return (rc); 8516 } 8517 8518 static int 8519 load_fw(struct adapter *sc, struct t4_data *fw) 8520 { 8521 int rc; 8522 uint8_t *fw_data; 8523 8524 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 8525 if (rc) 8526 return (rc); 8527 8528 if (sc->flags & FULL_INIT_DONE) { 8529 rc = EBUSY; 8530 goto done; 8531 } 8532 8533 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 8534 if (fw_data == NULL) { 8535 rc = ENOMEM; 8536 goto done; 8537 } 8538 8539 rc = copyin(fw->data, fw_data, fw->len); 8540 if (rc == 0) 8541 rc = -t4_load_fw(sc, fw_data, fw->len); 8542 8543 free(fw_data, M_CXGBE); 8544 done: 8545 end_synchronized_op(sc, 0); 8546 return (rc); 8547 } 8548 8549 static int 8550 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 8551 { 8552 uint32_t addr, off, remaining, i, n; 8553 uint32_t *buf, *b; 8554 uint32_t mw_base, mw_aperture; 8555 int rc; 8556 uint8_t *dst; 8557 8558 rc = validate_mem_range(sc, mr->addr, mr->len); 8559 if (rc != 0) 8560 return (rc); 8561 8562 memwin_info(sc, win, &mw_base, &mw_aperture); 8563 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 8564 addr = mr->addr; 8565 remaining = mr->len; 8566 dst = (void *)mr->data; 8567 8568 while (remaining) { 8569 off = position_memwin(sc, win, addr); 8570 8571 /* number of bytes that we'll copy in the inner loop */ 8572 n = min(remaining, mw_aperture - off); 8573 for (i = 0; i < n; i += 4) 8574 *b++ = t4_read_reg(sc, mw_base + off + i); 8575 8576 rc = copyout(buf, dst, n); 8577 if (rc != 0) 8578 break; 8579 8580 b = buf; 8581 dst += n; 8582 remaining -= n; 8583 addr += n; 8584 } 8585 8586 free(buf, M_CXGBE); 8587 return (rc); 8588 } 8589 8590 static int 8591 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 8592 { 8593 int rc; 8594 8595 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 8596 return (EINVAL); 8597 8598 if (i2cd->len > sizeof(i2cd->data)) 8599 return (EFBIG); 8600 8601 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 8602 if (rc) 8603 return (rc); 8604 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 8605 i2cd->offset, i2cd->len, &i2cd->data[0]); 8606 end_synchronized_op(sc, 0); 8607 8608 return (rc); 8609 } 8610 8611 static int 8612 in_range(int val, int lo, int hi) 8613 { 8614 8615 return (val < 0 || (val <= hi && val >= lo)); 8616 } 8617 8618 static int 8619 set_sched_class(struct adapter *sc, struct t4_sched_params *p) 8620 { 8621 int fw_subcmd, fw_type, rc; 8622 8623 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc"); 8624 if (rc) 8625 return (rc); 8626 8627 if (!(sc->flags & FULL_INIT_DONE)) { 8628 rc = EAGAIN; 8629 goto done; 8630 } 8631 8632 /* 8633 * Translate the cxgbetool parameters into T4 firmware parameters. (The 8634 * sub-command and type are in common locations.) 8635 */ 8636 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG) 8637 fw_subcmd = FW_SCHED_SC_CONFIG; 8638 else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS) 8639 fw_subcmd = FW_SCHED_SC_PARAMS; 8640 else { 8641 rc = EINVAL; 8642 goto done; 8643 } 8644 if (p->type == SCHED_CLASS_TYPE_PACKET) 8645 fw_type = FW_SCHED_TYPE_PKTSCHED; 8646 else { 8647 rc = EINVAL; 8648 goto done; 8649 } 8650 8651 if (fw_subcmd == FW_SCHED_SC_CONFIG) { 8652 /* Vet our parameters ..*/ 8653 if (p->u.config.minmax < 0) { 8654 rc = EINVAL; 8655 goto done; 8656 } 8657 8658 /* And pass the request to the firmware ...*/ 8659 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1); 8660 goto done; 8661 } 8662 8663 if (fw_subcmd == FW_SCHED_SC_PARAMS) { 8664 int fw_level; 8665 int fw_mode; 8666 int fw_rateunit; 8667 int fw_ratemode; 8668 8669 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL) 8670 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL; 8671 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) 8672 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR; 8673 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) 8674 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL; 8675 else { 8676 rc = EINVAL; 8677 goto done; 8678 } 8679 8680 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS) 8681 fw_mode = FW_SCHED_PARAMS_MODE_CLASS; 8682 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW) 8683 fw_mode = FW_SCHED_PARAMS_MODE_FLOW; 8684 else { 8685 rc = EINVAL; 8686 goto done; 8687 } 8688 8689 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS) 8690 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 8691 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS) 8692 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE; 8693 else { 8694 rc = EINVAL; 8695 goto done; 8696 } 8697 8698 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL) 8699 fw_ratemode = FW_SCHED_PARAMS_RATE_REL; 8700 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS) 8701 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS; 8702 else { 8703 rc = EINVAL; 8704 goto done; 8705 } 8706 8707 /* Vet our parameters ... */ 8708 if (!in_range(p->u.params.channel, 0, 3) || 8709 !in_range(p->u.params.cl, 0, sc->chip_params->nsched_cls) || 8710 !in_range(p->u.params.minrate, 0, 10000000) || 8711 !in_range(p->u.params.maxrate, 0, 10000000) || 8712 !in_range(p->u.params.weight, 0, 100)) { 8713 rc = ERANGE; 8714 goto done; 8715 } 8716 8717 /* 8718 * Translate any unset parameters into the firmware's 8719 * nomenclature and/or fail the call if the parameters 8720 * are required ... 8721 */ 8722 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 || 8723 p->u.params.channel < 0 || p->u.params.cl < 0) { 8724 rc = EINVAL; 8725 goto done; 8726 } 8727 if (p->u.params.minrate < 0) 8728 p->u.params.minrate = 0; 8729 if (p->u.params.maxrate < 0) { 8730 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 8731 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 8732 rc = EINVAL; 8733 goto done; 8734 } else 8735 p->u.params.maxrate = 0; 8736 } 8737 if (p->u.params.weight < 0) { 8738 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) { 8739 rc = EINVAL; 8740 goto done; 8741 } else 8742 p->u.params.weight = 0; 8743 } 8744 if (p->u.params.pktsize < 0) { 8745 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 8746 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 8747 rc = EINVAL; 8748 goto done; 8749 } else 8750 p->u.params.pktsize = 0; 8751 } 8752 8753 /* See what the firmware thinks of the request ... */ 8754 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode, 8755 fw_rateunit, fw_ratemode, p->u.params.channel, 8756 p->u.params.cl, p->u.params.minrate, p->u.params.maxrate, 8757 p->u.params.weight, p->u.params.pktsize, 1); 8758 goto done; 8759 } 8760 8761 rc = EINVAL; 8762 done: 8763 end_synchronized_op(sc, 0); 8764 return (rc); 8765 } 8766 8767 static int 8768 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p) 8769 { 8770 struct port_info *pi = NULL; 8771 struct vi_info *vi; 8772 struct sge_txq *txq; 8773 uint32_t fw_mnem, fw_queue, fw_class; 8774 int i, rc; 8775 8776 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq"); 8777 if (rc) 8778 return (rc); 8779 8780 if (!(sc->flags & FULL_INIT_DONE)) { 8781 rc = EAGAIN; 8782 goto done; 8783 } 8784 8785 if (p->port >= sc->params.nports) { 8786 rc = EINVAL; 8787 goto done; 8788 } 8789 8790 /* XXX: Only supported for the main VI. */ 8791 pi = sc->port[p->port]; 8792 vi = &pi->vi[0]; 8793 if (!in_range(p->queue, 0, vi->ntxq - 1) || !in_range(p->cl, 0, 7)) { 8794 rc = EINVAL; 8795 goto done; 8796 } 8797 8798 /* 8799 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX 8800 * Scheduling Class in this case). 8801 */ 8802 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 8803 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH)); 8804 fw_class = p->cl < 0 ? 0xffffffff : p->cl; 8805 8806 /* 8807 * If op.queue is non-negative, then we're only changing the scheduling 8808 * on a single specified TX queue. 8809 */ 8810 if (p->queue >= 0) { 8811 txq = &sc->sge.txq[vi->first_txq + p->queue]; 8812 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 8813 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 8814 &fw_class); 8815 goto done; 8816 } 8817 8818 /* 8819 * Change the scheduling on all the TX queues for the 8820 * interface. 8821 */ 8822 for_each_txq(vi, i, txq) { 8823 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 8824 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 8825 &fw_class); 8826 if (rc) 8827 goto done; 8828 } 8829 8830 rc = 0; 8831 done: 8832 end_synchronized_op(sc, 0); 8833 return (rc); 8834 } 8835 8836 int 8837 t4_os_find_pci_capability(struct adapter *sc, int cap) 8838 { 8839 int i; 8840 8841 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 8842 } 8843 8844 int 8845 t4_os_pci_save_state(struct adapter *sc) 8846 { 8847 device_t dev; 8848 struct pci_devinfo *dinfo; 8849 8850 dev = sc->dev; 8851 dinfo = device_get_ivars(dev); 8852 8853 pci_cfg_save(dev, dinfo, 0); 8854 return (0); 8855 } 8856 8857 int 8858 t4_os_pci_restore_state(struct adapter *sc) 8859 { 8860 device_t dev; 8861 struct pci_devinfo *dinfo; 8862 8863 dev = sc->dev; 8864 dinfo = device_get_ivars(dev); 8865 8866 pci_cfg_restore(dev, dinfo); 8867 return (0); 8868 } 8869 8870 void 8871 t4_os_portmod_changed(const struct adapter *sc, int idx) 8872 { 8873 struct port_info *pi = sc->port[idx]; 8874 struct vi_info *vi; 8875 struct ifnet *ifp; 8876 int v; 8877 static const char *mod_str[] = { 8878 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 8879 }; 8880 8881 for_each_vi(pi, v, vi) { 8882 build_medialist(pi, &vi->media); 8883 } 8884 8885 ifp = pi->vi[0].ifp; 8886 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 8887 if_printf(ifp, "transceiver unplugged.\n"); 8888 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 8889 if_printf(ifp, "unknown transceiver inserted.\n"); 8890 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 8891 if_printf(ifp, "unsupported transceiver inserted.\n"); 8892 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 8893 if_printf(ifp, "%s transceiver inserted.\n", 8894 mod_str[pi->mod_type]); 8895 } else { 8896 if_printf(ifp, "transceiver (type %d) inserted.\n", 8897 pi->mod_type); 8898 } 8899 } 8900 8901 void 8902 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason) 8903 { 8904 struct port_info *pi = sc->port[idx]; 8905 struct vi_info *vi; 8906 struct ifnet *ifp; 8907 int v; 8908 8909 if (link_stat) 8910 pi->linkdnrc = -1; 8911 else { 8912 if (reason >= 0) 8913 pi->linkdnrc = reason; 8914 } 8915 for_each_vi(pi, v, vi) { 8916 ifp = vi->ifp; 8917 if (ifp == NULL) 8918 continue; 8919 8920 if (link_stat) { 8921 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 8922 if_link_state_change(ifp, LINK_STATE_UP); 8923 } else { 8924 if_link_state_change(ifp, LINK_STATE_DOWN); 8925 } 8926 } 8927 } 8928 8929 void 8930 t4_iterate(void (*func)(struct adapter *, void *), void *arg) 8931 { 8932 struct adapter *sc; 8933 8934 sx_slock(&t4_list_lock); 8935 SLIST_FOREACH(sc, &t4_list, link) { 8936 /* 8937 * func should not make any assumptions about what state sc is 8938 * in - the only guarantee is that sc->sc_lock is a valid lock. 8939 */ 8940 func(sc, arg); 8941 } 8942 sx_sunlock(&t4_list_lock); 8943 } 8944 8945 static int 8946 t4_open(struct cdev *dev, int flags, int type, struct thread *td) 8947 { 8948 return (0); 8949 } 8950 8951 static int 8952 t4_close(struct cdev *dev, int flags, int type, struct thread *td) 8953 { 8954 return (0); 8955 } 8956 8957 static int 8958 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 8959 struct thread *td) 8960 { 8961 int rc; 8962 struct adapter *sc = dev->si_drv1; 8963 8964 rc = priv_check(td, PRIV_DRIVER); 8965 if (rc != 0) 8966 return (rc); 8967 8968 switch (cmd) { 8969 case CHELSIO_T4_GETREG: { 8970 struct t4_reg *edata = (struct t4_reg *)data; 8971 8972 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8973 return (EFAULT); 8974 8975 if (edata->size == 4) 8976 edata->val = t4_read_reg(sc, edata->addr); 8977 else if (edata->size == 8) 8978 edata->val = t4_read_reg64(sc, edata->addr); 8979 else 8980 return (EINVAL); 8981 8982 break; 8983 } 8984 case CHELSIO_T4_SETREG: { 8985 struct t4_reg *edata = (struct t4_reg *)data; 8986 8987 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8988 return (EFAULT); 8989 8990 if (edata->size == 4) { 8991 if (edata->val & 0xffffffff00000000) 8992 return (EINVAL); 8993 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 8994 } else if (edata->size == 8) 8995 t4_write_reg64(sc, edata->addr, edata->val); 8996 else 8997 return (EINVAL); 8998 break; 8999 } 9000 case CHELSIO_T4_REGDUMP: { 9001 struct t4_regdump *regs = (struct t4_regdump *)data; 9002 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 9003 uint8_t *buf; 9004 9005 if (regs->len < reglen) { 9006 regs->len = reglen; /* hint to the caller */ 9007 return (ENOBUFS); 9008 } 9009 9010 regs->len = reglen; 9011 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 9012 t4_get_regs(sc, regs, buf); 9013 rc = copyout(buf, regs->data, reglen); 9014 free(buf, M_CXGBE); 9015 break; 9016 } 9017 case CHELSIO_T4_GET_FILTER_MODE: 9018 rc = get_filter_mode(sc, (uint32_t *)data); 9019 break; 9020 case CHELSIO_T4_SET_FILTER_MODE: 9021 rc = set_filter_mode(sc, *(uint32_t *)data); 9022 break; 9023 case CHELSIO_T4_GET_FILTER: 9024 rc = get_filter(sc, (struct t4_filter *)data); 9025 break; 9026 case CHELSIO_T4_SET_FILTER: 9027 rc = set_filter(sc, (struct t4_filter *)data); 9028 break; 9029 case CHELSIO_T4_DEL_FILTER: 9030 rc = del_filter(sc, (struct t4_filter *)data); 9031 break; 9032 case CHELSIO_T4_GET_SGE_CONTEXT: 9033 rc = get_sge_context(sc, (struct t4_sge_context *)data); 9034 break; 9035 case CHELSIO_T4_LOAD_FW: 9036 rc = load_fw(sc, (struct t4_data *)data); 9037 break; 9038 case CHELSIO_T4_GET_MEM: 9039 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 9040 break; 9041 case CHELSIO_T4_GET_I2C: 9042 rc = read_i2c(sc, (struct t4_i2c_data *)data); 9043 break; 9044 case CHELSIO_T4_CLEAR_STATS: { 9045 int i, v; 9046 u_int port_id = *(uint32_t *)data; 9047 struct port_info *pi; 9048 struct vi_info *vi; 9049 9050 if (port_id >= sc->params.nports) 9051 return (EINVAL); 9052 pi = sc->port[port_id]; 9053 9054 /* MAC stats */ 9055 t4_clr_port_stats(sc, pi->tx_chan); 9056 pi->tx_parse_error = 0; 9057 mtx_lock(&sc->regwin_lock); 9058 for_each_vi(pi, v, vi) { 9059 if (vi->flags & VI_INIT_DONE) 9060 t4_clr_vi_stats(sc, vi->viid); 9061 } 9062 mtx_unlock(&sc->regwin_lock); 9063 9064 /* 9065 * Since this command accepts a port, clear stats for 9066 * all VIs on this port. 9067 */ 9068 for_each_vi(pi, v, vi) { 9069 if (vi->flags & VI_INIT_DONE) { 9070 struct sge_rxq *rxq; 9071 struct sge_txq *txq; 9072 struct sge_wrq *wrq; 9073 9074 if (vi->flags & VI_NETMAP) 9075 continue; 9076 9077 for_each_rxq(vi, i, rxq) { 9078 #if defined(INET) || defined(INET6) 9079 rxq->lro.lro_queued = 0; 9080 rxq->lro.lro_flushed = 0; 9081 #endif 9082 rxq->rxcsum = 0; 9083 rxq->vlan_extraction = 0; 9084 } 9085 9086 for_each_txq(vi, i, txq) { 9087 txq->txcsum = 0; 9088 txq->tso_wrs = 0; 9089 txq->vlan_insertion = 0; 9090 txq->imm_wrs = 0; 9091 txq->sgl_wrs = 0; 9092 txq->txpkt_wrs = 0; 9093 txq->txpkts0_wrs = 0; 9094 txq->txpkts1_wrs = 0; 9095 txq->txpkts0_pkts = 0; 9096 txq->txpkts1_pkts = 0; 9097 mp_ring_reset_stats(txq->r); 9098 } 9099 9100 #ifdef TCP_OFFLOAD 9101 /* nothing to clear for each ofld_rxq */ 9102 9103 for_each_ofld_txq(vi, i, wrq) { 9104 wrq->tx_wrs_direct = 0; 9105 wrq->tx_wrs_copied = 0; 9106 } 9107 #endif 9108 9109 if (IS_MAIN_VI(vi)) { 9110 wrq = &sc->sge.ctrlq[pi->port_id]; 9111 wrq->tx_wrs_direct = 0; 9112 wrq->tx_wrs_copied = 0; 9113 } 9114 } 9115 } 9116 break; 9117 } 9118 case CHELSIO_T4_SCHED_CLASS: 9119 rc = set_sched_class(sc, (struct t4_sched_params *)data); 9120 break; 9121 case CHELSIO_T4_SCHED_QUEUE: 9122 rc = set_sched_queue(sc, (struct t4_sched_queue *)data); 9123 break; 9124 case CHELSIO_T4_GET_TRACER: 9125 rc = t4_get_tracer(sc, (struct t4_tracer *)data); 9126 break; 9127 case CHELSIO_T4_SET_TRACER: 9128 rc = t4_set_tracer(sc, (struct t4_tracer *)data); 9129 break; 9130 default: 9131 rc = EINVAL; 9132 } 9133 9134 return (rc); 9135 } 9136 9137 #ifdef TCP_OFFLOAD 9138 void 9139 t4_iscsi_init(struct adapter *sc, u_int tag_mask, const u_int *pgsz_order) 9140 { 9141 9142 t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask); 9143 t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) | 9144 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) | 9145 V_HPZ3(pgsz_order[3])); 9146 } 9147 9148 static int 9149 toe_capability(struct vi_info *vi, int enable) 9150 { 9151 int rc; 9152 struct port_info *pi = vi->pi; 9153 struct adapter *sc = pi->adapter; 9154 9155 ASSERT_SYNCHRONIZED_OP(sc); 9156 9157 if (!is_offload(sc)) 9158 return (ENODEV); 9159 9160 if (enable) { 9161 if ((vi->ifp->if_capenable & IFCAP_TOE) != 0) { 9162 /* TOE is already enabled. */ 9163 return (0); 9164 } 9165 9166 /* 9167 * We need the port's queues around so that we're able to send 9168 * and receive CPLs to/from the TOE even if the ifnet for this 9169 * port has never been UP'd administratively. 9170 */ 9171 if (!(vi->flags & VI_INIT_DONE)) { 9172 rc = cxgbe_init_synchronized(vi); 9173 if (rc) 9174 return (rc); 9175 } 9176 if (!(pi->vi[0].flags & VI_INIT_DONE)) { 9177 rc = cxgbe_init_synchronized(&pi->vi[0]); 9178 if (rc) 9179 return (rc); 9180 } 9181 9182 if (isset(&sc->offload_map, pi->port_id)) { 9183 /* TOE is enabled on another VI of this port. */ 9184 pi->uld_vis++; 9185 return (0); 9186 } 9187 9188 if (!uld_active(sc, ULD_TOM)) { 9189 rc = t4_activate_uld(sc, ULD_TOM); 9190 if (rc == EAGAIN) { 9191 log(LOG_WARNING, 9192 "You must kldload t4_tom.ko before trying " 9193 "to enable TOE on a cxgbe interface.\n"); 9194 } 9195 if (rc != 0) 9196 return (rc); 9197 KASSERT(sc->tom_softc != NULL, 9198 ("%s: TOM activated but softc NULL", __func__)); 9199 KASSERT(uld_active(sc, ULD_TOM), 9200 ("%s: TOM activated but flag not set", __func__)); 9201 } 9202 9203 /* Activate iWARP and iSCSI too, if the modules are loaded. */ 9204 if (!uld_active(sc, ULD_IWARP)) 9205 (void) t4_activate_uld(sc, ULD_IWARP); 9206 if (!uld_active(sc, ULD_ISCSI)) 9207 (void) t4_activate_uld(sc, ULD_ISCSI); 9208 9209 pi->uld_vis++; 9210 setbit(&sc->offload_map, pi->port_id); 9211 } else { 9212 pi->uld_vis--; 9213 9214 if (!isset(&sc->offload_map, pi->port_id) || pi->uld_vis > 0) 9215 return (0); 9216 9217 KASSERT(uld_active(sc, ULD_TOM), 9218 ("%s: TOM never initialized?", __func__)); 9219 clrbit(&sc->offload_map, pi->port_id); 9220 } 9221 9222 return (0); 9223 } 9224 9225 /* 9226 * Add an upper layer driver to the global list. 9227 */ 9228 int 9229 t4_register_uld(struct uld_info *ui) 9230 { 9231 int rc = 0; 9232 struct uld_info *u; 9233 9234 sx_xlock(&t4_uld_list_lock); 9235 SLIST_FOREACH(u, &t4_uld_list, link) { 9236 if (u->uld_id == ui->uld_id) { 9237 rc = EEXIST; 9238 goto done; 9239 } 9240 } 9241 9242 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 9243 ui->refcount = 0; 9244 done: 9245 sx_xunlock(&t4_uld_list_lock); 9246 return (rc); 9247 } 9248 9249 int 9250 t4_unregister_uld(struct uld_info *ui) 9251 { 9252 int rc = EINVAL; 9253 struct uld_info *u; 9254 9255 sx_xlock(&t4_uld_list_lock); 9256 9257 SLIST_FOREACH(u, &t4_uld_list, link) { 9258 if (u == ui) { 9259 if (ui->refcount > 0) { 9260 rc = EBUSY; 9261 goto done; 9262 } 9263 9264 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 9265 rc = 0; 9266 goto done; 9267 } 9268 } 9269 done: 9270 sx_xunlock(&t4_uld_list_lock); 9271 return (rc); 9272 } 9273 9274 int 9275 t4_activate_uld(struct adapter *sc, int id) 9276 { 9277 int rc; 9278 struct uld_info *ui; 9279 9280 ASSERT_SYNCHRONIZED_OP(sc); 9281 9282 if (id < 0 || id > ULD_MAX) 9283 return (EINVAL); 9284 rc = EAGAIN; /* kldoad the module with this ULD and try again. */ 9285 9286 sx_slock(&t4_uld_list_lock); 9287 9288 SLIST_FOREACH(ui, &t4_uld_list, link) { 9289 if (ui->uld_id == id) { 9290 if (!(sc->flags & FULL_INIT_DONE)) { 9291 rc = adapter_full_init(sc); 9292 if (rc != 0) 9293 break; 9294 } 9295 9296 rc = ui->activate(sc); 9297 if (rc == 0) { 9298 setbit(&sc->active_ulds, id); 9299 ui->refcount++; 9300 } 9301 break; 9302 } 9303 } 9304 9305 sx_sunlock(&t4_uld_list_lock); 9306 9307 return (rc); 9308 } 9309 9310 int 9311 t4_deactivate_uld(struct adapter *sc, int id) 9312 { 9313 int rc; 9314 struct uld_info *ui; 9315 9316 ASSERT_SYNCHRONIZED_OP(sc); 9317 9318 if (id < 0 || id > ULD_MAX) 9319 return (EINVAL); 9320 rc = ENXIO; 9321 9322 sx_slock(&t4_uld_list_lock); 9323 9324 SLIST_FOREACH(ui, &t4_uld_list, link) { 9325 if (ui->uld_id == id) { 9326 rc = ui->deactivate(sc); 9327 if (rc == 0) { 9328 clrbit(&sc->active_ulds, id); 9329 ui->refcount--; 9330 } 9331 break; 9332 } 9333 } 9334 9335 sx_sunlock(&t4_uld_list_lock); 9336 9337 return (rc); 9338 } 9339 9340 int 9341 uld_active(struct adapter *sc, int uld_id) 9342 { 9343 9344 MPASS(uld_id >= 0 && uld_id <= ULD_MAX); 9345 9346 return (isset(&sc->active_ulds, uld_id)); 9347 } 9348 #endif 9349 9350 /* 9351 * Come up with reasonable defaults for some of the tunables, provided they're 9352 * not set by the user (in which case we'll use the values as is). 9353 */ 9354 static void 9355 tweak_tunables(void) 9356 { 9357 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 9358 9359 if (t4_ntxq10g < 1) { 9360 #ifdef RSS 9361 t4_ntxq10g = rss_getnumbuckets(); 9362 #else 9363 t4_ntxq10g = min(nc, NTXQ_10G); 9364 #endif 9365 } 9366 9367 if (t4_ntxq1g < 1) { 9368 #ifdef RSS 9369 /* XXX: way too many for 1GbE? */ 9370 t4_ntxq1g = rss_getnumbuckets(); 9371 #else 9372 t4_ntxq1g = min(nc, NTXQ_1G); 9373 #endif 9374 } 9375 9376 if (t4_nrxq10g < 1) { 9377 #ifdef RSS 9378 t4_nrxq10g = rss_getnumbuckets(); 9379 #else 9380 t4_nrxq10g = min(nc, NRXQ_10G); 9381 #endif 9382 } 9383 9384 if (t4_nrxq1g < 1) { 9385 #ifdef RSS 9386 /* XXX: way too many for 1GbE? */ 9387 t4_nrxq1g = rss_getnumbuckets(); 9388 #else 9389 t4_nrxq1g = min(nc, NRXQ_1G); 9390 #endif 9391 } 9392 9393 #ifdef TCP_OFFLOAD 9394 if (t4_nofldtxq10g < 1) 9395 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 9396 9397 if (t4_nofldtxq1g < 1) 9398 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 9399 9400 if (t4_nofldrxq10g < 1) 9401 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 9402 9403 if (t4_nofldrxq1g < 1) 9404 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 9405 9406 if (t4_toecaps_allowed == -1) 9407 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 9408 #else 9409 if (t4_toecaps_allowed == -1) 9410 t4_toecaps_allowed = 0; 9411 #endif 9412 9413 #ifdef DEV_NETMAP 9414 if (t4_nnmtxq10g < 1) 9415 t4_nnmtxq10g = min(nc, NNMTXQ_10G); 9416 9417 if (t4_nnmtxq1g < 1) 9418 t4_nnmtxq1g = min(nc, NNMTXQ_1G); 9419 9420 if (t4_nnmrxq10g < 1) 9421 t4_nnmrxq10g = min(nc, NNMRXQ_10G); 9422 9423 if (t4_nnmrxq1g < 1) 9424 t4_nnmrxq1g = min(nc, NNMRXQ_1G); 9425 #endif 9426 9427 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 9428 t4_tmr_idx_10g = TMR_IDX_10G; 9429 9430 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 9431 t4_pktc_idx_10g = PKTC_IDX_10G; 9432 9433 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 9434 t4_tmr_idx_1g = TMR_IDX_1G; 9435 9436 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 9437 t4_pktc_idx_1g = PKTC_IDX_1G; 9438 9439 if (t4_qsize_txq < 128) 9440 t4_qsize_txq = 128; 9441 9442 if (t4_qsize_rxq < 128) 9443 t4_qsize_rxq = 128; 9444 while (t4_qsize_rxq & 7) 9445 t4_qsize_rxq++; 9446 9447 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 9448 } 9449 9450 static struct sx mlu; /* mod load unload */ 9451 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload"); 9452 9453 static int 9454 mod_event(module_t mod, int cmd, void *arg) 9455 { 9456 int rc = 0; 9457 static int loaded = 0; 9458 9459 switch (cmd) { 9460 case MOD_LOAD: 9461 sx_xlock(&mlu); 9462 if (loaded++ == 0) { 9463 t4_sge_modload(); 9464 sx_init(&t4_list_lock, "T4/T5 adapters"); 9465 SLIST_INIT(&t4_list); 9466 #ifdef TCP_OFFLOAD 9467 sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); 9468 SLIST_INIT(&t4_uld_list); 9469 #endif 9470 t4_tracer_modload(); 9471 tweak_tunables(); 9472 } 9473 sx_xunlock(&mlu); 9474 break; 9475 9476 case MOD_UNLOAD: 9477 sx_xlock(&mlu); 9478 if (--loaded == 0) { 9479 int tries; 9480 9481 sx_slock(&t4_list_lock); 9482 if (!SLIST_EMPTY(&t4_list)) { 9483 rc = EBUSY; 9484 sx_sunlock(&t4_list_lock); 9485 goto done_unload; 9486 } 9487 #ifdef TCP_OFFLOAD 9488 sx_slock(&t4_uld_list_lock); 9489 if (!SLIST_EMPTY(&t4_uld_list)) { 9490 rc = EBUSY; 9491 sx_sunlock(&t4_uld_list_lock); 9492 sx_sunlock(&t4_list_lock); 9493 goto done_unload; 9494 } 9495 #endif 9496 tries = 0; 9497 while (tries++ < 5 && t4_sge_extfree_refs() != 0) { 9498 uprintf("%ju clusters with custom free routine " 9499 "still is use.\n", t4_sge_extfree_refs()); 9500 pause("t4unload", 2 * hz); 9501 } 9502 #ifdef TCP_OFFLOAD 9503 sx_sunlock(&t4_uld_list_lock); 9504 #endif 9505 sx_sunlock(&t4_list_lock); 9506 9507 if (t4_sge_extfree_refs() == 0) { 9508 t4_tracer_modunload(); 9509 #ifdef TCP_OFFLOAD 9510 sx_destroy(&t4_uld_list_lock); 9511 #endif 9512 sx_destroy(&t4_list_lock); 9513 t4_sge_modunload(); 9514 loaded = 0; 9515 } else { 9516 rc = EBUSY; 9517 loaded++; /* undo earlier decrement */ 9518 } 9519 } 9520 done_unload: 9521 sx_xunlock(&mlu); 9522 break; 9523 } 9524 9525 return (rc); 9526 } 9527 9528 static devclass_t t4_devclass, t5_devclass; 9529 static devclass_t cxgbe_devclass, cxl_devclass; 9530 static devclass_t vcxgbe_devclass, vcxl_devclass; 9531 9532 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 9533 MODULE_VERSION(t4nex, 1); 9534 MODULE_DEPEND(t4nex, firmware, 1, 1, 1); 9535 #ifdef DEV_NETMAP 9536 MODULE_DEPEND(t4nex, netmap, 1, 1, 1); 9537 #endif /* DEV_NETMAP */ 9538 9539 9540 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 9541 MODULE_VERSION(t5nex, 1); 9542 MODULE_DEPEND(t5nex, firmware, 1, 1, 1); 9543 #ifdef DEV_NETMAP 9544 MODULE_DEPEND(t5nex, netmap, 1, 1, 1); 9545 #endif /* DEV_NETMAP */ 9546 9547 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 9548 MODULE_VERSION(cxgbe, 1); 9549 9550 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 9551 MODULE_VERSION(cxl, 1); 9552 9553 DRIVER_MODULE(vcxgbe, cxgbe, vcxgbe_driver, vcxgbe_devclass, 0, 0); 9554 MODULE_VERSION(vcxgbe, 1); 9555 9556 DRIVER_MODULE(vcxl, cxl, vcxl_driver, vcxl_devclass, 0, 0); 9557 MODULE_VERSION(vcxl, 1); 9558