1 /*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_inet.h" 32 #include "opt_inet6.h" 33 #include "opt_rss.h" 34 35 #include <sys/param.h> 36 #include <sys/conf.h> 37 #include <sys/priv.h> 38 #include <sys/kernel.h> 39 #include <sys/bus.h> 40 #include <sys/module.h> 41 #include <sys/malloc.h> 42 #include <sys/queue.h> 43 #include <sys/taskqueue.h> 44 #include <sys/pciio.h> 45 #include <dev/pci/pcireg.h> 46 #include <dev/pci/pcivar.h> 47 #include <dev/pci/pci_private.h> 48 #include <sys/firmware.h> 49 #include <sys/sbuf.h> 50 #include <sys/smp.h> 51 #include <sys/socket.h> 52 #include <sys/sockio.h> 53 #include <sys/sysctl.h> 54 #include <net/ethernet.h> 55 #include <net/if.h> 56 #include <net/if_types.h> 57 #include <net/if_dl.h> 58 #include <net/if_vlan_var.h> 59 #ifdef RSS 60 #include <net/rss_config.h> 61 #endif 62 #if defined(__i386__) || defined(__amd64__) 63 #include <vm/vm.h> 64 #include <vm/pmap.h> 65 #endif 66 67 #include "common/common.h" 68 #include "common/t4_msg.h" 69 #include "common/t4_regs.h" 70 #include "common/t4_regs_values.h" 71 #include "t4_ioctl.h" 72 #include "t4_l2t.h" 73 #include "t4_mp_ring.h" 74 75 /* T4 bus driver interface */ 76 static int t4_probe(device_t); 77 static int t4_attach(device_t); 78 static int t4_detach(device_t); 79 static device_method_t t4_methods[] = { 80 DEVMETHOD(device_probe, t4_probe), 81 DEVMETHOD(device_attach, t4_attach), 82 DEVMETHOD(device_detach, t4_detach), 83 84 DEVMETHOD_END 85 }; 86 static driver_t t4_driver = { 87 "t4nex", 88 t4_methods, 89 sizeof(struct adapter) 90 }; 91 92 93 /* T4 port (cxgbe) interface */ 94 static int cxgbe_probe(device_t); 95 static int cxgbe_attach(device_t); 96 static int cxgbe_detach(device_t); 97 static device_method_t cxgbe_methods[] = { 98 DEVMETHOD(device_probe, cxgbe_probe), 99 DEVMETHOD(device_attach, cxgbe_attach), 100 DEVMETHOD(device_detach, cxgbe_detach), 101 { 0, 0 } 102 }; 103 static driver_t cxgbe_driver = { 104 "cxgbe", 105 cxgbe_methods, 106 sizeof(struct port_info) 107 }; 108 109 /* T4 VI (vcxgbe) interface */ 110 static int vcxgbe_probe(device_t); 111 static int vcxgbe_attach(device_t); 112 static int vcxgbe_detach(device_t); 113 static device_method_t vcxgbe_methods[] = { 114 DEVMETHOD(device_probe, vcxgbe_probe), 115 DEVMETHOD(device_attach, vcxgbe_attach), 116 DEVMETHOD(device_detach, vcxgbe_detach), 117 { 0, 0 } 118 }; 119 static driver_t vcxgbe_driver = { 120 "vcxgbe", 121 vcxgbe_methods, 122 sizeof(struct vi_info) 123 }; 124 125 static d_ioctl_t t4_ioctl; 126 static d_open_t t4_open; 127 static d_close_t t4_close; 128 129 static struct cdevsw t4_cdevsw = { 130 .d_version = D_VERSION, 131 .d_flags = 0, 132 .d_open = t4_open, 133 .d_close = t4_close, 134 .d_ioctl = t4_ioctl, 135 .d_name = "t4nex", 136 }; 137 138 /* T5 bus driver interface */ 139 static int t5_probe(device_t); 140 static device_method_t t5_methods[] = { 141 DEVMETHOD(device_probe, t5_probe), 142 DEVMETHOD(device_attach, t4_attach), 143 DEVMETHOD(device_detach, t4_detach), 144 145 DEVMETHOD_END 146 }; 147 static driver_t t5_driver = { 148 "t5nex", 149 t5_methods, 150 sizeof(struct adapter) 151 }; 152 153 154 /* T5 port (cxl) interface */ 155 static driver_t cxl_driver = { 156 "cxl", 157 cxgbe_methods, 158 sizeof(struct port_info) 159 }; 160 161 /* T5 VI (vcxl) interface */ 162 static driver_t vcxl_driver = { 163 "vcxl", 164 vcxgbe_methods, 165 sizeof(struct vi_info) 166 }; 167 168 static struct cdevsw t5_cdevsw = { 169 .d_version = D_VERSION, 170 .d_flags = 0, 171 .d_open = t4_open, 172 .d_close = t4_close, 173 .d_ioctl = t4_ioctl, 174 .d_name = "t5nex", 175 }; 176 177 /* ifnet + media interface */ 178 static void cxgbe_init(void *); 179 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 180 static int cxgbe_transmit(struct ifnet *, struct mbuf *); 181 static void cxgbe_qflush(struct ifnet *); 182 static int cxgbe_media_change(struct ifnet *); 183 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 184 185 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 186 187 /* 188 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 189 * then ADAPTER_LOCK, then t4_uld_list_lock. 190 */ 191 static struct sx t4_list_lock; 192 SLIST_HEAD(, adapter) t4_list; 193 #ifdef TCP_OFFLOAD 194 static struct sx t4_uld_list_lock; 195 SLIST_HEAD(, uld_info) t4_uld_list; 196 #endif 197 198 /* 199 * Tunables. See tweak_tunables() too. 200 * 201 * Each tunable is set to a default value here if it's known at compile-time. 202 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 203 * provide a reasonable default when the driver is loaded. 204 * 205 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 206 * T5 are under hw.cxl. 207 */ 208 209 /* 210 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 211 */ 212 #define NTXQ_10G 16 213 static int t4_ntxq10g = -1; 214 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 215 216 #define NRXQ_10G 8 217 static int t4_nrxq10g = -1; 218 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 219 220 #define NTXQ_1G 4 221 static int t4_ntxq1g = -1; 222 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 223 224 #define NRXQ_1G 2 225 static int t4_nrxq1g = -1; 226 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 227 228 static int t4_rsrv_noflowq = 0; 229 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq); 230 231 #ifdef TCP_OFFLOAD 232 #define NOFLDTXQ_10G 8 233 static int t4_nofldtxq10g = -1; 234 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 235 236 #define NOFLDRXQ_10G 2 237 static int t4_nofldrxq10g = -1; 238 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 239 240 #define NOFLDTXQ_1G 2 241 static int t4_nofldtxq1g = -1; 242 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 243 244 #define NOFLDRXQ_1G 1 245 static int t4_nofldrxq1g = -1; 246 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 247 #endif 248 249 #ifdef DEV_NETMAP 250 #define NNMTXQ_10G 2 251 static int t4_nnmtxq10g = -1; 252 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g); 253 254 #define NNMRXQ_10G 2 255 static int t4_nnmrxq10g = -1; 256 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g); 257 258 #define NNMTXQ_1G 1 259 static int t4_nnmtxq1g = -1; 260 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g); 261 262 #define NNMRXQ_1G 1 263 static int t4_nnmrxq1g = -1; 264 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g); 265 #endif 266 267 /* 268 * Holdoff parameters for 10G and 1G ports. 269 */ 270 #define TMR_IDX_10G 1 271 static int t4_tmr_idx_10g = TMR_IDX_10G; 272 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 273 274 #define PKTC_IDX_10G (-1) 275 static int t4_pktc_idx_10g = PKTC_IDX_10G; 276 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 277 278 #define TMR_IDX_1G 1 279 static int t4_tmr_idx_1g = TMR_IDX_1G; 280 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 281 282 #define PKTC_IDX_1G (-1) 283 static int t4_pktc_idx_1g = PKTC_IDX_1G; 284 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 285 286 /* 287 * Size (# of entries) of each tx and rx queue. 288 */ 289 static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 290 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 291 292 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 293 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 294 295 /* 296 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 297 */ 298 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 299 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 300 301 /* 302 * Configuration file. 303 */ 304 #define DEFAULT_CF "default" 305 #define FLASH_CF "flash" 306 #define UWIRE_CF "uwire" 307 #define FPGA_CF "fpga" 308 static char t4_cfg_file[32] = DEFAULT_CF; 309 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 310 311 /* 312 * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively). 313 * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them. 314 * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water 315 * mark or when signalled to do so, 0 to never emit PAUSE. 316 */ 317 static int t4_pause_settings = PAUSE_TX | PAUSE_RX; 318 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings); 319 320 /* 321 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 322 * encouraged respectively). 323 */ 324 static unsigned int t4_fw_install = 1; 325 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 326 327 /* 328 * ASIC features that will be used. Disable the ones you don't want so that the 329 * chip resources aren't wasted on features that will not be used. 330 */ 331 static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 332 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 333 334 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 335 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 336 337 static int t4_toecaps_allowed = -1; 338 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 339 340 static int t4_rdmacaps_allowed = 0; 341 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 342 343 static int t4_iscsicaps_allowed = 0; 344 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 345 346 static int t4_fcoecaps_allowed = 0; 347 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 348 349 static int t5_write_combine = 0; 350 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 351 352 static int t4_num_vis = 1; 353 TUNABLE_INT("hw.cxgbe.num_vis", &t4_num_vis); 354 355 /* Functions used by extra VIs to obtain unique MAC addresses for each VI. */ 356 static int vi_mac_funcs[] = { 357 FW_VI_FUNC_OFLD, 358 FW_VI_FUNC_IWARP, 359 FW_VI_FUNC_OPENISCSI, 360 FW_VI_FUNC_OPENFCOE, 361 FW_VI_FUNC_FOISCSI, 362 FW_VI_FUNC_FOFCOE, 363 }; 364 365 struct intrs_and_queues { 366 uint16_t intr_type; /* INTx, MSI, or MSI-X */ 367 uint16_t nirq; /* Total # of vectors */ 368 uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */ 369 uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */ 370 uint16_t ntxq10g; /* # of NIC txq's for each 10G port */ 371 uint16_t nrxq10g; /* # of NIC rxq's for each 10G port */ 372 uint16_t ntxq1g; /* # of NIC txq's for each 1G port */ 373 uint16_t nrxq1g; /* # of NIC rxq's for each 1G port */ 374 uint16_t rsrv_noflowq; /* Flag whether to reserve queue 0 */ 375 #ifdef TCP_OFFLOAD 376 uint16_t nofldtxq10g; /* # of TOE txq's for each 10G port */ 377 uint16_t nofldrxq10g; /* # of TOE rxq's for each 10G port */ 378 uint16_t nofldtxq1g; /* # of TOE txq's for each 1G port */ 379 uint16_t nofldrxq1g; /* # of TOE rxq's for each 1G port */ 380 #endif 381 #ifdef DEV_NETMAP 382 uint16_t nnmtxq10g; /* # of netmap txq's for each 10G port */ 383 uint16_t nnmrxq10g; /* # of netmap rxq's for each 10G port */ 384 uint16_t nnmtxq1g; /* # of netmap txq's for each 1G port */ 385 uint16_t nnmrxq1g; /* # of netmap rxq's for each 1G port */ 386 #endif 387 }; 388 389 struct filter_entry { 390 uint32_t valid:1; /* filter allocated and valid */ 391 uint32_t locked:1; /* filter is administratively locked */ 392 uint32_t pending:1; /* filter action is pending firmware reply */ 393 uint32_t smtidx:8; /* Source MAC Table index for smac */ 394 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 395 396 struct t4_filter_specification fs; 397 }; 398 399 static int map_bars_0_and_4(struct adapter *); 400 static int map_bar_2(struct adapter *); 401 static void setup_memwin(struct adapter *); 402 static int validate_mem_range(struct adapter *, uint32_t, int); 403 static int fwmtype_to_hwmtype(int); 404 static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 405 uint32_t *); 406 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 407 static uint32_t position_memwin(struct adapter *, int, uint32_t); 408 static int cfg_itype_and_nqueues(struct adapter *, int, int, int, 409 struct intrs_and_queues *); 410 static int prep_firmware(struct adapter *); 411 static int partition_resources(struct adapter *, const struct firmware *, 412 const char *); 413 static int get_params__pre_init(struct adapter *); 414 static int get_params__post_init(struct adapter *); 415 static int set_params__post_init(struct adapter *); 416 static void t4_set_desc(struct adapter *); 417 static void build_medialist(struct port_info *, struct ifmedia *); 418 static int cxgbe_init_synchronized(struct vi_info *); 419 static int cxgbe_uninit_synchronized(struct vi_info *); 420 static int setup_intr_handlers(struct adapter *); 421 static void quiesce_txq(struct adapter *, struct sge_txq *); 422 static void quiesce_wrq(struct adapter *, struct sge_wrq *); 423 static void quiesce_iq(struct adapter *, struct sge_iq *); 424 static void quiesce_fl(struct adapter *, struct sge_fl *); 425 static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 426 driver_intr_t *, void *, char *); 427 static int t4_free_irq(struct adapter *, struct irq *); 428 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 429 unsigned int); 430 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 431 static void vi_refresh_stats(struct adapter *, struct vi_info *); 432 static void cxgbe_refresh_stats(struct adapter *, struct port_info *); 433 static void cxgbe_tick(void *); 434 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 435 static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 436 struct mbuf *); 437 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 438 static int fw_msg_not_handled(struct adapter *, const __be64 *); 439 static void t4_sysctls(struct adapter *); 440 static void cxgbe_sysctls(struct port_info *); 441 static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 442 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 443 static int sysctl_btphy(SYSCTL_HANDLER_ARGS); 444 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); 445 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 446 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 447 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 448 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 449 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS); 450 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 451 static int sysctl_temperature(SYSCTL_HANDLER_ARGS); 452 #ifdef SBUF_DRAIN 453 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 454 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 455 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 456 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); 457 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); 458 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 459 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 460 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 461 static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 462 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 463 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 464 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 465 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); 466 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 467 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); 468 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 469 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 470 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 471 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 472 static int sysctl_tids(SYSCTL_HANDLER_ARGS); 473 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 474 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 475 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 476 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 477 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 478 #endif 479 static uint32_t fconf_to_mode(uint32_t); 480 static uint32_t mode_to_fconf(uint32_t); 481 static uint32_t fspec_to_fconf(struct t4_filter_specification *); 482 static int get_filter_mode(struct adapter *, uint32_t *); 483 static int set_filter_mode(struct adapter *, uint32_t); 484 static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 485 static int get_filter(struct adapter *, struct t4_filter *); 486 static int set_filter(struct adapter *, struct t4_filter *); 487 static int del_filter(struct adapter *, struct t4_filter *); 488 static void clear_filter(struct filter_entry *); 489 static int set_filter_wr(struct adapter *, int); 490 static int del_filter_wr(struct adapter *, int); 491 static int get_sge_context(struct adapter *, struct t4_sge_context *); 492 static int load_fw(struct adapter *, struct t4_data *); 493 static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 494 static int read_i2c(struct adapter *, struct t4_i2c_data *); 495 static int set_sched_class(struct adapter *, struct t4_sched_params *); 496 static int set_sched_queue(struct adapter *, struct t4_sched_queue *); 497 #ifdef TCP_OFFLOAD 498 static int toe_capability(struct vi_info *, int); 499 #endif 500 static int mod_event(module_t, int, void *); 501 502 struct { 503 uint16_t device; 504 char *desc; 505 } t4_pciids[] = { 506 {0xa000, "Chelsio Terminator 4 FPGA"}, 507 {0x4400, "Chelsio T440-dbg"}, 508 {0x4401, "Chelsio T420-CR"}, 509 {0x4402, "Chelsio T422-CR"}, 510 {0x4403, "Chelsio T440-CR"}, 511 {0x4404, "Chelsio T420-BCH"}, 512 {0x4405, "Chelsio T440-BCH"}, 513 {0x4406, "Chelsio T440-CH"}, 514 {0x4407, "Chelsio T420-SO"}, 515 {0x4408, "Chelsio T420-CX"}, 516 {0x4409, "Chelsio T420-BT"}, 517 {0x440a, "Chelsio T404-BT"}, 518 {0x440e, "Chelsio T440-LP-CR"}, 519 }, t5_pciids[] = { 520 {0xb000, "Chelsio Terminator 5 FPGA"}, 521 {0x5400, "Chelsio T580-dbg"}, 522 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ 523 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ 524 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ 525 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ 526 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ 527 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ 528 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ 529 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ 530 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 531 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ 532 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ 533 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ 534 {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */ 535 #ifdef notyet 536 {0x5404, "Chelsio T520-BCH"}, 537 {0x5405, "Chelsio T540-BCH"}, 538 {0x5406, "Chelsio T540-CH"}, 539 {0x5408, "Chelsio T520-CX"}, 540 {0x540b, "Chelsio B520-SR"}, 541 {0x540c, "Chelsio B504-BT"}, 542 {0x540f, "Chelsio Amsterdam"}, 543 {0x5413, "Chelsio T580-CHR"}, 544 #endif 545 }; 546 547 #ifdef TCP_OFFLOAD 548 /* 549 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 550 * exactly the same for both rxq and ofld_rxq. 551 */ 552 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 553 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 554 #endif 555 556 /* No easy way to include t4_msg.h before adapter.h so we check this way */ 557 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 558 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 559 560 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); 561 562 static int 563 t4_probe(device_t dev) 564 { 565 int i; 566 uint16_t v = pci_get_vendor(dev); 567 uint16_t d = pci_get_device(dev); 568 uint8_t f = pci_get_function(dev); 569 570 if (v != PCI_VENDOR_ID_CHELSIO) 571 return (ENXIO); 572 573 /* Attach only to PF0 of the FPGA */ 574 if (d == 0xa000 && f != 0) 575 return (ENXIO); 576 577 for (i = 0; i < nitems(t4_pciids); i++) { 578 if (d == t4_pciids[i].device) { 579 device_set_desc(dev, t4_pciids[i].desc); 580 return (BUS_PROBE_DEFAULT); 581 } 582 } 583 584 return (ENXIO); 585 } 586 587 static int 588 t5_probe(device_t dev) 589 { 590 int i; 591 uint16_t v = pci_get_vendor(dev); 592 uint16_t d = pci_get_device(dev); 593 uint8_t f = pci_get_function(dev); 594 595 if (v != PCI_VENDOR_ID_CHELSIO) 596 return (ENXIO); 597 598 /* Attach only to PF0 of the FPGA */ 599 if (d == 0xb000 && f != 0) 600 return (ENXIO); 601 602 for (i = 0; i < nitems(t5_pciids); i++) { 603 if (d == t5_pciids[i].device) { 604 device_set_desc(dev, t5_pciids[i].desc); 605 return (BUS_PROBE_DEFAULT); 606 } 607 } 608 609 return (ENXIO); 610 } 611 612 static void 613 t5_attribute_workaround(device_t dev) 614 { 615 device_t root_port; 616 uint32_t v; 617 618 /* 619 * The T5 chips do not properly echo the No Snoop and Relaxed 620 * Ordering attributes when replying to a TLP from a Root 621 * Port. As a workaround, find the parent Root Port and 622 * disable No Snoop and Relaxed Ordering. Note that this 623 * affects all devices under this root port. 624 */ 625 root_port = pci_find_pcie_root_port(dev); 626 if (root_port == NULL) { 627 device_printf(dev, "Unable to find parent root port\n"); 628 return; 629 } 630 631 v = pcie_adjust_config(root_port, PCIER_DEVICE_CTL, 632 PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE, 0, 2); 633 if ((v & (PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE)) != 634 0) 635 device_printf(dev, "Disabled No Snoop/Relaxed Ordering on %s\n", 636 device_get_nameunit(root_port)); 637 } 638 639 static int 640 t4_attach(device_t dev) 641 { 642 struct adapter *sc; 643 int rc = 0, i, j, n10g, n1g, rqidx, tqidx; 644 struct intrs_and_queues iaq; 645 struct sge *s; 646 #ifdef TCP_OFFLOAD 647 int ofld_rqidx, ofld_tqidx; 648 #endif 649 #ifdef DEV_NETMAP 650 int nm_rqidx, nm_tqidx; 651 #endif 652 int num_vis; 653 654 sc = device_get_softc(dev); 655 sc->dev = dev; 656 TUNABLE_INT_FETCH("hw.cxgbe.debug_flags", &sc->debug_flags); 657 658 if ((pci_get_device(dev) & 0xff00) == 0x5400) 659 t5_attribute_workaround(dev); 660 pci_enable_busmaster(dev); 661 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 662 uint32_t v; 663 664 pci_set_max_read_req(dev, 4096); 665 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 666 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 667 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 668 669 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5); 670 } 671 672 sc->traceq = -1; 673 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); 674 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", 675 device_get_nameunit(dev)); 676 677 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 678 device_get_nameunit(dev)); 679 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 680 sx_xlock(&t4_list_lock); 681 SLIST_INSERT_HEAD(&t4_list, sc, link); 682 sx_xunlock(&t4_list_lock); 683 684 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 685 TAILQ_INIT(&sc->sfl); 686 callout_init_mtx(&sc->sfl_callout, &sc->sfl_lock, 0); 687 688 mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF); 689 690 rc = map_bars_0_and_4(sc); 691 if (rc != 0) 692 goto done; /* error message displayed already */ 693 694 /* 695 * This is the real PF# to which we're attaching. Works from within PCI 696 * passthrough environments too, where pci_get_function() could return a 697 * different PF# depending on the passthrough configuration. We need to 698 * use the real PF# in all our communication with the firmware. 699 */ 700 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 701 sc->mbox = sc->pf; 702 703 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 704 sc->an_handler = an_not_handled; 705 for (i = 0; i < nitems(sc->cpl_handler); i++) 706 sc->cpl_handler[i] = cpl_not_handled; 707 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 708 sc->fw_msg_handler[i] = fw_msg_not_handled; 709 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 710 t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt); 711 t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt); 712 t4_init_sge_cpl_handlers(sc); 713 714 /* Prepare the adapter for operation */ 715 rc = -t4_prep_adapter(sc); 716 if (rc != 0) { 717 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 718 goto done; 719 } 720 721 /* 722 * Do this really early, with the memory windows set up even before the 723 * character device. The userland tool's register i/o and mem read 724 * will work even in "recovery mode". 725 */ 726 setup_memwin(sc); 727 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 728 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 729 device_get_nameunit(dev)); 730 if (sc->cdev == NULL) 731 device_printf(dev, "failed to create nexus char device.\n"); 732 else 733 sc->cdev->si_drv1 = sc; 734 735 /* Go no further if recovery mode has been requested. */ 736 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 737 device_printf(dev, "recovery mode.\n"); 738 goto done; 739 } 740 741 #if defined(__i386__) 742 if ((cpu_feature & CPUID_CX8) == 0) { 743 device_printf(dev, "64 bit atomics not available.\n"); 744 rc = ENOTSUP; 745 goto done; 746 } 747 #endif 748 749 /* Prepare the firmware for operation */ 750 rc = prep_firmware(sc); 751 if (rc != 0) 752 goto done; /* error message displayed already */ 753 754 rc = get_params__post_init(sc); 755 if (rc != 0) 756 goto done; /* error message displayed already */ 757 758 rc = set_params__post_init(sc); 759 if (rc != 0) 760 goto done; /* error message displayed already */ 761 762 rc = map_bar_2(sc); 763 if (rc != 0) 764 goto done; /* error message displayed already */ 765 766 rc = t4_create_dma_tag(sc); 767 if (rc != 0) 768 goto done; /* error message displayed already */ 769 770 /* 771 * Number of VIs to create per-port. The first VI is the 772 * "main" regular VI for the port. The second VI is used for 773 * netmap if present, and any remaining VIs are used for 774 * additional virtual interfaces. 775 * 776 * Limit the number of VIs per port to the number of available 777 * MAC addresses per port. 778 */ 779 if (t4_num_vis >= 1) 780 num_vis = t4_num_vis; 781 else 782 num_vis = 1; 783 #ifdef DEV_NETMAP 784 num_vis++; 785 #endif 786 if (num_vis > nitems(vi_mac_funcs)) { 787 num_vis = nitems(vi_mac_funcs); 788 device_printf(dev, "Number of VIs limited to %d\n", num_vis); 789 } 790 791 /* 792 * First pass over all the ports - allocate VIs and initialize some 793 * basic parameters like mac address, port type, etc. We also figure 794 * out whether a port is 10G or 1G and use that information when 795 * calculating how many interrupts to attempt to allocate. 796 */ 797 n10g = n1g = 0; 798 for_each_port(sc, i) { 799 struct port_info *pi; 800 struct vi_info *vi; 801 802 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 803 sc->port[i] = pi; 804 805 /* These must be set before t4_port_init */ 806 pi->adapter = sc; 807 pi->port_id = i; 808 pi->nvi = num_vis; 809 pi->vi = malloc(sizeof(struct vi_info) * num_vis, M_CXGBE, 810 M_ZERO | M_WAITOK); 811 812 /* 813 * Allocate the "main" VI and initialize parameters 814 * like mac addr. 815 */ 816 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 817 if (rc != 0) { 818 device_printf(dev, "unable to initialize port %d: %d\n", 819 i, rc); 820 free(pi->vi, M_CXGBE); 821 free(pi, M_CXGBE); 822 sc->port[i] = NULL; 823 goto done; 824 } 825 826 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX); 827 pi->link_cfg.requested_fc |= t4_pause_settings; 828 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX); 829 pi->link_cfg.fc |= t4_pause_settings; 830 831 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 832 if (rc != 0) { 833 device_printf(dev, "port %d l1cfg failed: %d\n", i, rc); 834 free(pi->vi, M_CXGBE); 835 free(pi, M_CXGBE); 836 sc->port[i] = NULL; 837 goto done; 838 } 839 840 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 841 device_get_nameunit(dev), i); 842 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 843 sc->chan_map[pi->tx_chan] = i; 844 845 if (is_10G_port(pi) || is_40G_port(pi)) { 846 n10g++; 847 for_each_vi(pi, j, vi) { 848 vi->tmr_idx = t4_tmr_idx_10g; 849 vi->pktc_idx = t4_pktc_idx_10g; 850 } 851 } else { 852 n1g++; 853 for_each_vi(pi, j, vi) { 854 vi->tmr_idx = t4_tmr_idx_1g; 855 vi->pktc_idx = t4_pktc_idx_1g; 856 } 857 } 858 859 pi->linkdnrc = -1; 860 861 for_each_vi(pi, j, vi) { 862 vi->qsize_rxq = t4_qsize_rxq; 863 vi->qsize_txq = t4_qsize_txq; 864 vi->pi = pi; 865 } 866 867 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 868 if (pi->dev == NULL) { 869 device_printf(dev, 870 "failed to add device for port %d.\n", i); 871 rc = ENXIO; 872 goto done; 873 } 874 pi->vi[0].dev = pi->dev; 875 device_set_softc(pi->dev, pi); 876 } 877 878 /* 879 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 880 */ 881 #ifdef DEV_NETMAP 882 num_vis--; 883 #endif 884 rc = cfg_itype_and_nqueues(sc, n10g, n1g, num_vis, &iaq); 885 if (rc != 0) 886 goto done; /* error message displayed already */ 887 888 sc->intr_type = iaq.intr_type; 889 sc->intr_count = iaq.nirq; 890 891 s = &sc->sge; 892 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 893 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 894 if (num_vis > 1) { 895 s->nrxq += (n10g + n1g) * (num_vis - 1); 896 s->ntxq += (n10g + n1g) * (num_vis - 1); 897 } 898 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 899 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 900 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 901 #ifdef TCP_OFFLOAD 902 if (is_offload(sc)) { 903 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 904 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 905 if (num_vis > 1) { 906 s->nofldrxq += (n10g + n1g) * (num_vis - 1); 907 s->nofldtxq += (n10g + n1g) * (num_vis - 1); 908 } 909 s->neq += s->nofldtxq + s->nofldrxq; 910 s->niq += s->nofldrxq; 911 912 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 913 M_CXGBE, M_ZERO | M_WAITOK); 914 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 915 M_CXGBE, M_ZERO | M_WAITOK); 916 } 917 #endif 918 #ifdef DEV_NETMAP 919 s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g; 920 s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g; 921 s->neq += s->nnmtxq + s->nnmrxq; 922 s->niq += s->nnmrxq; 923 924 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq), 925 M_CXGBE, M_ZERO | M_WAITOK); 926 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq), 927 M_CXGBE, M_ZERO | M_WAITOK); 928 #endif 929 930 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 931 M_ZERO | M_WAITOK); 932 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 933 M_ZERO | M_WAITOK); 934 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 935 M_ZERO | M_WAITOK); 936 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 937 M_ZERO | M_WAITOK); 938 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 939 M_ZERO | M_WAITOK); 940 941 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 942 M_ZERO | M_WAITOK); 943 944 t4_init_l2t(sc, M_WAITOK); 945 946 /* 947 * Second pass over the ports. This time we know the number of rx and 948 * tx queues that each port should get. 949 */ 950 rqidx = tqidx = 0; 951 #ifdef TCP_OFFLOAD 952 ofld_rqidx = ofld_tqidx = 0; 953 #endif 954 #ifdef DEV_NETMAP 955 nm_rqidx = nm_tqidx = 0; 956 #endif 957 for_each_port(sc, i) { 958 struct port_info *pi = sc->port[i]; 959 struct vi_info *vi; 960 961 if (pi == NULL) 962 continue; 963 964 for_each_vi(pi, j, vi) { 965 #ifdef DEV_NETMAP 966 if (j == 1) { 967 vi->flags |= VI_NETMAP | INTR_RXQ; 968 vi->first_rxq = nm_rqidx; 969 vi->first_txq = nm_tqidx; 970 if (is_10G_port(pi) || is_40G_port(pi)) { 971 vi->nrxq = iaq.nnmrxq10g; 972 vi->ntxq = iaq.nnmtxq10g; 973 } else { 974 vi->nrxq = iaq.nnmrxq1g; 975 vi->ntxq = iaq.nnmtxq1g; 976 } 977 nm_rqidx += vi->nrxq; 978 nm_tqidx += vi->ntxq; 979 continue; 980 } 981 #endif 982 983 vi->first_rxq = rqidx; 984 vi->first_txq = tqidx; 985 if (is_10G_port(pi) || is_40G_port(pi)) { 986 vi->flags |= iaq.intr_flags_10g & INTR_RXQ; 987 vi->nrxq = j == 0 ? iaq.nrxq10g : 1; 988 vi->ntxq = j == 0 ? iaq.ntxq10g : 1; 989 } else { 990 vi->flags |= iaq.intr_flags_1g & INTR_RXQ; 991 vi->nrxq = j == 0 ? iaq.nrxq1g : 1; 992 vi->ntxq = j == 0 ? iaq.ntxq1g : 1; 993 } 994 995 if (vi->ntxq > 1) 996 vi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0; 997 else 998 vi->rsrv_noflowq = 0; 999 1000 rqidx += vi->nrxq; 1001 tqidx += vi->ntxq; 1002 1003 #ifdef TCP_OFFLOAD 1004 if (!is_offload(sc)) 1005 continue; 1006 vi->first_ofld_rxq = ofld_rqidx; 1007 vi->first_ofld_txq = ofld_tqidx; 1008 if (is_10G_port(pi) || is_40G_port(pi)) { 1009 vi->flags |= iaq.intr_flags_10g & INTR_OFLD_RXQ; 1010 vi->nofldrxq = j == 0 ? iaq.nofldrxq10g : 1; 1011 vi->nofldtxq = j == 0 ? iaq.nofldtxq10g : 1; 1012 } else { 1013 vi->flags |= iaq.intr_flags_1g & INTR_OFLD_RXQ; 1014 vi->nofldrxq = j == 0 ? iaq.nofldrxq1g : 1; 1015 vi->nofldtxq = j == 0 ? iaq.nofldtxq1g : 1; 1016 } 1017 ofld_rqidx += vi->nofldrxq; 1018 ofld_tqidx += vi->nofldtxq; 1019 #endif 1020 } 1021 } 1022 1023 rc = setup_intr_handlers(sc); 1024 if (rc != 0) { 1025 device_printf(dev, 1026 "failed to setup interrupt handlers: %d\n", rc); 1027 goto done; 1028 } 1029 1030 rc = bus_generic_attach(dev); 1031 if (rc != 0) { 1032 device_printf(dev, 1033 "failed to attach all child ports: %d\n", rc); 1034 goto done; 1035 } 1036 1037 device_printf(dev, 1038 "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 1039 sc->params.pci.speed, sc->params.pci.width, sc->params.nports, 1040 sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" : 1041 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 1042 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 1043 1044 t4_set_desc(sc); 1045 1046 done: 1047 if (rc != 0 && sc->cdev) { 1048 /* cdev was created and so cxgbetool works; recover that way. */ 1049 device_printf(dev, 1050 "error during attach, adapter is now in recovery mode.\n"); 1051 rc = 0; 1052 } 1053 1054 if (rc != 0) 1055 t4_detach(dev); 1056 else 1057 t4_sysctls(sc); 1058 1059 return (rc); 1060 } 1061 1062 /* 1063 * Idempotent 1064 */ 1065 static int 1066 t4_detach(device_t dev) 1067 { 1068 struct adapter *sc; 1069 struct port_info *pi; 1070 int i, rc; 1071 1072 sc = device_get_softc(dev); 1073 1074 if (sc->flags & FULL_INIT_DONE) 1075 t4_intr_disable(sc); 1076 1077 if (sc->cdev) { 1078 destroy_dev(sc->cdev); 1079 sc->cdev = NULL; 1080 } 1081 1082 rc = bus_generic_detach(dev); 1083 if (rc) { 1084 device_printf(dev, 1085 "failed to detach child devices: %d\n", rc); 1086 return (rc); 1087 } 1088 1089 for (i = 0; i < sc->intr_count; i++) 1090 t4_free_irq(sc, &sc->irq[i]); 1091 1092 for (i = 0; i < MAX_NPORTS; i++) { 1093 pi = sc->port[i]; 1094 if (pi) { 1095 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->vi[0].viid); 1096 if (pi->dev) 1097 device_delete_child(dev, pi->dev); 1098 1099 mtx_destroy(&pi->pi_lock); 1100 free(pi->vi, M_CXGBE); 1101 free(pi, M_CXGBE); 1102 } 1103 } 1104 1105 if (sc->flags & FULL_INIT_DONE) 1106 adapter_full_uninit(sc); 1107 1108 if (sc->flags & FW_OK) 1109 t4_fw_bye(sc, sc->mbox); 1110 1111 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 1112 pci_release_msi(dev); 1113 1114 if (sc->regs_res) 1115 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 1116 sc->regs_res); 1117 1118 if (sc->udbs_res) 1119 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 1120 sc->udbs_res); 1121 1122 if (sc->msix_res) 1123 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 1124 sc->msix_res); 1125 1126 if (sc->l2t) 1127 t4_free_l2t(sc->l2t); 1128 1129 #ifdef TCP_OFFLOAD 1130 free(sc->sge.ofld_rxq, M_CXGBE); 1131 free(sc->sge.ofld_txq, M_CXGBE); 1132 #endif 1133 #ifdef DEV_NETMAP 1134 free(sc->sge.nm_rxq, M_CXGBE); 1135 free(sc->sge.nm_txq, M_CXGBE); 1136 #endif 1137 free(sc->irq, M_CXGBE); 1138 free(sc->sge.rxq, M_CXGBE); 1139 free(sc->sge.txq, M_CXGBE); 1140 free(sc->sge.ctrlq, M_CXGBE); 1141 free(sc->sge.iqmap, M_CXGBE); 1142 free(sc->sge.eqmap, M_CXGBE); 1143 free(sc->tids.ftid_tab, M_CXGBE); 1144 t4_destroy_dma_tag(sc); 1145 if (mtx_initialized(&sc->sc_lock)) { 1146 sx_xlock(&t4_list_lock); 1147 SLIST_REMOVE(&t4_list, sc, adapter, link); 1148 sx_xunlock(&t4_list_lock); 1149 mtx_destroy(&sc->sc_lock); 1150 } 1151 1152 callout_drain(&sc->sfl_callout); 1153 if (mtx_initialized(&sc->tids.ftid_lock)) 1154 mtx_destroy(&sc->tids.ftid_lock); 1155 if (mtx_initialized(&sc->sfl_lock)) 1156 mtx_destroy(&sc->sfl_lock); 1157 if (mtx_initialized(&sc->ifp_lock)) 1158 mtx_destroy(&sc->ifp_lock); 1159 if (mtx_initialized(&sc->regwin_lock)) 1160 mtx_destroy(&sc->regwin_lock); 1161 1162 bzero(sc, sizeof(*sc)); 1163 1164 return (0); 1165 } 1166 1167 static int 1168 cxgbe_probe(device_t dev) 1169 { 1170 char buf[128]; 1171 struct port_info *pi = device_get_softc(dev); 1172 1173 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 1174 device_set_desc_copy(dev, buf); 1175 1176 return (BUS_PROBE_DEFAULT); 1177 } 1178 1179 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 1180 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 1181 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS) 1182 #define T4_CAP_ENABLE (T4_CAP) 1183 1184 static int 1185 cxgbe_vi_attach(device_t dev, struct vi_info *vi) 1186 { 1187 struct ifnet *ifp; 1188 struct sbuf *sb; 1189 1190 vi->xact_addr_filt = -1; 1191 callout_init(&vi->tick, 1); 1192 1193 /* Allocate an ifnet and set it up */ 1194 ifp = if_alloc(IFT_ETHER); 1195 if (ifp == NULL) { 1196 device_printf(dev, "Cannot allocate ifnet\n"); 1197 return (ENOMEM); 1198 } 1199 vi->ifp = ifp; 1200 ifp->if_softc = vi; 1201 1202 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1203 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1204 1205 ifp->if_init = cxgbe_init; 1206 ifp->if_ioctl = cxgbe_ioctl; 1207 ifp->if_transmit = cxgbe_transmit; 1208 ifp->if_qflush = cxgbe_qflush; 1209 ifp->if_get_counter = cxgbe_get_counter; 1210 1211 ifp->if_capabilities = T4_CAP; 1212 #ifdef TCP_OFFLOAD 1213 if (vi->nofldrxq != 0) 1214 ifp->if_capabilities |= IFCAP_TOE; 1215 #endif 1216 ifp->if_capenable = T4_CAP_ENABLE; 1217 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1218 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1219 1220 ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1221 ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS; 1222 ifp->if_hw_tsomaxsegsize = 65536; 1223 1224 /* Initialize ifmedia for this VI */ 1225 ifmedia_init(&vi->media, IFM_IMASK, cxgbe_media_change, 1226 cxgbe_media_status); 1227 build_medialist(vi->pi, &vi->media); 1228 1229 vi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 1230 EVENTHANDLER_PRI_ANY); 1231 1232 ether_ifattach(ifp, vi->hw_addr); 1233 1234 sb = sbuf_new_auto(); 1235 sbuf_printf(sb, "%d txq, %d rxq (NIC)", vi->ntxq, vi->nrxq); 1236 #ifdef TCP_OFFLOAD 1237 if (ifp->if_capabilities & IFCAP_TOE) 1238 sbuf_printf(sb, "; %d txq, %d rxq (TOE)", 1239 vi->nofldtxq, vi->nofldrxq); 1240 #endif 1241 sbuf_finish(sb); 1242 device_printf(dev, "%s\n", sbuf_data(sb)); 1243 sbuf_delete(sb); 1244 1245 vi_sysctls(vi); 1246 1247 return (0); 1248 } 1249 1250 static int 1251 cxgbe_attach(device_t dev) 1252 { 1253 struct port_info *pi = device_get_softc(dev); 1254 struct vi_info *vi; 1255 int i, rc; 1256 1257 callout_init_mtx(&pi->tick, &pi->pi_lock, 0); 1258 1259 rc = cxgbe_vi_attach(dev, &pi->vi[0]); 1260 if (rc) 1261 return (rc); 1262 1263 for_each_vi(pi, i, vi) { 1264 if (i == 0) 1265 continue; 1266 #ifdef DEV_NETMAP 1267 if (vi->flags & VI_NETMAP) { 1268 /* 1269 * media handled here to keep 1270 * implementation private to this file 1271 */ 1272 ifmedia_init(&vi->media, IFM_IMASK, cxgbe_media_change, 1273 cxgbe_media_status); 1274 build_medialist(pi, &vi->media); 1275 vi->dev = device_add_child(dev, is_t4(pi->adapter) ? 1276 "ncxgbe" : "ncxl", device_get_unit(dev)); 1277 } else 1278 #endif 1279 vi->dev = device_add_child(dev, is_t4(pi->adapter) ? 1280 "vcxgbe" : "vcxl", -1); 1281 if (vi->dev == NULL) { 1282 device_printf(dev, "failed to add VI %d\n", i); 1283 continue; 1284 } 1285 device_set_softc(vi->dev, vi); 1286 } 1287 1288 cxgbe_sysctls(pi); 1289 1290 bus_generic_attach(dev); 1291 1292 return (0); 1293 } 1294 1295 static void 1296 cxgbe_vi_detach(struct vi_info *vi) 1297 { 1298 struct ifnet *ifp = vi->ifp; 1299 1300 ether_ifdetach(ifp); 1301 1302 if (vi->vlan_c) 1303 EVENTHANDLER_DEREGISTER(vlan_config, vi->vlan_c); 1304 1305 /* Let detach proceed even if these fail. */ 1306 cxgbe_uninit_synchronized(vi); 1307 callout_drain(&vi->tick); 1308 vi_full_uninit(vi); 1309 1310 ifmedia_removeall(&vi->media); 1311 if_free(vi->ifp); 1312 vi->ifp = NULL; 1313 } 1314 1315 static int 1316 cxgbe_detach(device_t dev) 1317 { 1318 struct port_info *pi = device_get_softc(dev); 1319 struct adapter *sc = pi->adapter; 1320 int rc; 1321 1322 /* Detach the extra VIs first. */ 1323 rc = bus_generic_detach(dev); 1324 if (rc) 1325 return (rc); 1326 device_delete_children(dev); 1327 1328 doom_vi(sc, &pi->vi[0]); 1329 1330 if (pi->flags & HAS_TRACEQ) { 1331 sc->traceq = -1; /* cloner should not create ifnet */ 1332 t4_tracer_port_detach(sc); 1333 } 1334 1335 cxgbe_vi_detach(&pi->vi[0]); 1336 callout_drain(&pi->tick); 1337 1338 end_synchronized_op(sc, 0); 1339 1340 return (0); 1341 } 1342 1343 static void 1344 cxgbe_init(void *arg) 1345 { 1346 struct vi_info *vi = arg; 1347 struct adapter *sc = vi->pi->adapter; 1348 1349 if (begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4init") != 0) 1350 return; 1351 cxgbe_init_synchronized(vi); 1352 end_synchronized_op(sc, 0); 1353 } 1354 1355 static int 1356 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1357 { 1358 int rc = 0, mtu, flags, can_sleep; 1359 struct vi_info *vi = ifp->if_softc; 1360 struct adapter *sc = vi->pi->adapter; 1361 struct ifreq *ifr = (struct ifreq *)data; 1362 uint32_t mask; 1363 1364 switch (cmd) { 1365 case SIOCSIFMTU: 1366 mtu = ifr->ifr_mtu; 1367 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1368 return (EINVAL); 1369 1370 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4mtu"); 1371 if (rc) 1372 return (rc); 1373 ifp->if_mtu = mtu; 1374 if (vi->flags & VI_INIT_DONE) { 1375 t4_update_fl_bufsize(ifp); 1376 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1377 rc = update_mac_settings(ifp, XGMAC_MTU); 1378 } 1379 end_synchronized_op(sc, 0); 1380 break; 1381 1382 case SIOCSIFFLAGS: 1383 can_sleep = 0; 1384 redo_sifflags: 1385 rc = begin_synchronized_op(sc, vi, 1386 can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg"); 1387 if (rc) 1388 return (rc); 1389 1390 if (ifp->if_flags & IFF_UP) { 1391 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1392 flags = vi->if_flags; 1393 if ((ifp->if_flags ^ flags) & 1394 (IFF_PROMISC | IFF_ALLMULTI)) { 1395 if (can_sleep == 1) { 1396 end_synchronized_op(sc, 0); 1397 can_sleep = 0; 1398 goto redo_sifflags; 1399 } 1400 rc = update_mac_settings(ifp, 1401 XGMAC_PROMISC | XGMAC_ALLMULTI); 1402 } 1403 } else { 1404 if (can_sleep == 0) { 1405 end_synchronized_op(sc, LOCK_HELD); 1406 can_sleep = 1; 1407 goto redo_sifflags; 1408 } 1409 rc = cxgbe_init_synchronized(vi); 1410 } 1411 vi->if_flags = ifp->if_flags; 1412 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1413 if (can_sleep == 0) { 1414 end_synchronized_op(sc, LOCK_HELD); 1415 can_sleep = 1; 1416 goto redo_sifflags; 1417 } 1418 rc = cxgbe_uninit_synchronized(vi); 1419 } 1420 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD); 1421 break; 1422 1423 case SIOCADDMULTI: 1424 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1425 rc = begin_synchronized_op(sc, vi, HOLD_LOCK, "t4multi"); 1426 if (rc) 1427 return (rc); 1428 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1429 rc = update_mac_settings(ifp, XGMAC_MCADDRS); 1430 end_synchronized_op(sc, LOCK_HELD); 1431 break; 1432 1433 case SIOCSIFCAP: 1434 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4cap"); 1435 if (rc) 1436 return (rc); 1437 1438 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1439 if (mask & IFCAP_TXCSUM) { 1440 ifp->if_capenable ^= IFCAP_TXCSUM; 1441 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1442 1443 if (IFCAP_TSO4 & ifp->if_capenable && 1444 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1445 ifp->if_capenable &= ~IFCAP_TSO4; 1446 if_printf(ifp, 1447 "tso4 disabled due to -txcsum.\n"); 1448 } 1449 } 1450 if (mask & IFCAP_TXCSUM_IPV6) { 1451 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1452 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1453 1454 if (IFCAP_TSO6 & ifp->if_capenable && 1455 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1456 ifp->if_capenable &= ~IFCAP_TSO6; 1457 if_printf(ifp, 1458 "tso6 disabled due to -txcsum6.\n"); 1459 } 1460 } 1461 if (mask & IFCAP_RXCSUM) 1462 ifp->if_capenable ^= IFCAP_RXCSUM; 1463 if (mask & IFCAP_RXCSUM_IPV6) 1464 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1465 1466 /* 1467 * Note that we leave CSUM_TSO alone (it is always set). The 1468 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1469 * sending a TSO request our way, so it's sufficient to toggle 1470 * IFCAP_TSOx only. 1471 */ 1472 if (mask & IFCAP_TSO4) { 1473 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1474 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1475 if_printf(ifp, "enable txcsum first.\n"); 1476 rc = EAGAIN; 1477 goto fail; 1478 } 1479 ifp->if_capenable ^= IFCAP_TSO4; 1480 } 1481 if (mask & IFCAP_TSO6) { 1482 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1483 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1484 if_printf(ifp, "enable txcsum6 first.\n"); 1485 rc = EAGAIN; 1486 goto fail; 1487 } 1488 ifp->if_capenable ^= IFCAP_TSO6; 1489 } 1490 if (mask & IFCAP_LRO) { 1491 #if defined(INET) || defined(INET6) 1492 int i; 1493 struct sge_rxq *rxq; 1494 1495 ifp->if_capenable ^= IFCAP_LRO; 1496 for_each_rxq(vi, i, rxq) { 1497 if (ifp->if_capenable & IFCAP_LRO) 1498 rxq->iq.flags |= IQ_LRO_ENABLED; 1499 else 1500 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1501 } 1502 #endif 1503 } 1504 #ifdef TCP_OFFLOAD 1505 if (mask & IFCAP_TOE) { 1506 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1507 1508 rc = toe_capability(vi, enable); 1509 if (rc != 0) 1510 goto fail; 1511 1512 ifp->if_capenable ^= mask; 1513 } 1514 #endif 1515 if (mask & IFCAP_VLAN_HWTAGGING) { 1516 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1517 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1518 rc = update_mac_settings(ifp, XGMAC_VLANEX); 1519 } 1520 if (mask & IFCAP_VLAN_MTU) { 1521 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1522 1523 /* Need to find out how to disable auto-mtu-inflation */ 1524 } 1525 if (mask & IFCAP_VLAN_HWTSO) 1526 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1527 if (mask & IFCAP_VLAN_HWCSUM) 1528 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1529 1530 #ifdef VLAN_CAPABILITIES 1531 VLAN_CAPABILITIES(ifp); 1532 #endif 1533 fail: 1534 end_synchronized_op(sc, 0); 1535 break; 1536 1537 case SIOCSIFMEDIA: 1538 case SIOCGIFMEDIA: 1539 ifmedia_ioctl(ifp, ifr, &vi->media, cmd); 1540 break; 1541 1542 case SIOCGI2C: { 1543 struct ifi2creq i2c; 1544 1545 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); 1546 if (rc != 0) 1547 break; 1548 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) { 1549 rc = EPERM; 1550 break; 1551 } 1552 if (i2c.len > sizeof(i2c.data)) { 1553 rc = EINVAL; 1554 break; 1555 } 1556 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4i2c"); 1557 if (rc) 1558 return (rc); 1559 rc = -t4_i2c_rd(sc, sc->mbox, vi->pi->port_id, i2c.dev_addr, 1560 i2c.offset, i2c.len, &i2c.data[0]); 1561 end_synchronized_op(sc, 0); 1562 if (rc == 0) 1563 rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); 1564 break; 1565 } 1566 1567 default: 1568 rc = ether_ioctl(ifp, cmd, data); 1569 } 1570 1571 return (rc); 1572 } 1573 1574 static int 1575 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1576 { 1577 struct vi_info *vi = ifp->if_softc; 1578 struct port_info *pi = vi->pi; 1579 struct adapter *sc = pi->adapter; 1580 struct sge_txq *txq; 1581 void *items[1]; 1582 int rc; 1583 1584 M_ASSERTPKTHDR(m); 1585 MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */ 1586 1587 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1588 m_freem(m); 1589 return (ENETDOWN); 1590 } 1591 1592 rc = parse_pkt(&m); 1593 if (__predict_false(rc != 0)) { 1594 MPASS(m == NULL); /* was freed already */ 1595 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */ 1596 return (rc); 1597 } 1598 1599 /* Select a txq. */ 1600 txq = &sc->sge.txq[vi->first_txq]; 1601 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 1602 txq += ((m->m_pkthdr.flowid % (vi->ntxq - vi->rsrv_noflowq)) + 1603 vi->rsrv_noflowq); 1604 1605 items[0] = m; 1606 rc = mp_ring_enqueue(txq->r, items, 1, 4096); 1607 if (__predict_false(rc != 0)) 1608 m_freem(m); 1609 1610 return (rc); 1611 } 1612 1613 static void 1614 cxgbe_qflush(struct ifnet *ifp) 1615 { 1616 struct vi_info *vi = ifp->if_softc; 1617 struct sge_txq *txq; 1618 int i; 1619 1620 /* queues do not exist if !VI_INIT_DONE. */ 1621 if (vi->flags & VI_INIT_DONE) { 1622 for_each_txq(vi, i, txq) { 1623 TXQ_LOCK(txq); 1624 txq->eq.flags &= ~EQ_ENABLED; 1625 TXQ_UNLOCK(txq); 1626 while (!mp_ring_is_idle(txq->r)) { 1627 mp_ring_check_drainage(txq->r, 0); 1628 pause("qflush", 1); 1629 } 1630 } 1631 } 1632 if_qflush(ifp); 1633 } 1634 1635 static uint64_t 1636 vi_get_counter(struct ifnet *ifp, ift_counter c) 1637 { 1638 struct vi_info *vi = ifp->if_softc; 1639 struct fw_vi_stats_vf *s = &vi->stats; 1640 1641 vi_refresh_stats(vi->pi->adapter, vi); 1642 1643 switch (c) { 1644 case IFCOUNTER_IPACKETS: 1645 return (s->rx_bcast_frames + s->rx_mcast_frames + 1646 s->rx_ucast_frames); 1647 case IFCOUNTER_IERRORS: 1648 return (s->rx_err_frames); 1649 case IFCOUNTER_OPACKETS: 1650 return (s->tx_bcast_frames + s->tx_mcast_frames + 1651 s->tx_ucast_frames + s->tx_offload_frames); 1652 case IFCOUNTER_OERRORS: 1653 return (s->tx_drop_frames); 1654 case IFCOUNTER_IBYTES: 1655 return (s->rx_bcast_bytes + s->rx_mcast_bytes + 1656 s->rx_ucast_bytes); 1657 case IFCOUNTER_OBYTES: 1658 return (s->tx_bcast_bytes + s->tx_mcast_bytes + 1659 s->tx_ucast_bytes + s->tx_offload_bytes); 1660 case IFCOUNTER_IMCASTS: 1661 return (s->rx_mcast_frames); 1662 case IFCOUNTER_OMCASTS: 1663 return (s->tx_mcast_frames); 1664 case IFCOUNTER_OQDROPS: { 1665 uint64_t drops; 1666 1667 drops = 0; 1668 if ((vi->flags & (VI_INIT_DONE | VI_NETMAP)) == VI_INIT_DONE) { 1669 int i; 1670 struct sge_txq *txq; 1671 1672 for_each_txq(vi, i, txq) 1673 drops += counter_u64_fetch(txq->r->drops); 1674 } 1675 1676 return (drops); 1677 1678 } 1679 1680 default: 1681 return (if_get_counter_default(ifp, c)); 1682 } 1683 } 1684 1685 uint64_t 1686 cxgbe_get_counter(struct ifnet *ifp, ift_counter c) 1687 { 1688 struct vi_info *vi = ifp->if_softc; 1689 struct port_info *pi = vi->pi; 1690 struct adapter *sc = pi->adapter; 1691 struct port_stats *s = &pi->stats; 1692 1693 if (pi->nvi > 1) 1694 return (vi_get_counter(ifp, c)); 1695 1696 cxgbe_refresh_stats(sc, pi); 1697 1698 switch (c) { 1699 case IFCOUNTER_IPACKETS: 1700 return (s->rx_frames - s->rx_pause); 1701 1702 case IFCOUNTER_IERRORS: 1703 return (s->rx_jabber + s->rx_runt + s->rx_too_long + 1704 s->rx_fcs_err + s->rx_len_err); 1705 1706 case IFCOUNTER_OPACKETS: 1707 return (s->tx_frames - s->tx_pause); 1708 1709 case IFCOUNTER_OERRORS: 1710 return (s->tx_error_frames); 1711 1712 case IFCOUNTER_IBYTES: 1713 return (s->rx_octets - s->rx_pause * 64); 1714 1715 case IFCOUNTER_OBYTES: 1716 return (s->tx_octets - s->tx_pause * 64); 1717 1718 case IFCOUNTER_IMCASTS: 1719 return (s->rx_mcast_frames - s->rx_pause); 1720 1721 case IFCOUNTER_OMCASTS: 1722 return (s->tx_mcast_frames - s->tx_pause); 1723 1724 case IFCOUNTER_IQDROPS: 1725 return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 1726 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 1727 s->rx_trunc3 + pi->tnl_cong_drops); 1728 1729 case IFCOUNTER_OQDROPS: { 1730 uint64_t drops; 1731 1732 drops = s->tx_drop; 1733 if (vi->flags & VI_INIT_DONE) { 1734 int i; 1735 struct sge_txq *txq; 1736 1737 for_each_txq(vi, i, txq) 1738 drops += counter_u64_fetch(txq->r->drops); 1739 } 1740 1741 return (drops); 1742 1743 } 1744 1745 default: 1746 return (if_get_counter_default(ifp, c)); 1747 } 1748 } 1749 1750 static int 1751 cxgbe_media_change(struct ifnet *ifp) 1752 { 1753 struct vi_info *vi = ifp->if_softc; 1754 1755 device_printf(vi->dev, "%s unimplemented.\n", __func__); 1756 1757 return (EOPNOTSUPP); 1758 } 1759 1760 static void 1761 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1762 { 1763 struct vi_info *vi = ifp->if_softc; 1764 struct port_info *pi = vi->pi; 1765 struct ifmedia_entry *cur; 1766 int speed = pi->link_cfg.speed; 1767 1768 cur = vi->media.ifm_cur; 1769 1770 ifmr->ifm_status = IFM_AVALID; 1771 if (!pi->link_cfg.link_ok) 1772 return; 1773 1774 ifmr->ifm_status |= IFM_ACTIVE; 1775 1776 /* active and current will differ iff current media is autoselect. */ 1777 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1778 return; 1779 1780 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1781 if (speed == SPEED_10000) 1782 ifmr->ifm_active |= IFM_10G_T; 1783 else if (speed == SPEED_1000) 1784 ifmr->ifm_active |= IFM_1000_T; 1785 else if (speed == SPEED_100) 1786 ifmr->ifm_active |= IFM_100_TX; 1787 else if (speed == SPEED_10) 1788 ifmr->ifm_active |= IFM_10_T; 1789 else 1790 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1791 speed)); 1792 } 1793 1794 static int 1795 vcxgbe_probe(device_t dev) 1796 { 1797 char buf[128]; 1798 struct vi_info *vi = device_get_softc(dev); 1799 1800 snprintf(buf, sizeof(buf), "port %d vi %td", vi->pi->port_id, 1801 vi - vi->pi->vi); 1802 device_set_desc_copy(dev, buf); 1803 1804 return (BUS_PROBE_DEFAULT); 1805 } 1806 1807 static int 1808 vcxgbe_attach(device_t dev) 1809 { 1810 struct vi_info *vi; 1811 struct port_info *pi; 1812 struct adapter *sc; 1813 int func, index, rc; 1814 u32 param, val; 1815 1816 vi = device_get_softc(dev); 1817 pi = vi->pi; 1818 sc = pi->adapter; 1819 1820 index = vi - pi->vi; 1821 KASSERT(index < nitems(vi_mac_funcs), 1822 ("%s: VI %s doesn't have a MAC func", __func__, 1823 device_get_nameunit(dev))); 1824 func = vi_mac_funcs[index]; 1825 rc = t4_alloc_vi_func(sc, sc->mbox, pi->tx_chan, sc->pf, 0, 1, 1826 vi->hw_addr, &vi->rss_size, func, 0); 1827 if (rc < 0) { 1828 device_printf(dev, "Failed to allocate virtual interface " 1829 "for port %d: %d\n", pi->port_id, -rc); 1830 return (-rc); 1831 } 1832 vi->viid = rc; 1833 1834 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 1835 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_RSSINFO) | 1836 V_FW_PARAMS_PARAM_YZ(vi->viid); 1837 rc = t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 1838 if (rc) 1839 vi->rss_base = 0xffff; 1840 else { 1841 /* MPASS((val >> 16) == rss_size); */ 1842 vi->rss_base = val & 0xffff; 1843 } 1844 1845 rc = cxgbe_vi_attach(dev, vi); 1846 if (rc) { 1847 t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid); 1848 return (rc); 1849 } 1850 return (0); 1851 } 1852 1853 static int 1854 vcxgbe_detach(device_t dev) 1855 { 1856 struct vi_info *vi; 1857 struct adapter *sc; 1858 1859 vi = device_get_softc(dev); 1860 sc = vi->pi->adapter; 1861 1862 doom_vi(sc, vi); 1863 1864 cxgbe_vi_detach(vi); 1865 t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid); 1866 1867 end_synchronized_op(sc, 0); 1868 1869 return (0); 1870 } 1871 1872 void 1873 t4_fatal_err(struct adapter *sc) 1874 { 1875 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1876 t4_intr_disable(sc); 1877 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1878 device_get_nameunit(sc->dev)); 1879 } 1880 1881 static int 1882 map_bars_0_and_4(struct adapter *sc) 1883 { 1884 sc->regs_rid = PCIR_BAR(0); 1885 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1886 &sc->regs_rid, RF_ACTIVE); 1887 if (sc->regs_res == NULL) { 1888 device_printf(sc->dev, "cannot map registers.\n"); 1889 return (ENXIO); 1890 } 1891 sc->bt = rman_get_bustag(sc->regs_res); 1892 sc->bh = rman_get_bushandle(sc->regs_res); 1893 sc->mmio_len = rman_get_size(sc->regs_res); 1894 setbit(&sc->doorbells, DOORBELL_KDB); 1895 1896 sc->msix_rid = PCIR_BAR(4); 1897 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1898 &sc->msix_rid, RF_ACTIVE); 1899 if (sc->msix_res == NULL) { 1900 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1901 return (ENXIO); 1902 } 1903 1904 return (0); 1905 } 1906 1907 static int 1908 map_bar_2(struct adapter *sc) 1909 { 1910 1911 /* 1912 * T4: only iWARP driver uses the userspace doorbells. There is no need 1913 * to map it if RDMA is disabled. 1914 */ 1915 if (is_t4(sc) && sc->rdmacaps == 0) 1916 return (0); 1917 1918 sc->udbs_rid = PCIR_BAR(2); 1919 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1920 &sc->udbs_rid, RF_ACTIVE); 1921 if (sc->udbs_res == NULL) { 1922 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1923 return (ENXIO); 1924 } 1925 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1926 1927 if (is_t5(sc)) { 1928 setbit(&sc->doorbells, DOORBELL_UDB); 1929 #if defined(__i386__) || defined(__amd64__) 1930 if (t5_write_combine) { 1931 int rc; 1932 1933 /* 1934 * Enable write combining on BAR2. This is the 1935 * userspace doorbell BAR and is split into 128B 1936 * (UDBS_SEG_SIZE) doorbell regions, each associated 1937 * with an egress queue. The first 64B has the doorbell 1938 * and the second 64B can be used to submit a tx work 1939 * request with an implicit doorbell. 1940 */ 1941 1942 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1943 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1944 if (rc == 0) { 1945 clrbit(&sc->doorbells, DOORBELL_UDB); 1946 setbit(&sc->doorbells, DOORBELL_WCWR); 1947 setbit(&sc->doorbells, DOORBELL_UDBWC); 1948 } else { 1949 device_printf(sc->dev, 1950 "couldn't enable write combining: %d\n", 1951 rc); 1952 } 1953 1954 t4_write_reg(sc, A_SGE_STAT_CFG, 1955 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1956 } 1957 #endif 1958 } 1959 1960 return (0); 1961 } 1962 1963 static const struct memwin t4_memwin[] = { 1964 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1965 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1966 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1967 }; 1968 1969 static const struct memwin t5_memwin[] = { 1970 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1971 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1972 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1973 }; 1974 1975 static void 1976 setup_memwin(struct adapter *sc) 1977 { 1978 const struct memwin *mw; 1979 int i, n; 1980 uint32_t bar0; 1981 1982 if (is_t4(sc)) { 1983 /* 1984 * Read low 32b of bar0 indirectly via the hardware backdoor 1985 * mechanism. Works from within PCI passthrough environments 1986 * too, where rman_get_start() can return a different value. We 1987 * need to program the T4 memory window decoders with the actual 1988 * addresses that will be coming across the PCIe link. 1989 */ 1990 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1991 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1992 1993 mw = &t4_memwin[0]; 1994 n = nitems(t4_memwin); 1995 } else { 1996 /* T5 uses the relative offset inside the PCIe BAR */ 1997 bar0 = 0; 1998 1999 mw = &t5_memwin[0]; 2000 n = nitems(t5_memwin); 2001 } 2002 2003 for (i = 0; i < n; i++, mw++) { 2004 t4_write_reg(sc, 2005 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 2006 (mw->base + bar0) | V_BIR(0) | 2007 V_WINDOW(ilog2(mw->aperture) - 10)); 2008 } 2009 2010 /* flush */ 2011 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 2012 } 2013 2014 /* 2015 * Verify that the memory range specified by the addr/len pair is valid and lies 2016 * entirely within a single region (EDCx or MCx). 2017 */ 2018 static int 2019 validate_mem_range(struct adapter *sc, uint32_t addr, int len) 2020 { 2021 uint32_t em, addr_len, maddr, mlen; 2022 2023 /* Memory can only be accessed in naturally aligned 4 byte units */ 2024 if (addr & 3 || len & 3 || len == 0) 2025 return (EINVAL); 2026 2027 /* Enabled memories */ 2028 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 2029 if (em & F_EDRAM0_ENABLE) { 2030 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 2031 maddr = G_EDRAM0_BASE(addr_len) << 20; 2032 mlen = G_EDRAM0_SIZE(addr_len) << 20; 2033 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 2034 addr + len <= maddr + mlen) 2035 return (0); 2036 } 2037 if (em & F_EDRAM1_ENABLE) { 2038 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 2039 maddr = G_EDRAM1_BASE(addr_len) << 20; 2040 mlen = G_EDRAM1_SIZE(addr_len) << 20; 2041 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 2042 addr + len <= maddr + mlen) 2043 return (0); 2044 } 2045 if (em & F_EXT_MEM_ENABLE) { 2046 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 2047 maddr = G_EXT_MEM_BASE(addr_len) << 20; 2048 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 2049 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 2050 addr + len <= maddr + mlen) 2051 return (0); 2052 } 2053 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 2054 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 2055 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 2056 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 2057 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 2058 addr + len <= maddr + mlen) 2059 return (0); 2060 } 2061 2062 return (EFAULT); 2063 } 2064 2065 static int 2066 fwmtype_to_hwmtype(int mtype) 2067 { 2068 2069 switch (mtype) { 2070 case FW_MEMTYPE_EDC0: 2071 return (MEM_EDC0); 2072 case FW_MEMTYPE_EDC1: 2073 return (MEM_EDC1); 2074 case FW_MEMTYPE_EXTMEM: 2075 return (MEM_MC0); 2076 case FW_MEMTYPE_EXTMEM1: 2077 return (MEM_MC1); 2078 default: 2079 panic("%s: cannot translate fw mtype %d.", __func__, mtype); 2080 } 2081 } 2082 2083 /* 2084 * Verify that the memory range specified by the memtype/offset/len pair is 2085 * valid and lies entirely within the memtype specified. The global address of 2086 * the start of the range is returned in addr. 2087 */ 2088 static int 2089 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 2090 uint32_t *addr) 2091 { 2092 uint32_t em, addr_len, maddr, mlen; 2093 2094 /* Memory can only be accessed in naturally aligned 4 byte units */ 2095 if (off & 3 || len & 3 || len == 0) 2096 return (EINVAL); 2097 2098 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 2099 switch (fwmtype_to_hwmtype(mtype)) { 2100 case MEM_EDC0: 2101 if (!(em & F_EDRAM0_ENABLE)) 2102 return (EINVAL); 2103 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 2104 maddr = G_EDRAM0_BASE(addr_len) << 20; 2105 mlen = G_EDRAM0_SIZE(addr_len) << 20; 2106 break; 2107 case MEM_EDC1: 2108 if (!(em & F_EDRAM1_ENABLE)) 2109 return (EINVAL); 2110 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 2111 maddr = G_EDRAM1_BASE(addr_len) << 20; 2112 mlen = G_EDRAM1_SIZE(addr_len) << 20; 2113 break; 2114 case MEM_MC: 2115 if (!(em & F_EXT_MEM_ENABLE)) 2116 return (EINVAL); 2117 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 2118 maddr = G_EXT_MEM_BASE(addr_len) << 20; 2119 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 2120 break; 2121 case MEM_MC1: 2122 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 2123 return (EINVAL); 2124 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 2125 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 2126 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 2127 break; 2128 default: 2129 return (EINVAL); 2130 } 2131 2132 if (mlen > 0 && off < mlen && off + len <= mlen) { 2133 *addr = maddr + off; /* global address */ 2134 return (0); 2135 } 2136 2137 return (EFAULT); 2138 } 2139 2140 static void 2141 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 2142 { 2143 const struct memwin *mw; 2144 2145 if (is_t4(sc)) { 2146 KASSERT(win >= 0 && win < nitems(t4_memwin), 2147 ("%s: incorrect memwin# (%d)", __func__, win)); 2148 mw = &t4_memwin[win]; 2149 } else { 2150 KASSERT(win >= 0 && win < nitems(t5_memwin), 2151 ("%s: incorrect memwin# (%d)", __func__, win)); 2152 mw = &t5_memwin[win]; 2153 } 2154 2155 if (base != NULL) 2156 *base = mw->base; 2157 if (aperture != NULL) 2158 *aperture = mw->aperture; 2159 } 2160 2161 /* 2162 * Positions the memory window such that it can be used to access the specified 2163 * address in the chip's address space. The return value is the offset of addr 2164 * from the start of the window. 2165 */ 2166 static uint32_t 2167 position_memwin(struct adapter *sc, int n, uint32_t addr) 2168 { 2169 uint32_t start, pf; 2170 uint32_t reg; 2171 2172 KASSERT(n >= 0 && n <= 3, 2173 ("%s: invalid window %d.", __func__, n)); 2174 KASSERT((addr & 3) == 0, 2175 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 2176 2177 if (is_t4(sc)) { 2178 pf = 0; 2179 start = addr & ~0xf; /* start must be 16B aligned */ 2180 } else { 2181 pf = V_PFNUM(sc->pf); 2182 start = addr & ~0x7f; /* start must be 128B aligned */ 2183 } 2184 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 2185 2186 t4_write_reg(sc, reg, start | pf); 2187 t4_read_reg(sc, reg); 2188 2189 return (addr - start); 2190 } 2191 2192 static int 2193 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, int num_vis, 2194 struct intrs_and_queues *iaq) 2195 { 2196 int rc, itype, navail, nrxq10g, nrxq1g, n; 2197 int nofldrxq10g = 0, nofldrxq1g = 0; 2198 int nnmrxq10g = 0, nnmrxq1g = 0; 2199 2200 bzero(iaq, sizeof(*iaq)); 2201 2202 iaq->ntxq10g = t4_ntxq10g; 2203 iaq->ntxq1g = t4_ntxq1g; 2204 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 2205 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 2206 iaq->rsrv_noflowq = t4_rsrv_noflowq; 2207 #ifdef TCP_OFFLOAD 2208 if (is_offload(sc)) { 2209 iaq->nofldtxq10g = t4_nofldtxq10g; 2210 iaq->nofldtxq1g = t4_nofldtxq1g; 2211 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 2212 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 2213 } 2214 #endif 2215 #ifdef DEV_NETMAP 2216 iaq->nnmtxq10g = t4_nnmtxq10g; 2217 iaq->nnmtxq1g = t4_nnmtxq1g; 2218 iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g; 2219 iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g; 2220 #endif 2221 2222 for (itype = INTR_MSIX; itype; itype >>= 1) { 2223 2224 if ((itype & t4_intr_types) == 0) 2225 continue; /* not allowed */ 2226 2227 if (itype == INTR_MSIX) 2228 navail = pci_msix_count(sc->dev); 2229 else if (itype == INTR_MSI) 2230 navail = pci_msi_count(sc->dev); 2231 else 2232 navail = 1; 2233 restart: 2234 if (navail == 0) 2235 continue; 2236 2237 iaq->intr_type = itype; 2238 iaq->intr_flags_10g = 0; 2239 iaq->intr_flags_1g = 0; 2240 2241 /* 2242 * Best option: an interrupt vector for errors, one for the 2243 * firmware event queue, and one for every rxq (NIC, TOE, and 2244 * netmap). 2245 */ 2246 iaq->nirq = T4_EXTRA_INTR; 2247 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g); 2248 iaq->nirq += n10g * 2 * (num_vis - 1); 2249 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g); 2250 iaq->nirq += n1g * 2 * (num_vis - 1); 2251 if (iaq->nirq <= navail && 2252 (itype != INTR_MSI || powerof2(iaq->nirq))) { 2253 iaq->intr_flags_10g = INTR_ALL; 2254 iaq->intr_flags_1g = INTR_ALL; 2255 goto allocate; 2256 } 2257 2258 /* 2259 * Second best option: a vector for errors, one for the firmware 2260 * event queue, and vectors for either all the NIC rx queues or 2261 * all the TOE rx queues. The queues that don't get vectors 2262 * will forward their interrupts to those that do. 2263 * 2264 * Note: netmap rx queues cannot be created early and so they 2265 * can't be setup to receive forwarded interrupts for others. 2266 */ 2267 iaq->nirq = T4_EXTRA_INTR; 2268 if (nrxq10g >= nofldrxq10g) { 2269 iaq->intr_flags_10g = INTR_RXQ; 2270 iaq->nirq += n10g * nrxq10g; 2271 iaq->nirq += n10g * (num_vis - 1); 2272 #ifdef DEV_NETMAP 2273 iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g); 2274 #endif 2275 } else { 2276 iaq->intr_flags_10g = INTR_OFLD_RXQ; 2277 iaq->nirq += n10g * nofldrxq10g; 2278 #ifdef DEV_NETMAP 2279 iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g); 2280 #endif 2281 } 2282 if (nrxq1g >= nofldrxq1g) { 2283 iaq->intr_flags_1g = INTR_RXQ; 2284 iaq->nirq += n1g * nrxq1g; 2285 iaq->nirq += n1g * (num_vis - 1); 2286 #ifdef DEV_NETMAP 2287 iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g); 2288 #endif 2289 } else { 2290 iaq->intr_flags_1g = INTR_OFLD_RXQ; 2291 iaq->nirq += n1g * nofldrxq1g; 2292 #ifdef DEV_NETMAP 2293 iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g); 2294 #endif 2295 } 2296 if (iaq->nirq <= navail && 2297 (itype != INTR_MSI || powerof2(iaq->nirq))) 2298 goto allocate; 2299 2300 /* 2301 * Next best option: an interrupt vector for errors, one for the 2302 * firmware event queue, and at least one per VI. At this 2303 * point we know we'll have to downsize nrxq and/or nofldrxq 2304 * and/or nnmrxq to fit what's available to us. 2305 */ 2306 iaq->nirq = T4_EXTRA_INTR; 2307 iaq->nirq += (n10g + n1g) * num_vis; 2308 if (iaq->nirq <= navail) { 2309 int leftover = navail - iaq->nirq; 2310 2311 if (n10g > 0) { 2312 int target = max(nrxq10g, nofldrxq10g); 2313 2314 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ? 2315 INTR_RXQ : INTR_OFLD_RXQ; 2316 2317 n = 1; 2318 while (n < target && leftover >= n10g) { 2319 leftover -= n10g; 2320 iaq->nirq += n10g; 2321 n++; 2322 } 2323 iaq->nrxq10g = min(n, nrxq10g); 2324 #ifdef TCP_OFFLOAD 2325 iaq->nofldrxq10g = min(n, nofldrxq10g); 2326 #endif 2327 #ifdef DEV_NETMAP 2328 iaq->nnmrxq10g = min(n, nnmrxq10g); 2329 #endif 2330 } 2331 2332 if (n1g > 0) { 2333 int target = max(nrxq1g, nofldrxq1g); 2334 2335 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ? 2336 INTR_RXQ : INTR_OFLD_RXQ; 2337 2338 n = 1; 2339 while (n < target && leftover >= n1g) { 2340 leftover -= n1g; 2341 iaq->nirq += n1g; 2342 n++; 2343 } 2344 iaq->nrxq1g = min(n, nrxq1g); 2345 #ifdef TCP_OFFLOAD 2346 iaq->nofldrxq1g = min(n, nofldrxq1g); 2347 #endif 2348 #ifdef DEV_NETMAP 2349 iaq->nnmrxq1g = min(n, nnmrxq1g); 2350 #endif 2351 } 2352 2353 if (itype != INTR_MSI || powerof2(iaq->nirq)) 2354 goto allocate; 2355 } 2356 2357 /* 2358 * Least desirable option: one interrupt vector for everything. 2359 */ 2360 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 2361 iaq->intr_flags_10g = iaq->intr_flags_1g = 0; 2362 #ifdef TCP_OFFLOAD 2363 if (is_offload(sc)) 2364 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 2365 #endif 2366 #ifdef DEV_NETMAP 2367 iaq->nnmrxq10g = iaq->nnmrxq1g = 1; 2368 #endif 2369 2370 allocate: 2371 navail = iaq->nirq; 2372 rc = 0; 2373 if (itype == INTR_MSIX) 2374 rc = pci_alloc_msix(sc->dev, &navail); 2375 else if (itype == INTR_MSI) 2376 rc = pci_alloc_msi(sc->dev, &navail); 2377 2378 if (rc == 0) { 2379 if (navail == iaq->nirq) 2380 return (0); 2381 2382 /* 2383 * Didn't get the number requested. Use whatever number 2384 * the kernel is willing to allocate (it's in navail). 2385 */ 2386 device_printf(sc->dev, "fewer vectors than requested, " 2387 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 2388 itype, iaq->nirq, navail); 2389 pci_release_msi(sc->dev); 2390 goto restart; 2391 } 2392 2393 device_printf(sc->dev, 2394 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 2395 itype, rc, iaq->nirq, navail); 2396 } 2397 2398 device_printf(sc->dev, 2399 "failed to find a usable interrupt type. " 2400 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 2401 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 2402 2403 return (ENXIO); 2404 } 2405 2406 #define FW_VERSION(chip) ( \ 2407 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ 2408 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ 2409 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ 2410 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) 2411 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) 2412 2413 struct fw_info { 2414 uint8_t chip; 2415 char *kld_name; 2416 char *fw_mod_name; 2417 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 2418 } fw_info[] = { 2419 { 2420 .chip = CHELSIO_T4, 2421 .kld_name = "t4fw_cfg", 2422 .fw_mod_name = "t4fw", 2423 .fw_hdr = { 2424 .chip = FW_HDR_CHIP_T4, 2425 .fw_ver = htobe32_const(FW_VERSION(T4)), 2426 .intfver_nic = FW_INTFVER(T4, NIC), 2427 .intfver_vnic = FW_INTFVER(T4, VNIC), 2428 .intfver_ofld = FW_INTFVER(T4, OFLD), 2429 .intfver_ri = FW_INTFVER(T4, RI), 2430 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 2431 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 2432 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 2433 .intfver_fcoe = FW_INTFVER(T4, FCOE), 2434 }, 2435 }, { 2436 .chip = CHELSIO_T5, 2437 .kld_name = "t5fw_cfg", 2438 .fw_mod_name = "t5fw", 2439 .fw_hdr = { 2440 .chip = FW_HDR_CHIP_T5, 2441 .fw_ver = htobe32_const(FW_VERSION(T5)), 2442 .intfver_nic = FW_INTFVER(T5, NIC), 2443 .intfver_vnic = FW_INTFVER(T5, VNIC), 2444 .intfver_ofld = FW_INTFVER(T5, OFLD), 2445 .intfver_ri = FW_INTFVER(T5, RI), 2446 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 2447 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 2448 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 2449 .intfver_fcoe = FW_INTFVER(T5, FCOE), 2450 }, 2451 } 2452 }; 2453 2454 static struct fw_info * 2455 find_fw_info(int chip) 2456 { 2457 int i; 2458 2459 for (i = 0; i < nitems(fw_info); i++) { 2460 if (fw_info[i].chip == chip) 2461 return (&fw_info[i]); 2462 } 2463 return (NULL); 2464 } 2465 2466 /* 2467 * Is the given firmware API compatible with the one the driver was compiled 2468 * with? 2469 */ 2470 static int 2471 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 2472 { 2473 2474 /* short circuit if it's the exact same firmware version */ 2475 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 2476 return (1); 2477 2478 /* 2479 * XXX: Is this too conservative? Perhaps I should limit this to the 2480 * features that are supported in the driver. 2481 */ 2482 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 2483 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 2484 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 2485 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 2486 return (1); 2487 #undef SAME_INTF 2488 2489 return (0); 2490 } 2491 2492 /* 2493 * The firmware in the KLD is usable, but should it be installed? This routine 2494 * explains itself in detail if it indicates the KLD firmware should be 2495 * installed. 2496 */ 2497 static int 2498 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 2499 { 2500 const char *reason; 2501 2502 if (!card_fw_usable) { 2503 reason = "incompatible or unusable"; 2504 goto install; 2505 } 2506 2507 if (k > c) { 2508 reason = "older than the version bundled with this driver"; 2509 goto install; 2510 } 2511 2512 if (t4_fw_install == 2 && k != c) { 2513 reason = "different than the version bundled with this driver"; 2514 goto install; 2515 } 2516 2517 return (0); 2518 2519 install: 2520 if (t4_fw_install == 0) { 2521 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2522 "but the driver is prohibited from installing a different " 2523 "firmware on the card.\n", 2524 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2525 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 2526 2527 return (0); 2528 } 2529 2530 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2531 "installing firmware %u.%u.%u.%u on card.\n", 2532 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2533 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 2534 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2535 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2536 2537 return (1); 2538 } 2539 /* 2540 * Establish contact with the firmware and determine if we are the master driver 2541 * or not, and whether we are responsible for chip initialization. 2542 */ 2543 static int 2544 prep_firmware(struct adapter *sc) 2545 { 2546 const struct firmware *fw = NULL, *default_cfg; 2547 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 2548 enum dev_state state; 2549 struct fw_info *fw_info; 2550 struct fw_hdr *card_fw; /* fw on the card */ 2551 const struct fw_hdr *kld_fw; /* fw in the KLD */ 2552 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 2553 against */ 2554 2555 /* Contact firmware. */ 2556 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 2557 if (rc < 0 || state == DEV_STATE_ERR) { 2558 rc = -rc; 2559 device_printf(sc->dev, 2560 "failed to connect to the firmware: %d, %d.\n", rc, state); 2561 return (rc); 2562 } 2563 pf = rc; 2564 if (pf == sc->mbox) 2565 sc->flags |= MASTER_PF; 2566 else if (state == DEV_STATE_UNINIT) { 2567 /* 2568 * We didn't get to be the master so we definitely won't be 2569 * configuring the chip. It's a bug if someone else hasn't 2570 * configured it already. 2571 */ 2572 device_printf(sc->dev, "couldn't be master(%d), " 2573 "device not already initialized either(%d).\n", rc, state); 2574 return (EDOOFUS); 2575 } 2576 2577 /* This is the firmware whose headers the driver was compiled against */ 2578 fw_info = find_fw_info(chip_id(sc)); 2579 if (fw_info == NULL) { 2580 device_printf(sc->dev, 2581 "unable to look up firmware information for chip %d.\n", 2582 chip_id(sc)); 2583 return (EINVAL); 2584 } 2585 drv_fw = &fw_info->fw_hdr; 2586 2587 /* 2588 * The firmware KLD contains many modules. The KLD name is also the 2589 * name of the module that contains the default config file. 2590 */ 2591 default_cfg = firmware_get(fw_info->kld_name); 2592 2593 /* Read the header of the firmware on the card */ 2594 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 2595 rc = -t4_read_flash(sc, FLASH_FW_START, 2596 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 2597 if (rc == 0) 2598 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 2599 else { 2600 device_printf(sc->dev, 2601 "Unable to read card's firmware header: %d\n", rc); 2602 card_fw_usable = 0; 2603 } 2604 2605 /* This is the firmware in the KLD */ 2606 fw = firmware_get(fw_info->fw_mod_name); 2607 if (fw != NULL) { 2608 kld_fw = (const void *)fw->data; 2609 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 2610 } else { 2611 kld_fw = NULL; 2612 kld_fw_usable = 0; 2613 } 2614 2615 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2616 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) { 2617 /* 2618 * Common case: the firmware on the card is an exact match and 2619 * the KLD is an exact match too, or the KLD is 2620 * absent/incompatible. Note that t4_fw_install = 2 is ignored 2621 * here -- use cxgbetool loadfw if you want to reinstall the 2622 * same firmware as the one on the card. 2623 */ 2624 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2625 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2626 be32toh(card_fw->fw_ver))) { 2627 2628 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 2629 if (rc != 0) { 2630 device_printf(sc->dev, 2631 "failed to install firmware: %d\n", rc); 2632 goto done; 2633 } 2634 2635 /* Installed successfully, update the cached header too. */ 2636 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2637 card_fw_usable = 1; 2638 need_fw_reset = 0; /* already reset as part of load_fw */ 2639 } 2640 2641 if (!card_fw_usable) { 2642 uint32_t d, c, k; 2643 2644 d = ntohl(drv_fw->fw_ver); 2645 c = ntohl(card_fw->fw_ver); 2646 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2647 2648 device_printf(sc->dev, "Cannot find a usable firmware: " 2649 "fw_install %d, chip state %d, " 2650 "driver compiled with %d.%d.%d.%d, " 2651 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2652 t4_fw_install, state, 2653 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2654 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2655 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2656 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2657 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2658 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2659 rc = EINVAL; 2660 goto done; 2661 } 2662 2663 /* We're using whatever's on the card and it's known to be good. */ 2664 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2665 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2666 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2667 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2668 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2669 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2670 t4_get_tp_version(sc, &sc->params.tp_vers); 2671 2672 /* Reset device */ 2673 if (need_fw_reset && 2674 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2675 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2676 if (rc != ETIMEDOUT && rc != EIO) 2677 t4_fw_bye(sc, sc->mbox); 2678 goto done; 2679 } 2680 sc->flags |= FW_OK; 2681 2682 rc = get_params__pre_init(sc); 2683 if (rc != 0) 2684 goto done; /* error message displayed already */ 2685 2686 /* Partition adapter resources as specified in the config file. */ 2687 if (state == DEV_STATE_UNINIT) { 2688 2689 KASSERT(sc->flags & MASTER_PF, 2690 ("%s: trying to change chip settings when not master.", 2691 __func__)); 2692 2693 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2694 if (rc != 0) 2695 goto done; /* error message displayed already */ 2696 2697 t4_tweak_chip_settings(sc); 2698 2699 /* get basic stuff going */ 2700 rc = -t4_fw_initialize(sc, sc->mbox); 2701 if (rc != 0) { 2702 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2703 goto done; 2704 } 2705 } else { 2706 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2707 sc->cfcsum = 0; 2708 } 2709 2710 done: 2711 free(card_fw, M_CXGBE); 2712 if (fw != NULL) 2713 firmware_put(fw, FIRMWARE_UNLOAD); 2714 if (default_cfg != NULL) 2715 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2716 2717 return (rc); 2718 } 2719 2720 #define FW_PARAM_DEV(param) \ 2721 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2722 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2723 #define FW_PARAM_PFVF(param) \ 2724 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2725 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2726 2727 /* 2728 * Partition chip resources for use between various PFs, VFs, etc. 2729 */ 2730 static int 2731 partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2732 const char *name_prefix) 2733 { 2734 const struct firmware *cfg = NULL; 2735 int rc = 0; 2736 struct fw_caps_config_cmd caps; 2737 uint32_t mtype, moff, finicsum, cfcsum; 2738 2739 /* 2740 * Figure out what configuration file to use. Pick the default config 2741 * file for the card if the user hasn't specified one explicitly. 2742 */ 2743 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2744 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2745 /* Card specific overrides go here. */ 2746 if (pci_get_device(sc->dev) == 0x440a) 2747 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2748 if (is_fpga(sc)) 2749 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2750 } 2751 2752 /* 2753 * We need to load another module if the profile is anything except 2754 * "default" or "flash". 2755 */ 2756 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2757 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2758 char s[32]; 2759 2760 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2761 cfg = firmware_get(s); 2762 if (cfg == NULL) { 2763 if (default_cfg != NULL) { 2764 device_printf(sc->dev, 2765 "unable to load module \"%s\" for " 2766 "configuration profile \"%s\", will use " 2767 "the default config file instead.\n", 2768 s, sc->cfg_file); 2769 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2770 "%s", DEFAULT_CF); 2771 } else { 2772 device_printf(sc->dev, 2773 "unable to load module \"%s\" for " 2774 "configuration profile \"%s\", will use " 2775 "the config file on the card's flash " 2776 "instead.\n", s, sc->cfg_file); 2777 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2778 "%s", FLASH_CF); 2779 } 2780 } 2781 } 2782 2783 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2784 default_cfg == NULL) { 2785 device_printf(sc->dev, 2786 "default config file not available, will use the config " 2787 "file on the card's flash instead.\n"); 2788 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2789 } 2790 2791 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2792 u_int cflen, i, n; 2793 const uint32_t *cfdata; 2794 uint32_t param, val, addr, off, mw_base, mw_aperture; 2795 2796 KASSERT(cfg != NULL || default_cfg != NULL, 2797 ("%s: no config to upload", __func__)); 2798 2799 /* 2800 * Ask the firmware where it wants us to upload the config file. 2801 */ 2802 param = FW_PARAM_DEV(CF); 2803 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2804 if (rc != 0) { 2805 /* No support for config file? Shouldn't happen. */ 2806 device_printf(sc->dev, 2807 "failed to query config file location: %d.\n", rc); 2808 goto done; 2809 } 2810 mtype = G_FW_PARAMS_PARAM_Y(val); 2811 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2812 2813 /* 2814 * XXX: sheer laziness. We deliberately added 4 bytes of 2815 * useless stuffing/comments at the end of the config file so 2816 * it's ok to simply throw away the last remaining bytes when 2817 * the config file is not an exact multiple of 4. This also 2818 * helps with the validate_mt_off_len check. 2819 */ 2820 if (cfg != NULL) { 2821 cflen = cfg->datasize & ~3; 2822 cfdata = cfg->data; 2823 } else { 2824 cflen = default_cfg->datasize & ~3; 2825 cfdata = default_cfg->data; 2826 } 2827 2828 if (cflen > FLASH_CFG_MAX_SIZE) { 2829 device_printf(sc->dev, 2830 "config file too long (%d, max allowed is %d). " 2831 "Will try to use the config on the card, if any.\n", 2832 cflen, FLASH_CFG_MAX_SIZE); 2833 goto use_config_on_flash; 2834 } 2835 2836 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2837 if (rc != 0) { 2838 device_printf(sc->dev, 2839 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2840 "Will try to use the config on the card, if any.\n", 2841 __func__, mtype, moff, cflen, rc); 2842 goto use_config_on_flash; 2843 } 2844 2845 memwin_info(sc, 2, &mw_base, &mw_aperture); 2846 while (cflen) { 2847 off = position_memwin(sc, 2, addr); 2848 n = min(cflen, mw_aperture - off); 2849 for (i = 0; i < n; i += 4) 2850 t4_write_reg(sc, mw_base + off + i, *cfdata++); 2851 cflen -= n; 2852 addr += n; 2853 } 2854 } else { 2855 use_config_on_flash: 2856 mtype = FW_MEMTYPE_FLASH; 2857 moff = t4_flash_cfg_addr(sc); 2858 } 2859 2860 bzero(&caps, sizeof(caps)); 2861 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2862 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2863 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2864 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2865 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2866 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2867 if (rc != 0) { 2868 device_printf(sc->dev, 2869 "failed to pre-process config file: %d " 2870 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2871 goto done; 2872 } 2873 2874 finicsum = be32toh(caps.finicsum); 2875 cfcsum = be32toh(caps.cfcsum); 2876 if (finicsum != cfcsum) { 2877 device_printf(sc->dev, 2878 "WARNING: config file checksum mismatch: %08x %08x\n", 2879 finicsum, cfcsum); 2880 } 2881 sc->cfcsum = cfcsum; 2882 2883 #define LIMIT_CAPS(x) do { \ 2884 caps.x &= htobe16(t4_##x##_allowed); \ 2885 } while (0) 2886 2887 /* 2888 * Let the firmware know what features will (not) be used so it can tune 2889 * things accordingly. 2890 */ 2891 LIMIT_CAPS(linkcaps); 2892 LIMIT_CAPS(niccaps); 2893 LIMIT_CAPS(toecaps); 2894 LIMIT_CAPS(rdmacaps); 2895 LIMIT_CAPS(iscsicaps); 2896 LIMIT_CAPS(fcoecaps); 2897 #undef LIMIT_CAPS 2898 2899 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2900 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2901 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2902 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2903 if (rc != 0) { 2904 device_printf(sc->dev, 2905 "failed to process config file: %d.\n", rc); 2906 } 2907 done: 2908 if (cfg != NULL) 2909 firmware_put(cfg, FIRMWARE_UNLOAD); 2910 return (rc); 2911 } 2912 2913 /* 2914 * Retrieve parameters that are needed (or nice to have) very early. 2915 */ 2916 static int 2917 get_params__pre_init(struct adapter *sc) 2918 { 2919 int rc; 2920 uint32_t param[2], val[2]; 2921 struct fw_devlog_cmd cmd; 2922 struct devlog_params *dlog = &sc->params.devlog; 2923 2924 param[0] = FW_PARAM_DEV(PORTVEC); 2925 param[1] = FW_PARAM_DEV(CCLK); 2926 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2927 if (rc != 0) { 2928 device_printf(sc->dev, 2929 "failed to query parameters (pre_init): %d.\n", rc); 2930 return (rc); 2931 } 2932 2933 sc->params.portvec = val[0]; 2934 sc->params.nports = bitcount32(val[0]); 2935 sc->params.vpd.cclk = val[1]; 2936 2937 /* Read device log parameters. */ 2938 bzero(&cmd, sizeof(cmd)); 2939 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2940 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2941 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2942 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2943 if (rc != 0) { 2944 device_printf(sc->dev, 2945 "failed to get devlog parameters: %d.\n", rc); 2946 bzero(dlog, sizeof (*dlog)); 2947 rc = 0; /* devlog isn't critical for device operation */ 2948 } else { 2949 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2950 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2951 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2952 dlog->size = be32toh(cmd.memsize_devlog); 2953 } 2954 2955 return (rc); 2956 } 2957 2958 /* 2959 * Retrieve various parameters that are of interest to the driver. The device 2960 * has been initialized by the firmware at this point. 2961 */ 2962 static int 2963 get_params__post_init(struct adapter *sc) 2964 { 2965 int rc; 2966 uint32_t param[7], val[7]; 2967 struct fw_caps_config_cmd caps; 2968 2969 param[0] = FW_PARAM_PFVF(IQFLINT_START); 2970 param[1] = FW_PARAM_PFVF(EQ_START); 2971 param[2] = FW_PARAM_PFVF(FILTER_START); 2972 param[3] = FW_PARAM_PFVF(FILTER_END); 2973 param[4] = FW_PARAM_PFVF(L2T_START); 2974 param[5] = FW_PARAM_PFVF(L2T_END); 2975 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2976 if (rc != 0) { 2977 device_printf(sc->dev, 2978 "failed to query parameters (post_init): %d.\n", rc); 2979 return (rc); 2980 } 2981 2982 sc->sge.iq_start = val[0]; 2983 sc->sge.eq_start = val[1]; 2984 sc->tids.ftid_base = val[2]; 2985 sc->tids.nftids = val[3] - val[2] + 1; 2986 sc->params.ftid_min = val[2]; 2987 sc->params.ftid_max = val[3]; 2988 sc->vres.l2t.start = val[4]; 2989 sc->vres.l2t.size = val[5] - val[4] + 1; 2990 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2991 ("%s: L2 table size (%u) larger than expected (%u)", 2992 __func__, sc->vres.l2t.size, L2T_SIZE)); 2993 2994 /* get capabilites */ 2995 bzero(&caps, sizeof(caps)); 2996 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2997 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2998 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2999 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 3000 if (rc != 0) { 3001 device_printf(sc->dev, 3002 "failed to get card capabilities: %d.\n", rc); 3003 return (rc); 3004 } 3005 3006 #define READ_CAPS(x) do { \ 3007 sc->x = htobe16(caps.x); \ 3008 } while (0) 3009 READ_CAPS(linkcaps); 3010 READ_CAPS(niccaps); 3011 READ_CAPS(toecaps); 3012 READ_CAPS(rdmacaps); 3013 READ_CAPS(iscsicaps); 3014 READ_CAPS(fcoecaps); 3015 3016 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { 3017 param[0] = FW_PARAM_PFVF(ETHOFLD_START); 3018 param[1] = FW_PARAM_PFVF(ETHOFLD_END); 3019 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 3020 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); 3021 if (rc != 0) { 3022 device_printf(sc->dev, 3023 "failed to query NIC parameters: %d.\n", rc); 3024 return (rc); 3025 } 3026 sc->tids.etid_base = val[0]; 3027 sc->params.etid_min = val[0]; 3028 sc->tids.netids = val[1] - val[0] + 1; 3029 sc->params.netids = sc->tids.netids; 3030 sc->params.eo_wr_cred = val[2]; 3031 sc->params.ethoffload = 1; 3032 } 3033 3034 if (sc->toecaps) { 3035 /* query offload-related parameters */ 3036 param[0] = FW_PARAM_DEV(NTID); 3037 param[1] = FW_PARAM_PFVF(SERVER_START); 3038 param[2] = FW_PARAM_PFVF(SERVER_END); 3039 param[3] = FW_PARAM_PFVF(TDDP_START); 3040 param[4] = FW_PARAM_PFVF(TDDP_END); 3041 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 3042 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 3043 if (rc != 0) { 3044 device_printf(sc->dev, 3045 "failed to query TOE parameters: %d.\n", rc); 3046 return (rc); 3047 } 3048 sc->tids.ntids = val[0]; 3049 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 3050 sc->tids.stid_base = val[1]; 3051 sc->tids.nstids = val[2] - val[1] + 1; 3052 sc->vres.ddp.start = val[3]; 3053 sc->vres.ddp.size = val[4] - val[3] + 1; 3054 sc->params.ofldq_wr_cred = val[5]; 3055 sc->params.offload = 1; 3056 } 3057 if (sc->rdmacaps) { 3058 param[0] = FW_PARAM_PFVF(STAG_START); 3059 param[1] = FW_PARAM_PFVF(STAG_END); 3060 param[2] = FW_PARAM_PFVF(RQ_START); 3061 param[3] = FW_PARAM_PFVF(RQ_END); 3062 param[4] = FW_PARAM_PFVF(PBL_START); 3063 param[5] = FW_PARAM_PFVF(PBL_END); 3064 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 3065 if (rc != 0) { 3066 device_printf(sc->dev, 3067 "failed to query RDMA parameters(1): %d.\n", rc); 3068 return (rc); 3069 } 3070 sc->vres.stag.start = val[0]; 3071 sc->vres.stag.size = val[1] - val[0] + 1; 3072 sc->vres.rq.start = val[2]; 3073 sc->vres.rq.size = val[3] - val[2] + 1; 3074 sc->vres.pbl.start = val[4]; 3075 sc->vres.pbl.size = val[5] - val[4] + 1; 3076 3077 param[0] = FW_PARAM_PFVF(SQRQ_START); 3078 param[1] = FW_PARAM_PFVF(SQRQ_END); 3079 param[2] = FW_PARAM_PFVF(CQ_START); 3080 param[3] = FW_PARAM_PFVF(CQ_END); 3081 param[4] = FW_PARAM_PFVF(OCQ_START); 3082 param[5] = FW_PARAM_PFVF(OCQ_END); 3083 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 3084 if (rc != 0) { 3085 device_printf(sc->dev, 3086 "failed to query RDMA parameters(2): %d.\n", rc); 3087 return (rc); 3088 } 3089 sc->vres.qp.start = val[0]; 3090 sc->vres.qp.size = val[1] - val[0] + 1; 3091 sc->vres.cq.start = val[2]; 3092 sc->vres.cq.size = val[3] - val[2] + 1; 3093 sc->vres.ocq.start = val[4]; 3094 sc->vres.ocq.size = val[5] - val[4] + 1; 3095 } 3096 if (sc->iscsicaps) { 3097 param[0] = FW_PARAM_PFVF(ISCSI_START); 3098 param[1] = FW_PARAM_PFVF(ISCSI_END); 3099 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 3100 if (rc != 0) { 3101 device_printf(sc->dev, 3102 "failed to query iSCSI parameters: %d.\n", rc); 3103 return (rc); 3104 } 3105 sc->vres.iscsi.start = val[0]; 3106 sc->vres.iscsi.size = val[1] - val[0] + 1; 3107 } 3108 3109 /* 3110 * We've got the params we wanted to query via the firmware. Now grab 3111 * some others directly from the chip. 3112 */ 3113 rc = t4_read_chip_settings(sc); 3114 3115 return (rc); 3116 } 3117 3118 static int 3119 set_params__post_init(struct adapter *sc) 3120 { 3121 uint32_t param, val; 3122 3123 /* ask for encapsulated CPLs */ 3124 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 3125 val = 1; 3126 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 3127 3128 return (0); 3129 } 3130 3131 #undef FW_PARAM_PFVF 3132 #undef FW_PARAM_DEV 3133 3134 static void 3135 t4_set_desc(struct adapter *sc) 3136 { 3137 char buf[128]; 3138 struct adapter_params *p = &sc->params; 3139 3140 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, " 3141 "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "", 3142 chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec); 3143 3144 device_set_desc_copy(sc->dev, buf); 3145 } 3146 3147 static void 3148 build_medialist(struct port_info *pi, struct ifmedia *media) 3149 { 3150 int m; 3151 3152 PORT_LOCK(pi); 3153 3154 ifmedia_removeall(media); 3155 3156 m = IFM_ETHER | IFM_FDX; 3157 3158 switch(pi->port_type) { 3159 case FW_PORT_TYPE_BT_XFI: 3160 case FW_PORT_TYPE_BT_XAUI: 3161 ifmedia_add(media, m | IFM_10G_T, 0, NULL); 3162 /* fall through */ 3163 3164 case FW_PORT_TYPE_BT_SGMII: 3165 ifmedia_add(media, m | IFM_1000_T, 0, NULL); 3166 ifmedia_add(media, m | IFM_100_TX, 0, NULL); 3167 ifmedia_add(media, IFM_ETHER | IFM_AUTO, 0, NULL); 3168 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 3169 break; 3170 3171 case FW_PORT_TYPE_CX4: 3172 ifmedia_add(media, m | IFM_10G_CX4, 0, NULL); 3173 ifmedia_set(media, m | IFM_10G_CX4); 3174 break; 3175 3176 case FW_PORT_TYPE_QSFP_10G: 3177 case FW_PORT_TYPE_SFP: 3178 case FW_PORT_TYPE_FIBER_XFI: 3179 case FW_PORT_TYPE_FIBER_XAUI: 3180 switch (pi->mod_type) { 3181 3182 case FW_PORT_MOD_TYPE_LR: 3183 ifmedia_add(media, m | IFM_10G_LR, 0, NULL); 3184 ifmedia_set(media, m | IFM_10G_LR); 3185 break; 3186 3187 case FW_PORT_MOD_TYPE_SR: 3188 ifmedia_add(media, m | IFM_10G_SR, 0, NULL); 3189 ifmedia_set(media, m | IFM_10G_SR); 3190 break; 3191 3192 case FW_PORT_MOD_TYPE_LRM: 3193 ifmedia_add(media, m | IFM_10G_LRM, 0, NULL); 3194 ifmedia_set(media, m | IFM_10G_LRM); 3195 break; 3196 3197 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 3198 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 3199 ifmedia_add(media, m | IFM_10G_TWINAX, 0, NULL); 3200 ifmedia_set(media, m | IFM_10G_TWINAX); 3201 break; 3202 3203 case FW_PORT_MOD_TYPE_NONE: 3204 m &= ~IFM_FDX; 3205 ifmedia_add(media, m | IFM_NONE, 0, NULL); 3206 ifmedia_set(media, m | IFM_NONE); 3207 break; 3208 3209 case FW_PORT_MOD_TYPE_NA: 3210 case FW_PORT_MOD_TYPE_ER: 3211 default: 3212 device_printf(pi->dev, 3213 "unknown port_type (%d), mod_type (%d)\n", 3214 pi->port_type, pi->mod_type); 3215 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 3216 ifmedia_set(media, m | IFM_UNKNOWN); 3217 break; 3218 } 3219 break; 3220 3221 case FW_PORT_TYPE_QSFP: 3222 switch (pi->mod_type) { 3223 3224 case FW_PORT_MOD_TYPE_LR: 3225 ifmedia_add(media, m | IFM_40G_LR4, 0, NULL); 3226 ifmedia_set(media, m | IFM_40G_LR4); 3227 break; 3228 3229 case FW_PORT_MOD_TYPE_SR: 3230 ifmedia_add(media, m | IFM_40G_SR4, 0, NULL); 3231 ifmedia_set(media, m | IFM_40G_SR4); 3232 break; 3233 3234 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 3235 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 3236 ifmedia_add(media, m | IFM_40G_CR4, 0, NULL); 3237 ifmedia_set(media, m | IFM_40G_CR4); 3238 break; 3239 3240 case FW_PORT_MOD_TYPE_NONE: 3241 m &= ~IFM_FDX; 3242 ifmedia_add(media, m | IFM_NONE, 0, NULL); 3243 ifmedia_set(media, m | IFM_NONE); 3244 break; 3245 3246 default: 3247 device_printf(pi->dev, 3248 "unknown port_type (%d), mod_type (%d)\n", 3249 pi->port_type, pi->mod_type); 3250 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 3251 ifmedia_set(media, m | IFM_UNKNOWN); 3252 break; 3253 } 3254 break; 3255 3256 default: 3257 device_printf(pi->dev, 3258 "unknown port_type (%d), mod_type (%d)\n", pi->port_type, 3259 pi->mod_type); 3260 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 3261 ifmedia_set(media, m | IFM_UNKNOWN); 3262 break; 3263 } 3264 3265 PORT_UNLOCK(pi); 3266 } 3267 3268 #define FW_MAC_EXACT_CHUNK 7 3269 3270 /* 3271 * Program the port's XGMAC based on parameters in ifnet. The caller also 3272 * indicates which parameters should be programmed (the rest are left alone). 3273 */ 3274 int 3275 update_mac_settings(struct ifnet *ifp, int flags) 3276 { 3277 int rc = 0; 3278 struct vi_info *vi = ifp->if_softc; 3279 struct port_info *pi = vi->pi; 3280 struct adapter *sc = pi->adapter; 3281 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 3282 3283 ASSERT_SYNCHRONIZED_OP(sc); 3284 KASSERT(flags, ("%s: not told what to update.", __func__)); 3285 3286 if (flags & XGMAC_MTU) 3287 mtu = ifp->if_mtu; 3288 3289 if (flags & XGMAC_PROMISC) 3290 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 3291 3292 if (flags & XGMAC_ALLMULTI) 3293 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 3294 3295 if (flags & XGMAC_VLANEX) 3296 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 3297 3298 if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) { 3299 rc = -t4_set_rxmode(sc, sc->mbox, vi->viid, mtu, promisc, 3300 allmulti, 1, vlanex, false); 3301 if (rc) { 3302 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, 3303 rc); 3304 return (rc); 3305 } 3306 } 3307 3308 if (flags & XGMAC_UCADDR) { 3309 uint8_t ucaddr[ETHER_ADDR_LEN]; 3310 3311 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 3312 rc = t4_change_mac(sc, sc->mbox, vi->viid, vi->xact_addr_filt, 3313 ucaddr, true, true); 3314 if (rc < 0) { 3315 rc = -rc; 3316 if_printf(ifp, "change_mac failed: %d\n", rc); 3317 return (rc); 3318 } else { 3319 vi->xact_addr_filt = rc; 3320 rc = 0; 3321 } 3322 } 3323 3324 if (flags & XGMAC_MCADDRS) { 3325 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 3326 int del = 1; 3327 uint64_t hash = 0; 3328 struct ifmultiaddr *ifma; 3329 int i = 0, j; 3330 3331 if_maddr_rlock(ifp); 3332 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3333 if (ifma->ifma_addr->sa_family != AF_LINK) 3334 continue; 3335 mcaddr[i] = 3336 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 3337 MPASS(ETHER_IS_MULTICAST(mcaddr[i])); 3338 i++; 3339 3340 if (i == FW_MAC_EXACT_CHUNK) { 3341 rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid, 3342 del, i, mcaddr, NULL, &hash, 0); 3343 if (rc < 0) { 3344 rc = -rc; 3345 for (j = 0; j < i; j++) { 3346 if_printf(ifp, 3347 "failed to add mc address" 3348 " %02x:%02x:%02x:" 3349 "%02x:%02x:%02x rc=%d\n", 3350 mcaddr[j][0], mcaddr[j][1], 3351 mcaddr[j][2], mcaddr[j][3], 3352 mcaddr[j][4], mcaddr[j][5], 3353 rc); 3354 } 3355 goto mcfail; 3356 } 3357 del = 0; 3358 i = 0; 3359 } 3360 } 3361 if (i > 0) { 3362 rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid, del, i, 3363 mcaddr, NULL, &hash, 0); 3364 if (rc < 0) { 3365 rc = -rc; 3366 for (j = 0; j < i; j++) { 3367 if_printf(ifp, 3368 "failed to add mc address" 3369 " %02x:%02x:%02x:" 3370 "%02x:%02x:%02x rc=%d\n", 3371 mcaddr[j][0], mcaddr[j][1], 3372 mcaddr[j][2], mcaddr[j][3], 3373 mcaddr[j][4], mcaddr[j][5], 3374 rc); 3375 } 3376 goto mcfail; 3377 } 3378 } 3379 3380 rc = -t4_set_addr_hash(sc, sc->mbox, vi->viid, 0, hash, 0); 3381 if (rc != 0) 3382 if_printf(ifp, "failed to set mc address hash: %d", rc); 3383 mcfail: 3384 if_maddr_runlock(ifp); 3385 } 3386 3387 return (rc); 3388 } 3389 3390 /* 3391 * {begin|end}_synchronized_op must be called from the same thread. 3392 */ 3393 int 3394 begin_synchronized_op(struct adapter *sc, struct vi_info *vi, int flags, 3395 char *wmesg) 3396 { 3397 int rc, pri; 3398 3399 #ifdef WITNESS 3400 /* the caller thinks it's ok to sleep, but is it really? */ 3401 if (flags & SLEEP_OK) 3402 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 3403 "begin_synchronized_op"); 3404 #endif 3405 3406 if (INTR_OK) 3407 pri = PCATCH; 3408 else 3409 pri = 0; 3410 3411 ADAPTER_LOCK(sc); 3412 for (;;) { 3413 3414 if (vi && IS_DOOMED(vi)) { 3415 rc = ENXIO; 3416 goto done; 3417 } 3418 3419 if (!IS_BUSY(sc)) { 3420 rc = 0; 3421 break; 3422 } 3423 3424 if (!(flags & SLEEP_OK)) { 3425 rc = EBUSY; 3426 goto done; 3427 } 3428 3429 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 3430 rc = EINTR; 3431 goto done; 3432 } 3433 } 3434 3435 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 3436 SET_BUSY(sc); 3437 #ifdef INVARIANTS 3438 sc->last_op = wmesg; 3439 sc->last_op_thr = curthread; 3440 sc->last_op_flags = flags; 3441 #endif 3442 3443 done: 3444 if (!(flags & HOLD_LOCK) || rc) 3445 ADAPTER_UNLOCK(sc); 3446 3447 return (rc); 3448 } 3449 3450 /* 3451 * Tell if_ioctl and if_init that the VI is going away. This is 3452 * special variant of begin_synchronized_op and must be paired with a 3453 * call to end_synchronized_op. 3454 */ 3455 void 3456 doom_vi(struct adapter *sc, struct vi_info *vi) 3457 { 3458 3459 ADAPTER_LOCK(sc); 3460 SET_DOOMED(vi); 3461 wakeup(&sc->flags); 3462 while (IS_BUSY(sc)) 3463 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 3464 SET_BUSY(sc); 3465 #ifdef INVARIANTS 3466 sc->last_op = "t4detach"; 3467 sc->last_op_thr = curthread; 3468 sc->last_op_flags = 0; 3469 #endif 3470 ADAPTER_UNLOCK(sc); 3471 } 3472 3473 /* 3474 * {begin|end}_synchronized_op must be called from the same thread. 3475 */ 3476 void 3477 end_synchronized_op(struct adapter *sc, int flags) 3478 { 3479 3480 if (flags & LOCK_HELD) 3481 ADAPTER_LOCK_ASSERT_OWNED(sc); 3482 else 3483 ADAPTER_LOCK(sc); 3484 3485 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 3486 CLR_BUSY(sc); 3487 wakeup(&sc->flags); 3488 ADAPTER_UNLOCK(sc); 3489 } 3490 3491 static int 3492 cxgbe_init_synchronized(struct vi_info *vi) 3493 { 3494 struct port_info *pi = vi->pi; 3495 struct adapter *sc = pi->adapter; 3496 struct ifnet *ifp = vi->ifp; 3497 int rc = 0, i; 3498 struct sge_txq *txq; 3499 3500 ASSERT_SYNCHRONIZED_OP(sc); 3501 3502 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 3503 return (0); /* already running */ 3504 3505 if (!(sc->flags & FULL_INIT_DONE) && 3506 ((rc = adapter_full_init(sc)) != 0)) 3507 return (rc); /* error message displayed already */ 3508 3509 if (!(vi->flags & VI_INIT_DONE) && 3510 ((rc = vi_full_init(vi)) != 0)) 3511 return (rc); /* error message displayed already */ 3512 3513 rc = update_mac_settings(ifp, XGMAC_ALL); 3514 if (rc) 3515 goto done; /* error message displayed already */ 3516 3517 rc = -t4_enable_vi(sc, sc->mbox, vi->viid, true, true); 3518 if (rc != 0) { 3519 if_printf(ifp, "enable_vi failed: %d\n", rc); 3520 goto done; 3521 } 3522 3523 /* 3524 * Can't fail from this point onwards. Review cxgbe_uninit_synchronized 3525 * if this changes. 3526 */ 3527 3528 for_each_txq(vi, i, txq) { 3529 TXQ_LOCK(txq); 3530 txq->eq.flags |= EQ_ENABLED; 3531 TXQ_UNLOCK(txq); 3532 } 3533 3534 /* 3535 * The first iq of the first port to come up is used for tracing. 3536 */ 3537 if (sc->traceq < 0 && IS_MAIN_VI(vi)) { 3538 sc->traceq = sc->sge.rxq[vi->first_rxq].iq.abs_id; 3539 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : 3540 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | 3541 V_QUEUENUMBER(sc->traceq)); 3542 pi->flags |= HAS_TRACEQ; 3543 } 3544 3545 /* all ok */ 3546 PORT_LOCK(pi); 3547 ifp->if_drv_flags |= IFF_DRV_RUNNING; 3548 pi->up_vis++; 3549 3550 if (pi->nvi > 1) 3551 callout_reset(&vi->tick, hz, vi_tick, vi); 3552 else 3553 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 3554 PORT_UNLOCK(pi); 3555 done: 3556 if (rc != 0) 3557 cxgbe_uninit_synchronized(vi); 3558 3559 return (rc); 3560 } 3561 3562 /* 3563 * Idempotent. 3564 */ 3565 static int 3566 cxgbe_uninit_synchronized(struct vi_info *vi) 3567 { 3568 struct port_info *pi = vi->pi; 3569 struct adapter *sc = pi->adapter; 3570 struct ifnet *ifp = vi->ifp; 3571 int rc, i; 3572 struct sge_txq *txq; 3573 3574 ASSERT_SYNCHRONIZED_OP(sc); 3575 3576 if (!(vi->flags & VI_INIT_DONE)) { 3577 KASSERT(!(ifp->if_drv_flags & IFF_DRV_RUNNING), 3578 ("uninited VI is running")); 3579 return (0); 3580 } 3581 3582 /* 3583 * Disable the VI so that all its data in either direction is discarded 3584 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 3585 * tick) intact as the TP can deliver negative advice or data that it's 3586 * holding in its RAM (for an offloaded connection) even after the VI is 3587 * disabled. 3588 */ 3589 rc = -t4_enable_vi(sc, sc->mbox, vi->viid, false, false); 3590 if (rc) { 3591 if_printf(ifp, "disable_vi failed: %d\n", rc); 3592 return (rc); 3593 } 3594 3595 for_each_txq(vi, i, txq) { 3596 TXQ_LOCK(txq); 3597 txq->eq.flags &= ~EQ_ENABLED; 3598 TXQ_UNLOCK(txq); 3599 } 3600 3601 PORT_LOCK(pi); 3602 if (pi->nvi == 1) 3603 callout_stop(&pi->tick); 3604 else 3605 callout_stop(&vi->tick); 3606 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 3607 PORT_UNLOCK(pi); 3608 return (0); 3609 } 3610 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3611 pi->up_vis--; 3612 if (pi->up_vis > 0) { 3613 PORT_UNLOCK(pi); 3614 return (0); 3615 } 3616 PORT_UNLOCK(pi); 3617 3618 pi->link_cfg.link_ok = 0; 3619 pi->link_cfg.speed = 0; 3620 pi->linkdnrc = -1; 3621 t4_os_link_changed(sc, pi->port_id, 0, -1); 3622 3623 return (0); 3624 } 3625 3626 /* 3627 * It is ok for this function to fail midway and return right away. t4_detach 3628 * will walk the entire sc->irq list and clean up whatever is valid. 3629 */ 3630 static int 3631 setup_intr_handlers(struct adapter *sc) 3632 { 3633 int rc, rid, p, q, v; 3634 char s[8]; 3635 struct irq *irq; 3636 struct port_info *pi; 3637 struct vi_info *vi; 3638 struct sge_rxq *rxq; 3639 #ifdef TCP_OFFLOAD 3640 struct sge_ofld_rxq *ofld_rxq; 3641 #endif 3642 #ifdef DEV_NETMAP 3643 struct sge_nm_rxq *nm_rxq; 3644 #endif 3645 3646 /* 3647 * Setup interrupts. 3648 */ 3649 irq = &sc->irq[0]; 3650 rid = sc->intr_type == INTR_INTX ? 0 : 1; 3651 if (sc->intr_count == 1) 3652 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all")); 3653 3654 /* Multiple interrupts. */ 3655 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 3656 ("%s: too few intr.", __func__)); 3657 3658 /* The first one is always error intr */ 3659 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 3660 if (rc != 0) 3661 return (rc); 3662 irq++; 3663 rid++; 3664 3665 /* The second one is always the firmware event queue */ 3666 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt"); 3667 if (rc != 0) 3668 return (rc); 3669 irq++; 3670 rid++; 3671 3672 for_each_port(sc, p) { 3673 pi = sc->port[p]; 3674 for_each_vi(pi, v, vi) { 3675 vi->first_intr = rid - 1; 3676 #ifdef DEV_NETMAP 3677 if (vi->flags & VI_NETMAP) { 3678 for_each_nm_rxq(vi, q, nm_rxq) { 3679 snprintf(s, sizeof(s), "%d-%d", p, q); 3680 rc = t4_alloc_irq(sc, irq, rid, 3681 t4_nm_intr, nm_rxq, s); 3682 if (rc != 0) 3683 return (rc); 3684 irq++; 3685 rid++; 3686 vi->nintr++; 3687 } 3688 continue; 3689 } 3690 #endif 3691 if (vi->flags & INTR_RXQ) { 3692 for_each_rxq(vi, q, rxq) { 3693 if (v == 0) 3694 snprintf(s, sizeof(s), "%d.%d", 3695 p, q); 3696 else 3697 snprintf(s, sizeof(s), 3698 "%d(%d).%d", p, v, q); 3699 rc = t4_alloc_irq(sc, irq, rid, 3700 t4_intr, rxq, s); 3701 if (rc != 0) 3702 return (rc); 3703 irq++; 3704 rid++; 3705 vi->nintr++; 3706 } 3707 } 3708 #ifdef TCP_OFFLOAD 3709 if (vi->flags & INTR_OFLD_RXQ) { 3710 for_each_ofld_rxq(vi, q, ofld_rxq) { 3711 snprintf(s, sizeof(s), "%d,%d", p, q); 3712 rc = t4_alloc_irq(sc, irq, rid, 3713 t4_intr, ofld_rxq, s); 3714 if (rc != 0) 3715 return (rc); 3716 irq++; 3717 rid++; 3718 vi->nintr++; 3719 } 3720 } 3721 #endif 3722 } 3723 } 3724 MPASS(irq == &sc->irq[sc->intr_count]); 3725 3726 return (0); 3727 } 3728 3729 int 3730 adapter_full_init(struct adapter *sc) 3731 { 3732 int rc, i; 3733 3734 ASSERT_SYNCHRONIZED_OP(sc); 3735 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3736 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 3737 ("%s: FULL_INIT_DONE already", __func__)); 3738 3739 /* 3740 * queues that belong to the adapter (not any particular port). 3741 */ 3742 rc = t4_setup_adapter_queues(sc); 3743 if (rc != 0) 3744 goto done; 3745 3746 for (i = 0; i < nitems(sc->tq); i++) { 3747 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 3748 taskqueue_thread_enqueue, &sc->tq[i]); 3749 if (sc->tq[i] == NULL) { 3750 device_printf(sc->dev, 3751 "failed to allocate task queue %d\n", i); 3752 rc = ENOMEM; 3753 goto done; 3754 } 3755 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3756 device_get_nameunit(sc->dev), i); 3757 } 3758 3759 t4_intr_enable(sc); 3760 sc->flags |= FULL_INIT_DONE; 3761 done: 3762 if (rc != 0) 3763 adapter_full_uninit(sc); 3764 3765 return (rc); 3766 } 3767 3768 int 3769 adapter_full_uninit(struct adapter *sc) 3770 { 3771 int i; 3772 3773 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3774 3775 t4_teardown_adapter_queues(sc); 3776 3777 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3778 taskqueue_free(sc->tq[i]); 3779 sc->tq[i] = NULL; 3780 } 3781 3782 sc->flags &= ~FULL_INIT_DONE; 3783 3784 return (0); 3785 } 3786 3787 #ifdef RSS 3788 #define SUPPORTED_RSS_HASHTYPES (RSS_HASHTYPE_RSS_IPV4 | \ 3789 RSS_HASHTYPE_RSS_TCP_IPV4 | RSS_HASHTYPE_RSS_IPV6 | \ 3790 RSS_HASHTYPE_RSS_TCP_IPV6 | RSS_HASHTYPE_RSS_UDP_IPV4 | \ 3791 RSS_HASHTYPE_RSS_UDP_IPV6) 3792 3793 /* Translates kernel hash types to hardware. */ 3794 static int 3795 hashconfig_to_hashen(int hashconfig) 3796 { 3797 int hashen = 0; 3798 3799 if (hashconfig & RSS_HASHTYPE_RSS_IPV4) 3800 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN; 3801 if (hashconfig & RSS_HASHTYPE_RSS_IPV6) 3802 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN; 3803 if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV4) { 3804 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN | 3805 F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN; 3806 } 3807 if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV6) { 3808 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN | 3809 F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN; 3810 } 3811 if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV4) 3812 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN; 3813 if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV6) 3814 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN; 3815 3816 return (hashen); 3817 } 3818 3819 /* Translates hardware hash types to kernel. */ 3820 static int 3821 hashen_to_hashconfig(int hashen) 3822 { 3823 int hashconfig = 0; 3824 3825 if (hashen & F_FW_RSS_VI_CONFIG_CMD_UDPEN) { 3826 /* 3827 * If UDP hashing was enabled it must have been enabled for 3828 * either IPv4 or IPv6 (inclusive or). Enabling UDP without 3829 * enabling any 4-tuple hash is nonsense configuration. 3830 */ 3831 MPASS(hashen & (F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN | 3832 F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)); 3833 3834 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) 3835 hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV4; 3836 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) 3837 hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV6; 3838 } 3839 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) 3840 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV4; 3841 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) 3842 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV6; 3843 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN) 3844 hashconfig |= RSS_HASHTYPE_RSS_IPV4; 3845 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN) 3846 hashconfig |= RSS_HASHTYPE_RSS_IPV6; 3847 3848 return (hashconfig); 3849 } 3850 #endif 3851 3852 int 3853 vi_full_init(struct vi_info *vi) 3854 { 3855 struct adapter *sc = vi->pi->adapter; 3856 struct ifnet *ifp = vi->ifp; 3857 uint16_t *rss; 3858 struct sge_rxq *rxq; 3859 int rc, i, j, hashen; 3860 #ifdef RSS 3861 int nbuckets = rss_getnumbuckets(); 3862 int hashconfig = rss_gethashconfig(); 3863 int extra; 3864 uint32_t raw_rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; 3865 uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; 3866 #endif 3867 3868 ASSERT_SYNCHRONIZED_OP(sc); 3869 KASSERT((vi->flags & VI_INIT_DONE) == 0, 3870 ("%s: VI_INIT_DONE already", __func__)); 3871 3872 sysctl_ctx_init(&vi->ctx); 3873 vi->flags |= VI_SYSCTL_CTX; 3874 3875 /* 3876 * Allocate tx/rx/fl queues for this VI. 3877 */ 3878 rc = t4_setup_vi_queues(vi); 3879 if (rc != 0) 3880 goto done; /* error message displayed already */ 3881 3882 #ifdef DEV_NETMAP 3883 /* Netmap VIs configure RSS when netmap is enabled. */ 3884 if (vi->flags & VI_NETMAP) { 3885 vi->flags |= VI_INIT_DONE; 3886 return (0); 3887 } 3888 #endif 3889 3890 /* 3891 * Setup RSS for this VI. Save a copy of the RSS table for later use. 3892 */ 3893 if (vi->nrxq > vi->rss_size) { 3894 if_printf(ifp, "nrxq (%d) > hw RSS table size (%d); " 3895 "some queues will never receive traffic.\n", vi->nrxq, 3896 vi->rss_size); 3897 } else if (vi->rss_size % vi->nrxq) { 3898 if_printf(ifp, "nrxq (%d), hw RSS table size (%d); " 3899 "expect uneven traffic distribution.\n", vi->nrxq, 3900 vi->rss_size); 3901 } 3902 #ifdef RSS 3903 MPASS(RSS_KEYSIZE == 40); 3904 if (vi->nrxq != nbuckets) { 3905 if_printf(ifp, "nrxq (%d) != kernel RSS buckets (%d);" 3906 "performance will be impacted.\n", vi->nrxq, nbuckets); 3907 } 3908 3909 rss_getkey((void *)&raw_rss_key[0]); 3910 for (i = 0; i < nitems(rss_key); i++) { 3911 rss_key[i] = htobe32(raw_rss_key[nitems(rss_key) - 1 - i]); 3912 } 3913 t4_write_rss_key(sc, (void *)&rss_key[0], -1); 3914 #endif 3915 rss = malloc(vi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); 3916 for (i = 0; i < vi->rss_size;) { 3917 #ifdef RSS 3918 j = rss_get_indirection_to_bucket(i); 3919 j %= pi->vrxq; 3920 rxq = &sc->sge.rxq[vi->first_rxq + j]; 3921 rss[i++] = rxq->iq.abs_id; 3922 #else 3923 for_each_rxq(vi, j, rxq) { 3924 rss[i++] = rxq->iq.abs_id; 3925 if (i == vi->rss_size) 3926 break; 3927 } 3928 #endif 3929 } 3930 3931 rc = -t4_config_rss_range(sc, sc->mbox, vi->viid, 0, vi->rss_size, rss, 3932 vi->rss_size); 3933 if (rc != 0) { 3934 if_printf(ifp, "rss_config failed: %d\n", rc); 3935 goto done; 3936 } 3937 3938 #ifdef RSS 3939 hashen = hashconfig_to_hashen(hashconfig); 3940 3941 /* 3942 * We may have had to enable some hashes even though the global config 3943 * wants them disabled. This is a potential problem that must be 3944 * reported to the user. 3945 */ 3946 extra = hashen_to_hashconfig(hashen) ^ hashconfig; 3947 3948 /* 3949 * If we consider only the supported hash types, then the enabled hashes 3950 * are a superset of the requested hashes. In other words, there cannot 3951 * be any supported hash that was requested but not enabled, but there 3952 * can be hashes that were not requested but had to be enabled. 3953 */ 3954 extra &= SUPPORTED_RSS_HASHTYPES; 3955 MPASS((extra & hashconfig) == 0); 3956 3957 if (extra) { 3958 if_printf(ifp, 3959 "global RSS config (0x%x) cannot be accomodated.\n", 3960 hashconfig); 3961 } 3962 if (extra & RSS_HASHTYPE_RSS_IPV4) 3963 if_printf(ifp, "IPv4 2-tuple hashing forced on.\n"); 3964 if (extra & RSS_HASHTYPE_RSS_TCP_IPV4) 3965 if_printf(ifp, "TCP/IPv4 4-tuple hashing forced on.\n"); 3966 if (extra & RSS_HASHTYPE_RSS_IPV6) 3967 if_printf(ifp, "IPv6 2-tuple hashing forced on.\n"); 3968 if (extra & RSS_HASHTYPE_RSS_TCP_IPV6) 3969 if_printf(ifp, "TCP/IPv6 4-tuple hashing forced on.\n"); 3970 if (extra & RSS_HASHTYPE_RSS_UDP_IPV4) 3971 if_printf(ifp, "UDP/IPv4 4-tuple hashing forced on.\n"); 3972 if (extra & RSS_HASHTYPE_RSS_UDP_IPV6) 3973 if_printf(ifp, "UDP/IPv6 4-tuple hashing forced on.\n"); 3974 #else 3975 hashen = F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN | 3976 F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN | 3977 F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN | 3978 F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN | F_FW_RSS_VI_CONFIG_CMD_UDPEN; 3979 #endif 3980 rc = -t4_config_vi_rss(sc, sc->mbox, vi->viid, hashen, rss[0]); 3981 if (rc != 0) { 3982 if_printf(ifp, "rss hash/defaultq config failed: %d\n", rc); 3983 goto done; 3984 } 3985 3986 vi->rss = rss; 3987 vi->flags |= VI_INIT_DONE; 3988 done: 3989 if (rc != 0) 3990 vi_full_uninit(vi); 3991 3992 return (rc); 3993 } 3994 3995 /* 3996 * Idempotent. 3997 */ 3998 int 3999 vi_full_uninit(struct vi_info *vi) 4000 { 4001 struct port_info *pi = vi->pi; 4002 struct adapter *sc = pi->adapter; 4003 int i; 4004 struct sge_rxq *rxq; 4005 struct sge_txq *txq; 4006 #ifdef TCP_OFFLOAD 4007 struct sge_ofld_rxq *ofld_rxq; 4008 struct sge_wrq *ofld_txq; 4009 #endif 4010 4011 if (vi->flags & VI_INIT_DONE) { 4012 4013 /* Need to quiesce queues. */ 4014 #ifdef DEV_NETMAP 4015 if (vi->flags & VI_NETMAP) 4016 goto skip; 4017 #endif 4018 4019 /* XXX: Only for the first VI? */ 4020 if (IS_MAIN_VI(vi)) 4021 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]); 4022 4023 for_each_txq(vi, i, txq) { 4024 quiesce_txq(sc, txq); 4025 } 4026 4027 #ifdef TCP_OFFLOAD 4028 for_each_ofld_txq(vi, i, ofld_txq) { 4029 quiesce_wrq(sc, ofld_txq); 4030 } 4031 #endif 4032 4033 for_each_rxq(vi, i, rxq) { 4034 quiesce_iq(sc, &rxq->iq); 4035 quiesce_fl(sc, &rxq->fl); 4036 } 4037 4038 #ifdef TCP_OFFLOAD 4039 for_each_ofld_rxq(vi, i, ofld_rxq) { 4040 quiesce_iq(sc, &ofld_rxq->iq); 4041 quiesce_fl(sc, &ofld_rxq->fl); 4042 } 4043 #endif 4044 free(vi->rss, M_CXGBE); 4045 } 4046 #ifdef DEV_NETMAP 4047 skip: 4048 #endif 4049 4050 t4_teardown_vi_queues(vi); 4051 vi->flags &= ~VI_INIT_DONE; 4052 4053 return (0); 4054 } 4055 4056 static void 4057 quiesce_txq(struct adapter *sc, struct sge_txq *txq) 4058 { 4059 struct sge_eq *eq = &txq->eq; 4060 struct sge_qstat *spg = (void *)&eq->desc[eq->sidx]; 4061 4062 (void) sc; /* unused */ 4063 4064 #ifdef INVARIANTS 4065 TXQ_LOCK(txq); 4066 MPASS((eq->flags & EQ_ENABLED) == 0); 4067 TXQ_UNLOCK(txq); 4068 #endif 4069 4070 /* Wait for the mp_ring to empty. */ 4071 while (!mp_ring_is_idle(txq->r)) { 4072 mp_ring_check_drainage(txq->r, 0); 4073 pause("rquiesce", 1); 4074 } 4075 4076 /* Then wait for the hardware to finish. */ 4077 while (spg->cidx != htobe16(eq->pidx)) 4078 pause("equiesce", 1); 4079 4080 /* Finally, wait for the driver to reclaim all descriptors. */ 4081 while (eq->cidx != eq->pidx) 4082 pause("dquiesce", 1); 4083 } 4084 4085 static void 4086 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq) 4087 { 4088 4089 /* XXXTX */ 4090 } 4091 4092 static void 4093 quiesce_iq(struct adapter *sc, struct sge_iq *iq) 4094 { 4095 (void) sc; /* unused */ 4096 4097 /* Synchronize with the interrupt handler */ 4098 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 4099 pause("iqfree", 1); 4100 } 4101 4102 static void 4103 quiesce_fl(struct adapter *sc, struct sge_fl *fl) 4104 { 4105 mtx_lock(&sc->sfl_lock); 4106 FL_LOCK(fl); 4107 fl->flags |= FL_DOOMED; 4108 FL_UNLOCK(fl); 4109 callout_stop(&sc->sfl_callout); 4110 mtx_unlock(&sc->sfl_lock); 4111 4112 KASSERT((fl->flags & FL_STARVING) == 0, 4113 ("%s: still starving", __func__)); 4114 } 4115 4116 static int 4117 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 4118 driver_intr_t *handler, void *arg, char *name) 4119 { 4120 int rc; 4121 4122 irq->rid = rid; 4123 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 4124 RF_SHAREABLE | RF_ACTIVE); 4125 if (irq->res == NULL) { 4126 device_printf(sc->dev, 4127 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 4128 return (ENOMEM); 4129 } 4130 4131 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 4132 NULL, handler, arg, &irq->tag); 4133 if (rc != 0) { 4134 device_printf(sc->dev, 4135 "failed to setup interrupt for rid %d, name %s: %d\n", 4136 rid, name, rc); 4137 } else if (name) 4138 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 4139 4140 return (rc); 4141 } 4142 4143 static int 4144 t4_free_irq(struct adapter *sc, struct irq *irq) 4145 { 4146 if (irq->tag) 4147 bus_teardown_intr(sc->dev, irq->res, irq->tag); 4148 if (irq->res) 4149 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 4150 4151 bzero(irq, sizeof(*irq)); 4152 4153 return (0); 4154 } 4155 4156 static void 4157 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 4158 unsigned int end) 4159 { 4160 uint32_t *p = (uint32_t *)(buf + start); 4161 4162 for ( ; start <= end; start += sizeof(uint32_t)) 4163 *p++ = t4_read_reg(sc, start); 4164 } 4165 4166 static void 4167 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 4168 { 4169 int i, n; 4170 const unsigned int *reg_ranges; 4171 static const unsigned int t4_reg_ranges[] = { 4172 0x1008, 0x1108, 4173 0x1180, 0x11b4, 4174 0x11fc, 0x123c, 4175 0x1300, 0x173c, 4176 0x1800, 0x18fc, 4177 0x3000, 0x30d8, 4178 0x30e0, 0x5924, 4179 0x5960, 0x59d4, 4180 0x5a00, 0x5af8, 4181 0x6000, 0x6098, 4182 0x6100, 0x6150, 4183 0x6200, 0x6208, 4184 0x6240, 0x6248, 4185 0x6280, 0x6338, 4186 0x6370, 0x638c, 4187 0x6400, 0x643c, 4188 0x6500, 0x6524, 4189 0x6a00, 0x6a38, 4190 0x6a60, 0x6a78, 4191 0x6b00, 0x6b84, 4192 0x6bf0, 0x6c84, 4193 0x6cf0, 0x6d84, 4194 0x6df0, 0x6e84, 4195 0x6ef0, 0x6f84, 4196 0x6ff0, 0x7084, 4197 0x70f0, 0x7184, 4198 0x71f0, 0x7284, 4199 0x72f0, 0x7384, 4200 0x73f0, 0x7450, 4201 0x7500, 0x7530, 4202 0x7600, 0x761c, 4203 0x7680, 0x76cc, 4204 0x7700, 0x7798, 4205 0x77c0, 0x77fc, 4206 0x7900, 0x79fc, 4207 0x7b00, 0x7c38, 4208 0x7d00, 0x7efc, 4209 0x8dc0, 0x8e1c, 4210 0x8e30, 0x8e78, 4211 0x8ea0, 0x8f6c, 4212 0x8fc0, 0x9074, 4213 0x90fc, 0x90fc, 4214 0x9400, 0x9458, 4215 0x9600, 0x96bc, 4216 0x9800, 0x9808, 4217 0x9820, 0x983c, 4218 0x9850, 0x9864, 4219 0x9c00, 0x9c6c, 4220 0x9c80, 0x9cec, 4221 0x9d00, 0x9d6c, 4222 0x9d80, 0x9dec, 4223 0x9e00, 0x9e6c, 4224 0x9e80, 0x9eec, 4225 0x9f00, 0x9f6c, 4226 0x9f80, 0x9fec, 4227 0xd004, 0xd03c, 4228 0xdfc0, 0xdfe0, 4229 0xe000, 0xea7c, 4230 0xf000, 0x11110, 4231 0x11118, 0x11190, 4232 0x19040, 0x1906c, 4233 0x19078, 0x19080, 4234 0x1908c, 0x19124, 4235 0x19150, 0x191b0, 4236 0x191d0, 0x191e8, 4237 0x19238, 0x1924c, 4238 0x193f8, 0x19474, 4239 0x19490, 0x194f8, 4240 0x19800, 0x19f30, 4241 0x1a000, 0x1a06c, 4242 0x1a0b0, 0x1a120, 4243 0x1a128, 0x1a138, 4244 0x1a190, 0x1a1c4, 4245 0x1a1fc, 0x1a1fc, 4246 0x1e040, 0x1e04c, 4247 0x1e284, 0x1e28c, 4248 0x1e2c0, 0x1e2c0, 4249 0x1e2e0, 0x1e2e0, 4250 0x1e300, 0x1e384, 4251 0x1e3c0, 0x1e3c8, 4252 0x1e440, 0x1e44c, 4253 0x1e684, 0x1e68c, 4254 0x1e6c0, 0x1e6c0, 4255 0x1e6e0, 0x1e6e0, 4256 0x1e700, 0x1e784, 4257 0x1e7c0, 0x1e7c8, 4258 0x1e840, 0x1e84c, 4259 0x1ea84, 0x1ea8c, 4260 0x1eac0, 0x1eac0, 4261 0x1eae0, 0x1eae0, 4262 0x1eb00, 0x1eb84, 4263 0x1ebc0, 0x1ebc8, 4264 0x1ec40, 0x1ec4c, 4265 0x1ee84, 0x1ee8c, 4266 0x1eec0, 0x1eec0, 4267 0x1eee0, 0x1eee0, 4268 0x1ef00, 0x1ef84, 4269 0x1efc0, 0x1efc8, 4270 0x1f040, 0x1f04c, 4271 0x1f284, 0x1f28c, 4272 0x1f2c0, 0x1f2c0, 4273 0x1f2e0, 0x1f2e0, 4274 0x1f300, 0x1f384, 4275 0x1f3c0, 0x1f3c8, 4276 0x1f440, 0x1f44c, 4277 0x1f684, 0x1f68c, 4278 0x1f6c0, 0x1f6c0, 4279 0x1f6e0, 0x1f6e0, 4280 0x1f700, 0x1f784, 4281 0x1f7c0, 0x1f7c8, 4282 0x1f840, 0x1f84c, 4283 0x1fa84, 0x1fa8c, 4284 0x1fac0, 0x1fac0, 4285 0x1fae0, 0x1fae0, 4286 0x1fb00, 0x1fb84, 4287 0x1fbc0, 0x1fbc8, 4288 0x1fc40, 0x1fc4c, 4289 0x1fe84, 0x1fe8c, 4290 0x1fec0, 0x1fec0, 4291 0x1fee0, 0x1fee0, 4292 0x1ff00, 0x1ff84, 4293 0x1ffc0, 0x1ffc8, 4294 0x20000, 0x2002c, 4295 0x20100, 0x2013c, 4296 0x20190, 0x201c8, 4297 0x20200, 0x20318, 4298 0x20400, 0x20528, 4299 0x20540, 0x20614, 4300 0x21000, 0x21040, 4301 0x2104c, 0x21060, 4302 0x210c0, 0x210ec, 4303 0x21200, 0x21268, 4304 0x21270, 0x21284, 4305 0x212fc, 0x21388, 4306 0x21400, 0x21404, 4307 0x21500, 0x21518, 4308 0x2152c, 0x2153c, 4309 0x21550, 0x21554, 4310 0x21600, 0x21600, 4311 0x21608, 0x21628, 4312 0x21630, 0x2163c, 4313 0x21700, 0x2171c, 4314 0x21780, 0x2178c, 4315 0x21800, 0x21c38, 4316 0x21c80, 0x21d7c, 4317 0x21e00, 0x21e04, 4318 0x22000, 0x2202c, 4319 0x22100, 0x2213c, 4320 0x22190, 0x221c8, 4321 0x22200, 0x22318, 4322 0x22400, 0x22528, 4323 0x22540, 0x22614, 4324 0x23000, 0x23040, 4325 0x2304c, 0x23060, 4326 0x230c0, 0x230ec, 4327 0x23200, 0x23268, 4328 0x23270, 0x23284, 4329 0x232fc, 0x23388, 4330 0x23400, 0x23404, 4331 0x23500, 0x23518, 4332 0x2352c, 0x2353c, 4333 0x23550, 0x23554, 4334 0x23600, 0x23600, 4335 0x23608, 0x23628, 4336 0x23630, 0x2363c, 4337 0x23700, 0x2371c, 4338 0x23780, 0x2378c, 4339 0x23800, 0x23c38, 4340 0x23c80, 0x23d7c, 4341 0x23e00, 0x23e04, 4342 0x24000, 0x2402c, 4343 0x24100, 0x2413c, 4344 0x24190, 0x241c8, 4345 0x24200, 0x24318, 4346 0x24400, 0x24528, 4347 0x24540, 0x24614, 4348 0x25000, 0x25040, 4349 0x2504c, 0x25060, 4350 0x250c0, 0x250ec, 4351 0x25200, 0x25268, 4352 0x25270, 0x25284, 4353 0x252fc, 0x25388, 4354 0x25400, 0x25404, 4355 0x25500, 0x25518, 4356 0x2552c, 0x2553c, 4357 0x25550, 0x25554, 4358 0x25600, 0x25600, 4359 0x25608, 0x25628, 4360 0x25630, 0x2563c, 4361 0x25700, 0x2571c, 4362 0x25780, 0x2578c, 4363 0x25800, 0x25c38, 4364 0x25c80, 0x25d7c, 4365 0x25e00, 0x25e04, 4366 0x26000, 0x2602c, 4367 0x26100, 0x2613c, 4368 0x26190, 0x261c8, 4369 0x26200, 0x26318, 4370 0x26400, 0x26528, 4371 0x26540, 0x26614, 4372 0x27000, 0x27040, 4373 0x2704c, 0x27060, 4374 0x270c0, 0x270ec, 4375 0x27200, 0x27268, 4376 0x27270, 0x27284, 4377 0x272fc, 0x27388, 4378 0x27400, 0x27404, 4379 0x27500, 0x27518, 4380 0x2752c, 0x2753c, 4381 0x27550, 0x27554, 4382 0x27600, 0x27600, 4383 0x27608, 0x27628, 4384 0x27630, 0x2763c, 4385 0x27700, 0x2771c, 4386 0x27780, 0x2778c, 4387 0x27800, 0x27c38, 4388 0x27c80, 0x27d7c, 4389 0x27e00, 0x27e04 4390 }; 4391 static const unsigned int t5_reg_ranges[] = { 4392 0x1008, 0x1148, 4393 0x1180, 0x11b4, 4394 0x11fc, 0x123c, 4395 0x1280, 0x173c, 4396 0x1800, 0x18fc, 4397 0x3000, 0x3028, 4398 0x3060, 0x30d8, 4399 0x30e0, 0x30fc, 4400 0x3140, 0x357c, 4401 0x35a8, 0x35cc, 4402 0x35ec, 0x35ec, 4403 0x3600, 0x5624, 4404 0x56cc, 0x575c, 4405 0x580c, 0x5814, 4406 0x5890, 0x58bc, 4407 0x5940, 0x59dc, 4408 0x59fc, 0x5a18, 4409 0x5a60, 0x5a9c, 4410 0x5b94, 0x5bfc, 4411 0x6000, 0x6040, 4412 0x6058, 0x614c, 4413 0x7700, 0x7798, 4414 0x77c0, 0x78fc, 4415 0x7b00, 0x7c54, 4416 0x7d00, 0x7efc, 4417 0x8dc0, 0x8de0, 4418 0x8df8, 0x8e84, 4419 0x8ea0, 0x8f84, 4420 0x8fc0, 0x90f8, 4421 0x9400, 0x9470, 4422 0x9600, 0x96f4, 4423 0x9800, 0x9808, 4424 0x9820, 0x983c, 4425 0x9850, 0x9864, 4426 0x9c00, 0x9c6c, 4427 0x9c80, 0x9cec, 4428 0x9d00, 0x9d6c, 4429 0x9d80, 0x9dec, 4430 0x9e00, 0x9e6c, 4431 0x9e80, 0x9eec, 4432 0x9f00, 0x9f6c, 4433 0x9f80, 0xa020, 4434 0xd004, 0xd03c, 4435 0xdfc0, 0xdfe0, 4436 0xe000, 0x11088, 4437 0x1109c, 0x11110, 4438 0x11118, 0x1117c, 4439 0x11190, 0x11204, 4440 0x19040, 0x1906c, 4441 0x19078, 0x19080, 4442 0x1908c, 0x19124, 4443 0x19150, 0x191b0, 4444 0x191d0, 0x191e8, 4445 0x19238, 0x19290, 4446 0x193f8, 0x19474, 4447 0x19490, 0x194cc, 4448 0x194f0, 0x194f8, 4449 0x19c00, 0x19c60, 4450 0x19c94, 0x19e10, 4451 0x19e50, 0x19f34, 4452 0x19f40, 0x19f50, 4453 0x19f90, 0x19fe4, 4454 0x1a000, 0x1a06c, 4455 0x1a0b0, 0x1a120, 4456 0x1a128, 0x1a138, 4457 0x1a190, 0x1a1c4, 4458 0x1a1fc, 0x1a1fc, 4459 0x1e008, 0x1e00c, 4460 0x1e040, 0x1e04c, 4461 0x1e284, 0x1e290, 4462 0x1e2c0, 0x1e2c0, 4463 0x1e2e0, 0x1e2e0, 4464 0x1e300, 0x1e384, 4465 0x1e3c0, 0x1e3c8, 4466 0x1e408, 0x1e40c, 4467 0x1e440, 0x1e44c, 4468 0x1e684, 0x1e690, 4469 0x1e6c0, 0x1e6c0, 4470 0x1e6e0, 0x1e6e0, 4471 0x1e700, 0x1e784, 4472 0x1e7c0, 0x1e7c8, 4473 0x1e808, 0x1e80c, 4474 0x1e840, 0x1e84c, 4475 0x1ea84, 0x1ea90, 4476 0x1eac0, 0x1eac0, 4477 0x1eae0, 0x1eae0, 4478 0x1eb00, 0x1eb84, 4479 0x1ebc0, 0x1ebc8, 4480 0x1ec08, 0x1ec0c, 4481 0x1ec40, 0x1ec4c, 4482 0x1ee84, 0x1ee90, 4483 0x1eec0, 0x1eec0, 4484 0x1eee0, 0x1eee0, 4485 0x1ef00, 0x1ef84, 4486 0x1efc0, 0x1efc8, 4487 0x1f008, 0x1f00c, 4488 0x1f040, 0x1f04c, 4489 0x1f284, 0x1f290, 4490 0x1f2c0, 0x1f2c0, 4491 0x1f2e0, 0x1f2e0, 4492 0x1f300, 0x1f384, 4493 0x1f3c0, 0x1f3c8, 4494 0x1f408, 0x1f40c, 4495 0x1f440, 0x1f44c, 4496 0x1f684, 0x1f690, 4497 0x1f6c0, 0x1f6c0, 4498 0x1f6e0, 0x1f6e0, 4499 0x1f700, 0x1f784, 4500 0x1f7c0, 0x1f7c8, 4501 0x1f808, 0x1f80c, 4502 0x1f840, 0x1f84c, 4503 0x1fa84, 0x1fa90, 4504 0x1fac0, 0x1fac0, 4505 0x1fae0, 0x1fae0, 4506 0x1fb00, 0x1fb84, 4507 0x1fbc0, 0x1fbc8, 4508 0x1fc08, 0x1fc0c, 4509 0x1fc40, 0x1fc4c, 4510 0x1fe84, 0x1fe90, 4511 0x1fec0, 0x1fec0, 4512 0x1fee0, 0x1fee0, 4513 0x1ff00, 0x1ff84, 4514 0x1ffc0, 0x1ffc8, 4515 0x30000, 0x30030, 4516 0x30100, 0x30144, 4517 0x30190, 0x301d0, 4518 0x30200, 0x30318, 4519 0x30400, 0x3052c, 4520 0x30540, 0x3061c, 4521 0x30800, 0x30834, 4522 0x308c0, 0x30908, 4523 0x30910, 0x309ac, 4524 0x30a00, 0x30a2c, 4525 0x30a44, 0x30a50, 4526 0x30a74, 0x30c24, 4527 0x30d00, 0x30d00, 4528 0x30d08, 0x30d14, 4529 0x30d1c, 0x30d20, 4530 0x30d3c, 0x30d50, 4531 0x31200, 0x3120c, 4532 0x31220, 0x31220, 4533 0x31240, 0x31240, 4534 0x31600, 0x3160c, 4535 0x31a00, 0x31a1c, 4536 0x31e00, 0x31e20, 4537 0x31e38, 0x31e3c, 4538 0x31e80, 0x31e80, 4539 0x31e88, 0x31ea8, 4540 0x31eb0, 0x31eb4, 4541 0x31ec8, 0x31ed4, 4542 0x31fb8, 0x32004, 4543 0x32200, 0x32200, 4544 0x32208, 0x32240, 4545 0x32248, 0x32280, 4546 0x32288, 0x322c0, 4547 0x322c8, 0x322fc, 4548 0x32600, 0x32630, 4549 0x32a00, 0x32abc, 4550 0x32b00, 0x32b70, 4551 0x33000, 0x33048, 4552 0x33060, 0x3309c, 4553 0x330f0, 0x33148, 4554 0x33160, 0x3319c, 4555 0x331f0, 0x332e4, 4556 0x332f8, 0x333e4, 4557 0x333f8, 0x33448, 4558 0x33460, 0x3349c, 4559 0x334f0, 0x33548, 4560 0x33560, 0x3359c, 4561 0x335f0, 0x336e4, 4562 0x336f8, 0x337e4, 4563 0x337f8, 0x337fc, 4564 0x33814, 0x33814, 4565 0x3382c, 0x3382c, 4566 0x33880, 0x3388c, 4567 0x338e8, 0x338ec, 4568 0x33900, 0x33948, 4569 0x33960, 0x3399c, 4570 0x339f0, 0x33ae4, 4571 0x33af8, 0x33b10, 4572 0x33b28, 0x33b28, 4573 0x33b3c, 0x33b50, 4574 0x33bf0, 0x33c10, 4575 0x33c28, 0x33c28, 4576 0x33c3c, 0x33c50, 4577 0x33cf0, 0x33cfc, 4578 0x34000, 0x34030, 4579 0x34100, 0x34144, 4580 0x34190, 0x341d0, 4581 0x34200, 0x34318, 4582 0x34400, 0x3452c, 4583 0x34540, 0x3461c, 4584 0x34800, 0x34834, 4585 0x348c0, 0x34908, 4586 0x34910, 0x349ac, 4587 0x34a00, 0x34a2c, 4588 0x34a44, 0x34a50, 4589 0x34a74, 0x34c24, 4590 0x34d00, 0x34d00, 4591 0x34d08, 0x34d14, 4592 0x34d1c, 0x34d20, 4593 0x34d3c, 0x34d50, 4594 0x35200, 0x3520c, 4595 0x35220, 0x35220, 4596 0x35240, 0x35240, 4597 0x35600, 0x3560c, 4598 0x35a00, 0x35a1c, 4599 0x35e00, 0x35e20, 4600 0x35e38, 0x35e3c, 4601 0x35e80, 0x35e80, 4602 0x35e88, 0x35ea8, 4603 0x35eb0, 0x35eb4, 4604 0x35ec8, 0x35ed4, 4605 0x35fb8, 0x36004, 4606 0x36200, 0x36200, 4607 0x36208, 0x36240, 4608 0x36248, 0x36280, 4609 0x36288, 0x362c0, 4610 0x362c8, 0x362fc, 4611 0x36600, 0x36630, 4612 0x36a00, 0x36abc, 4613 0x36b00, 0x36b70, 4614 0x37000, 0x37048, 4615 0x37060, 0x3709c, 4616 0x370f0, 0x37148, 4617 0x37160, 0x3719c, 4618 0x371f0, 0x372e4, 4619 0x372f8, 0x373e4, 4620 0x373f8, 0x37448, 4621 0x37460, 0x3749c, 4622 0x374f0, 0x37548, 4623 0x37560, 0x3759c, 4624 0x375f0, 0x376e4, 4625 0x376f8, 0x377e4, 4626 0x377f8, 0x377fc, 4627 0x37814, 0x37814, 4628 0x3782c, 0x3782c, 4629 0x37880, 0x3788c, 4630 0x378e8, 0x378ec, 4631 0x37900, 0x37948, 4632 0x37960, 0x3799c, 4633 0x379f0, 0x37ae4, 4634 0x37af8, 0x37b10, 4635 0x37b28, 0x37b28, 4636 0x37b3c, 0x37b50, 4637 0x37bf0, 0x37c10, 4638 0x37c28, 0x37c28, 4639 0x37c3c, 0x37c50, 4640 0x37cf0, 0x37cfc, 4641 0x38000, 0x38030, 4642 0x38100, 0x38144, 4643 0x38190, 0x381d0, 4644 0x38200, 0x38318, 4645 0x38400, 0x3852c, 4646 0x38540, 0x3861c, 4647 0x38800, 0x38834, 4648 0x388c0, 0x38908, 4649 0x38910, 0x389ac, 4650 0x38a00, 0x38a2c, 4651 0x38a44, 0x38a50, 4652 0x38a74, 0x38c24, 4653 0x38d00, 0x38d00, 4654 0x38d08, 0x38d14, 4655 0x38d1c, 0x38d20, 4656 0x38d3c, 0x38d50, 4657 0x39200, 0x3920c, 4658 0x39220, 0x39220, 4659 0x39240, 0x39240, 4660 0x39600, 0x3960c, 4661 0x39a00, 0x39a1c, 4662 0x39e00, 0x39e20, 4663 0x39e38, 0x39e3c, 4664 0x39e80, 0x39e80, 4665 0x39e88, 0x39ea8, 4666 0x39eb0, 0x39eb4, 4667 0x39ec8, 0x39ed4, 4668 0x39fb8, 0x3a004, 4669 0x3a200, 0x3a200, 4670 0x3a208, 0x3a240, 4671 0x3a248, 0x3a280, 4672 0x3a288, 0x3a2c0, 4673 0x3a2c8, 0x3a2fc, 4674 0x3a600, 0x3a630, 4675 0x3aa00, 0x3aabc, 4676 0x3ab00, 0x3ab70, 4677 0x3b000, 0x3b048, 4678 0x3b060, 0x3b09c, 4679 0x3b0f0, 0x3b148, 4680 0x3b160, 0x3b19c, 4681 0x3b1f0, 0x3b2e4, 4682 0x3b2f8, 0x3b3e4, 4683 0x3b3f8, 0x3b448, 4684 0x3b460, 0x3b49c, 4685 0x3b4f0, 0x3b548, 4686 0x3b560, 0x3b59c, 4687 0x3b5f0, 0x3b6e4, 4688 0x3b6f8, 0x3b7e4, 4689 0x3b7f8, 0x3b7fc, 4690 0x3b814, 0x3b814, 4691 0x3b82c, 0x3b82c, 4692 0x3b880, 0x3b88c, 4693 0x3b8e8, 0x3b8ec, 4694 0x3b900, 0x3b948, 4695 0x3b960, 0x3b99c, 4696 0x3b9f0, 0x3bae4, 4697 0x3baf8, 0x3bb10, 4698 0x3bb28, 0x3bb28, 4699 0x3bb3c, 0x3bb50, 4700 0x3bbf0, 0x3bc10, 4701 0x3bc28, 0x3bc28, 4702 0x3bc3c, 0x3bc50, 4703 0x3bcf0, 0x3bcfc, 4704 0x3c000, 0x3c030, 4705 0x3c100, 0x3c144, 4706 0x3c190, 0x3c1d0, 4707 0x3c200, 0x3c318, 4708 0x3c400, 0x3c52c, 4709 0x3c540, 0x3c61c, 4710 0x3c800, 0x3c834, 4711 0x3c8c0, 0x3c908, 4712 0x3c910, 0x3c9ac, 4713 0x3ca00, 0x3ca2c, 4714 0x3ca44, 0x3ca50, 4715 0x3ca74, 0x3cc24, 4716 0x3cd00, 0x3cd00, 4717 0x3cd08, 0x3cd14, 4718 0x3cd1c, 0x3cd20, 4719 0x3cd3c, 0x3cd50, 4720 0x3d200, 0x3d20c, 4721 0x3d220, 0x3d220, 4722 0x3d240, 0x3d240, 4723 0x3d600, 0x3d60c, 4724 0x3da00, 0x3da1c, 4725 0x3de00, 0x3de20, 4726 0x3de38, 0x3de3c, 4727 0x3de80, 0x3de80, 4728 0x3de88, 0x3dea8, 4729 0x3deb0, 0x3deb4, 4730 0x3dec8, 0x3ded4, 4731 0x3dfb8, 0x3e004, 4732 0x3e200, 0x3e200, 4733 0x3e208, 0x3e240, 4734 0x3e248, 0x3e280, 4735 0x3e288, 0x3e2c0, 4736 0x3e2c8, 0x3e2fc, 4737 0x3e600, 0x3e630, 4738 0x3ea00, 0x3eabc, 4739 0x3eb00, 0x3eb70, 4740 0x3f000, 0x3f048, 4741 0x3f060, 0x3f09c, 4742 0x3f0f0, 0x3f148, 4743 0x3f160, 0x3f19c, 4744 0x3f1f0, 0x3f2e4, 4745 0x3f2f8, 0x3f3e4, 4746 0x3f3f8, 0x3f448, 4747 0x3f460, 0x3f49c, 4748 0x3f4f0, 0x3f548, 4749 0x3f560, 0x3f59c, 4750 0x3f5f0, 0x3f6e4, 4751 0x3f6f8, 0x3f7e4, 4752 0x3f7f8, 0x3f7fc, 4753 0x3f814, 0x3f814, 4754 0x3f82c, 0x3f82c, 4755 0x3f880, 0x3f88c, 4756 0x3f8e8, 0x3f8ec, 4757 0x3f900, 0x3f948, 4758 0x3f960, 0x3f99c, 4759 0x3f9f0, 0x3fae4, 4760 0x3faf8, 0x3fb10, 4761 0x3fb28, 0x3fb28, 4762 0x3fb3c, 0x3fb50, 4763 0x3fbf0, 0x3fc10, 4764 0x3fc28, 0x3fc28, 4765 0x3fc3c, 0x3fc50, 4766 0x3fcf0, 0x3fcfc, 4767 0x40000, 0x4000c, 4768 0x40040, 0x40068, 4769 0x4007c, 0x40144, 4770 0x40180, 0x4018c, 4771 0x40200, 0x40298, 4772 0x402ac, 0x4033c, 4773 0x403f8, 0x403fc, 4774 0x41304, 0x413c4, 4775 0x41400, 0x4141c, 4776 0x41480, 0x414d0, 4777 0x44000, 0x44078, 4778 0x440c0, 0x44278, 4779 0x442c0, 0x44478, 4780 0x444c0, 0x44678, 4781 0x446c0, 0x44878, 4782 0x448c0, 0x449fc, 4783 0x45000, 0x45068, 4784 0x45080, 0x45084, 4785 0x450a0, 0x450b0, 4786 0x45200, 0x45268, 4787 0x45280, 0x45284, 4788 0x452a0, 0x452b0, 4789 0x460c0, 0x460e4, 4790 0x47000, 0x4708c, 4791 0x47200, 0x47250, 4792 0x47400, 0x47420, 4793 0x47600, 0x47618, 4794 0x47800, 0x47814, 4795 0x48000, 0x4800c, 4796 0x48040, 0x48068, 4797 0x4807c, 0x48144, 4798 0x48180, 0x4818c, 4799 0x48200, 0x48298, 4800 0x482ac, 0x4833c, 4801 0x483f8, 0x483fc, 4802 0x49304, 0x493c4, 4803 0x49400, 0x4941c, 4804 0x49480, 0x494d0, 4805 0x4c000, 0x4c078, 4806 0x4c0c0, 0x4c278, 4807 0x4c2c0, 0x4c478, 4808 0x4c4c0, 0x4c678, 4809 0x4c6c0, 0x4c878, 4810 0x4c8c0, 0x4c9fc, 4811 0x4d000, 0x4d068, 4812 0x4d080, 0x4d084, 4813 0x4d0a0, 0x4d0b0, 4814 0x4d200, 0x4d268, 4815 0x4d280, 0x4d284, 4816 0x4d2a0, 0x4d2b0, 4817 0x4e0c0, 0x4e0e4, 4818 0x4f000, 0x4f08c, 4819 0x4f200, 0x4f250, 4820 0x4f400, 0x4f420, 4821 0x4f600, 0x4f618, 4822 0x4f800, 0x4f814, 4823 0x50000, 0x500cc, 4824 0x50400, 0x50400, 4825 0x50800, 0x508cc, 4826 0x50c00, 0x50c00, 4827 0x51000, 0x5101c, 4828 0x51300, 0x51308, 4829 }; 4830 4831 if (is_t4(sc)) { 4832 reg_ranges = &t4_reg_ranges[0]; 4833 n = nitems(t4_reg_ranges); 4834 } else { 4835 reg_ranges = &t5_reg_ranges[0]; 4836 n = nitems(t5_reg_ranges); 4837 } 4838 4839 regs->version = chip_id(sc) | chip_rev(sc) << 10; 4840 for (i = 0; i < n; i += 2) 4841 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 4842 } 4843 4844 #define A_PL_INDIR_CMD 0x1f8 4845 4846 #define S_PL_AUTOINC 31 4847 #define M_PL_AUTOINC 0x1U 4848 #define V_PL_AUTOINC(x) ((x) << S_PL_AUTOINC) 4849 #define G_PL_AUTOINC(x) (((x) >> S_PL_AUTOINC) & M_PL_AUTOINC) 4850 4851 #define S_PL_VFID 20 4852 #define M_PL_VFID 0xffU 4853 #define V_PL_VFID(x) ((x) << S_PL_VFID) 4854 #define G_PL_VFID(x) (((x) >> S_PL_VFID) & M_PL_VFID) 4855 4856 #define S_PL_ADDR 0 4857 #define M_PL_ADDR 0xfffffU 4858 #define V_PL_ADDR(x) ((x) << S_PL_ADDR) 4859 #define G_PL_ADDR(x) (((x) >> S_PL_ADDR) & M_PL_ADDR) 4860 4861 #define A_PL_INDIR_DATA 0x1fc 4862 4863 static uint64_t 4864 read_vf_stat(struct adapter *sc, unsigned int viid, int reg) 4865 { 4866 u32 stats[2]; 4867 4868 mtx_assert(&sc->regwin_lock, MA_OWNED); 4869 t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) | 4870 V_PL_VFID(G_FW_VIID_VIN(viid)) | V_PL_ADDR(VF_MPS_REG(reg))); 4871 stats[0] = t4_read_reg(sc, A_PL_INDIR_DATA); 4872 stats[1] = t4_read_reg(sc, A_PL_INDIR_DATA); 4873 return (((uint64_t)stats[1]) << 32 | stats[0]); 4874 } 4875 4876 static void 4877 t4_get_vi_stats(struct adapter *sc, unsigned int viid, 4878 struct fw_vi_stats_vf *stats) 4879 { 4880 4881 #define GET_STAT(name) \ 4882 read_vf_stat(sc, viid, A_MPS_VF_STAT_##name##_L) 4883 4884 stats->tx_bcast_bytes = GET_STAT(TX_VF_BCAST_BYTES); 4885 stats->tx_bcast_frames = GET_STAT(TX_VF_BCAST_FRAMES); 4886 stats->tx_mcast_bytes = GET_STAT(TX_VF_MCAST_BYTES); 4887 stats->tx_mcast_frames = GET_STAT(TX_VF_MCAST_FRAMES); 4888 stats->tx_ucast_bytes = GET_STAT(TX_VF_UCAST_BYTES); 4889 stats->tx_ucast_frames = GET_STAT(TX_VF_UCAST_FRAMES); 4890 stats->tx_drop_frames = GET_STAT(TX_VF_DROP_FRAMES); 4891 stats->tx_offload_bytes = GET_STAT(TX_VF_OFFLOAD_BYTES); 4892 stats->tx_offload_frames = GET_STAT(TX_VF_OFFLOAD_FRAMES); 4893 stats->rx_bcast_bytes = GET_STAT(RX_VF_BCAST_BYTES); 4894 stats->rx_bcast_frames = GET_STAT(RX_VF_BCAST_FRAMES); 4895 stats->rx_mcast_bytes = GET_STAT(RX_VF_MCAST_BYTES); 4896 stats->rx_mcast_frames = GET_STAT(RX_VF_MCAST_FRAMES); 4897 stats->rx_ucast_bytes = GET_STAT(RX_VF_UCAST_BYTES); 4898 stats->rx_ucast_frames = GET_STAT(RX_VF_UCAST_FRAMES); 4899 stats->rx_err_frames = GET_STAT(RX_VF_ERR_FRAMES); 4900 4901 #undef GET_STAT 4902 } 4903 4904 static void 4905 t4_clr_vi_stats(struct adapter *sc, unsigned int viid) 4906 { 4907 int reg; 4908 4909 t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) | 4910 V_PL_VFID(G_FW_VIID_VIN(viid)) | 4911 V_PL_ADDR(VF_MPS_REG(A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L))); 4912 for (reg = A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L; 4913 reg <= A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H; reg += 4) 4914 t4_write_reg(sc, A_PL_INDIR_DATA, 0); 4915 } 4916 4917 static void 4918 vi_refresh_stats(struct adapter *sc, struct vi_info *vi) 4919 { 4920 struct timeval tv; 4921 const struct timeval interval = {0, 250000}; /* 250ms */ 4922 4923 if (!(vi->flags & VI_INIT_DONE)) 4924 return; 4925 4926 getmicrotime(&tv); 4927 timevalsub(&tv, &interval); 4928 if (timevalcmp(&tv, &vi->last_refreshed, <)) 4929 return; 4930 4931 mtx_lock(&sc->regwin_lock); 4932 t4_get_vi_stats(sc, vi->viid, &vi->stats); 4933 getmicrotime(&vi->last_refreshed); 4934 mtx_unlock(&sc->regwin_lock); 4935 } 4936 4937 static void 4938 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi) 4939 { 4940 int i; 4941 u_int v, tnl_cong_drops; 4942 struct timeval tv; 4943 const struct timeval interval = {0, 250000}; /* 250ms */ 4944 4945 getmicrotime(&tv); 4946 timevalsub(&tv, &interval); 4947 if (timevalcmp(&tv, &pi->last_refreshed, <)) 4948 return; 4949 4950 tnl_cong_drops = 0; 4951 t4_get_port_stats(sc, pi->tx_chan, &pi->stats); 4952 for (i = 0; i < NCHAN; i++) { 4953 if (pi->rx_chan_map & (1 << i)) { 4954 mtx_lock(&sc->regwin_lock); 4955 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 4956 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 4957 mtx_unlock(&sc->regwin_lock); 4958 tnl_cong_drops += v; 4959 } 4960 } 4961 pi->tnl_cong_drops = tnl_cong_drops; 4962 getmicrotime(&pi->last_refreshed); 4963 } 4964 4965 static void 4966 cxgbe_tick(void *arg) 4967 { 4968 struct port_info *pi = arg; 4969 struct adapter *sc = pi->adapter; 4970 4971 PORT_LOCK_ASSERT_OWNED(pi); 4972 cxgbe_refresh_stats(sc, pi); 4973 4974 callout_schedule(&pi->tick, hz); 4975 } 4976 4977 void 4978 vi_tick(void *arg) 4979 { 4980 struct vi_info *vi = arg; 4981 struct adapter *sc = vi->pi->adapter; 4982 4983 vi_refresh_stats(sc, vi); 4984 4985 callout_schedule(&vi->tick, hz); 4986 } 4987 4988 static void 4989 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 4990 { 4991 struct ifnet *vlan; 4992 4993 if (arg != ifp || ifp->if_type != IFT_ETHER) 4994 return; 4995 4996 vlan = VLAN_DEVAT(ifp, vid); 4997 VLAN_SETCOOKIE(vlan, ifp); 4998 } 4999 5000 static int 5001 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 5002 { 5003 5004 #ifdef INVARIANTS 5005 panic("%s: opcode 0x%02x on iq %p with payload %p", 5006 __func__, rss->opcode, iq, m); 5007 #else 5008 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 5009 __func__, rss->opcode, iq, m); 5010 m_freem(m); 5011 #endif 5012 return (EDOOFUS); 5013 } 5014 5015 int 5016 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 5017 { 5018 uintptr_t *loc, new; 5019 5020 if (opcode >= nitems(sc->cpl_handler)) 5021 return (EINVAL); 5022 5023 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 5024 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 5025 atomic_store_rel_ptr(loc, new); 5026 5027 return (0); 5028 } 5029 5030 static int 5031 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 5032 { 5033 5034 #ifdef INVARIANTS 5035 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 5036 #else 5037 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 5038 __func__, iq, ctrl); 5039 #endif 5040 return (EDOOFUS); 5041 } 5042 5043 int 5044 t4_register_an_handler(struct adapter *sc, an_handler_t h) 5045 { 5046 uintptr_t *loc, new; 5047 5048 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 5049 loc = (uintptr_t *) &sc->an_handler; 5050 atomic_store_rel_ptr(loc, new); 5051 5052 return (0); 5053 } 5054 5055 static int 5056 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 5057 { 5058 const struct cpl_fw6_msg *cpl = 5059 __containerof(rpl, struct cpl_fw6_msg, data[0]); 5060 5061 #ifdef INVARIANTS 5062 panic("%s: fw_msg type %d", __func__, cpl->type); 5063 #else 5064 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 5065 #endif 5066 return (EDOOFUS); 5067 } 5068 5069 int 5070 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 5071 { 5072 uintptr_t *loc, new; 5073 5074 if (type >= nitems(sc->fw_msg_handler)) 5075 return (EINVAL); 5076 5077 /* 5078 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 5079 * handler dispatch table. Reject any attempt to install a handler for 5080 * this subtype. 5081 */ 5082 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 5083 return (EINVAL); 5084 5085 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 5086 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 5087 atomic_store_rel_ptr(loc, new); 5088 5089 return (0); 5090 } 5091 5092 static void 5093 t4_sysctls(struct adapter *sc) 5094 { 5095 struct sysctl_ctx_list *ctx; 5096 struct sysctl_oid *oid; 5097 struct sysctl_oid_list *children, *c0; 5098 static char *caps[] = { 5099 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 5100 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL" /* caps[1] niccaps */ 5101 "\6HASHFILTER\7ETHOFLD", 5102 "\20\1TOE", /* caps[2] toecaps */ 5103 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 5104 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 5105 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 5106 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 5107 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 5108 "\4PO_INITIAOR\5PO_TARGET" 5109 }; 5110 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 5111 5112 ctx = device_get_sysctl_ctx(sc->dev); 5113 5114 /* 5115 * dev.t4nex.X. 5116 */ 5117 oid = device_get_sysctl_tree(sc->dev); 5118 c0 = children = SYSCTL_CHILDREN(oid); 5119 5120 sc->sc_do_rxcopy = 1; 5121 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, 5122 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); 5123 5124 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 5125 sc->params.nports, "# of ports"); 5126 5127 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 5128 NULL, chip_rev(sc), "chip hardware revision"); 5129 5130 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 5131 CTLFLAG_RD, sc->fw_version, 0, "firmware version"); 5132 5133 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 5134 CTLFLAG_RD, sc->cfg_file, 0, "configuration file"); 5135 5136 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 5137 sc->cfcsum, "config file checksum"); 5138 5139 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 5140 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 5141 sysctl_bitfield, "A", "available doorbells"); 5142 5143 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 5144 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 5145 sysctl_bitfield, "A", "available link capabilities"); 5146 5147 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 5148 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 5149 sysctl_bitfield, "A", "available NIC capabilities"); 5150 5151 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 5152 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 5153 sysctl_bitfield, "A", "available TCP offload capabilities"); 5154 5155 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 5156 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 5157 sysctl_bitfield, "A", "available RDMA capabilities"); 5158 5159 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 5160 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 5161 sysctl_bitfield, "A", "available iSCSI capabilities"); 5162 5163 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 5164 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 5165 sysctl_bitfield, "A", "available FCoE capabilities"); 5166 5167 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 5168 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 5169 5170 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 5171 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 5172 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 5173 "interrupt holdoff timer values (us)"); 5174 5175 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 5176 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 5177 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 5178 "interrupt holdoff packet counter values"); 5179 5180 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, 5181 NULL, sc->tids.nftids, "number of filters"); 5182 5183 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | 5184 CTLFLAG_RD, sc, 0, sysctl_temperature, "I", 5185 "chip temperature (in Celsius)"); 5186 5187 t4_sge_sysctls(sc, ctx, children); 5188 5189 sc->lro_timeout = 100; 5190 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, 5191 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); 5192 5193 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "debug_flags", CTLFLAG_RW, 5194 &sc->debug_flags, 0, "flags to enable runtime debugging"); 5195 5196 #ifdef SBUF_DRAIN 5197 /* 5198 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 5199 */ 5200 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 5201 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 5202 "logs and miscellaneous information"); 5203 children = SYSCTL_CHILDREN(oid); 5204 5205 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 5206 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5207 sysctl_cctrl, "A", "congestion control"); 5208 5209 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 5210 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5211 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 5212 5213 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 5214 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 5215 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 5216 5217 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 5218 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 5219 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 5220 5221 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 5222 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 5223 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 5224 5225 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 5226 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 5227 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 5228 5229 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 5230 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 5231 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 5232 5233 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 5234 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5235 sysctl_cim_la, "A", "CIM logic analyzer"); 5236 5237 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", 5238 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5239 sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); 5240 5241 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 5242 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 5243 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 5244 5245 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 5246 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 5247 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 5248 5249 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 5250 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 5251 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 5252 5253 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 5254 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 5255 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 5256 5257 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 5258 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 5259 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 5260 5261 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 5262 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 5263 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 5264 5265 if (is_t5(sc)) { 5266 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 5267 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 5268 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 5269 5270 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 5271 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 5272 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 5273 } 5274 5275 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", 5276 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5277 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); 5278 5279 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 5280 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5281 sysctl_cim_qcfg, "A", "CIM queue configuration"); 5282 5283 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 5284 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5285 sysctl_cpl_stats, "A", "CPL statistics"); 5286 5287 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 5288 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5289 sysctl_ddp_stats, "A", "non-TCP DDP statistics"); 5290 5291 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 5292 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5293 sysctl_devlog, "A", "firmware's device log"); 5294 5295 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 5296 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5297 sysctl_fcoe_stats, "A", "FCoE statistics"); 5298 5299 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 5300 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5301 sysctl_hw_sched, "A", "hardware scheduler "); 5302 5303 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 5304 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5305 sysctl_l2t, "A", "hardware L2 table"); 5306 5307 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 5308 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5309 sysctl_lb_stats, "A", "loopback statistics"); 5310 5311 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 5312 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5313 sysctl_meminfo, "A", "memory regions"); 5314 5315 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", 5316 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5317 sysctl_mps_tcam, "A", "MPS TCAM entries"); 5318 5319 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 5320 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5321 sysctl_path_mtus, "A", "path MTUs"); 5322 5323 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 5324 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5325 sysctl_pm_stats, "A", "PM statistics"); 5326 5327 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 5328 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5329 sysctl_rdma_stats, "A", "RDMA statistics"); 5330 5331 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 5332 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5333 sysctl_tcp_stats, "A", "TCP statistics"); 5334 5335 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 5336 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5337 sysctl_tids, "A", "TID information"); 5338 5339 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 5340 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5341 sysctl_tp_err_stats, "A", "TP error statistics"); 5342 5343 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", 5344 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5345 sysctl_tp_la, "A", "TP logic analyzer"); 5346 5347 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 5348 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5349 sysctl_tx_rate, "A", "Tx rate"); 5350 5351 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", 5352 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5353 sysctl_ulprx_la, "A", "ULPRX logic analyzer"); 5354 5355 if (is_t5(sc)) { 5356 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 5357 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5358 sysctl_wcwr_stats, "A", "write combined work requests"); 5359 } 5360 #endif 5361 5362 #ifdef TCP_OFFLOAD 5363 if (is_offload(sc)) { 5364 /* 5365 * dev.t4nex.X.toe. 5366 */ 5367 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 5368 NULL, "TOE parameters"); 5369 children = SYSCTL_CHILDREN(oid); 5370 5371 sc->tt.sndbuf = 256 * 1024; 5372 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 5373 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 5374 5375 sc->tt.ddp = 0; 5376 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 5377 &sc->tt.ddp, 0, "DDP allowed"); 5378 5379 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 5380 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 5381 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 5382 5383 sc->tt.ddp_thres = 5384 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 5385 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 5386 &sc->tt.ddp_thres, 0, "DDP threshold"); 5387 5388 sc->tt.rx_coalesce = 1; 5389 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", 5390 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); 5391 5392 sc->tt.tx_align = 1; 5393 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align", 5394 CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload"); 5395 } 5396 #endif 5397 } 5398 5399 void 5400 vi_sysctls(struct vi_info *vi) 5401 { 5402 struct sysctl_ctx_list *ctx; 5403 struct sysctl_oid *oid; 5404 struct sysctl_oid_list *children; 5405 5406 ctx = device_get_sysctl_ctx(vi->dev); 5407 5408 /* 5409 * dev.[nv](cxgbe|cxl).X. 5410 */ 5411 oid = device_get_sysctl_tree(vi->dev); 5412 children = SYSCTL_CHILDREN(oid); 5413 5414 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "viid", CTLFLAG_RD, NULL, 5415 vi->viid, "VI identifer"); 5416 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 5417 &vi->nrxq, 0, "# of rx queues"); 5418 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 5419 &vi->ntxq, 0, "# of tx queues"); 5420 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 5421 &vi->first_rxq, 0, "index of first rx queue"); 5422 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 5423 &vi->first_txq, 0, "index of first tx queue"); 5424 5425 if (vi->flags & VI_NETMAP) 5426 return; 5427 5428 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT | 5429 CTLFLAG_RW, vi, 0, sysctl_noflowq, "IU", 5430 "Reserve queue 0 for non-flowid packets"); 5431 5432 #ifdef TCP_OFFLOAD 5433 if (vi->nofldrxq != 0) { 5434 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 5435 &vi->nofldrxq, 0, 5436 "# of rx queues for offloaded TCP connections"); 5437 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 5438 &vi->nofldtxq, 0, 5439 "# of tx queues for offloaded TCP connections"); 5440 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 5441 CTLFLAG_RD, &vi->first_ofld_rxq, 0, 5442 "index of first TOE rx queue"); 5443 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 5444 CTLFLAG_RD, &vi->first_ofld_txq, 0, 5445 "index of first TOE tx queue"); 5446 } 5447 #endif 5448 5449 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 5450 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_holdoff_tmr_idx, "I", 5451 "holdoff timer index"); 5452 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 5453 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_holdoff_pktc_idx, "I", 5454 "holdoff packet counter index"); 5455 5456 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 5457 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_qsize_rxq, "I", 5458 "rx queue size"); 5459 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 5460 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_qsize_txq, "I", 5461 "tx queue size"); 5462 } 5463 5464 static void 5465 cxgbe_sysctls(struct port_info *pi) 5466 { 5467 struct sysctl_ctx_list *ctx; 5468 struct sysctl_oid *oid; 5469 struct sysctl_oid_list *children; 5470 struct adapter *sc = pi->adapter; 5471 5472 ctx = device_get_sysctl_ctx(pi->dev); 5473 5474 /* 5475 * dev.cxgbe.X. 5476 */ 5477 oid = device_get_sysctl_tree(pi->dev); 5478 children = SYSCTL_CHILDREN(oid); 5479 5480 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | 5481 CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); 5482 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { 5483 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 5484 CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I", 5485 "PHY temperature (in Celsius)"); 5486 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", 5487 CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I", 5488 "PHY firmware version"); 5489 } 5490 5491 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings", 5492 CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings, 5493 "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)"); 5494 5495 /* 5496 * dev.cxgbe.X.stats. 5497 */ 5498 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 5499 NULL, "port statistics"); 5500 children = SYSCTL_CHILDREN(oid); 5501 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD, 5502 &pi->tx_parse_error, 0, 5503 "# of tx packets with invalid length or # of segments"); 5504 5505 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 5506 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 5507 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ 5508 sysctl_handle_t4_reg64, "QU", desc) 5509 5510 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 5511 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 5512 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 5513 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 5514 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 5515 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 5516 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 5517 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 5518 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 5519 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 5520 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 5521 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 5522 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 5523 "# of tx frames in this range", 5524 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 5525 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 5526 "# of tx frames in this range", 5527 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 5528 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 5529 "# of tx frames in this range", 5530 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 5531 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 5532 "# of tx frames in this range", 5533 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 5534 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 5535 "# of tx frames in this range", 5536 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 5537 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 5538 "# of tx frames in this range", 5539 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 5540 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 5541 "# of tx frames in this range", 5542 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 5543 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 5544 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 5545 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 5546 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 5547 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 5548 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 5549 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 5550 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 5551 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 5552 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 5553 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 5554 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 5555 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 5556 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 5557 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 5558 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 5559 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 5560 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 5561 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 5562 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 5563 5564 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 5565 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 5566 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 5567 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 5568 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 5569 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 5570 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 5571 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 5572 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 5573 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 5574 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 5575 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 5576 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 5577 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 5578 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 5579 "# of frames received with bad FCS", 5580 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 5581 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 5582 "# of frames received with length error", 5583 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 5584 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 5585 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 5586 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 5587 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 5588 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 5589 "# of rx frames in this range", 5590 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 5591 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 5592 "# of rx frames in this range", 5593 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 5594 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 5595 "# of rx frames in this range", 5596 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 5597 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 5598 "# of rx frames in this range", 5599 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 5600 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 5601 "# of rx frames in this range", 5602 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 5603 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 5604 "# of rx frames in this range", 5605 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 5606 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 5607 "# of rx frames in this range", 5608 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 5609 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 5610 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 5611 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 5612 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 5613 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 5614 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 5615 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 5616 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 5617 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 5618 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 5619 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 5620 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 5621 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 5622 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 5623 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 5624 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 5625 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 5626 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 5627 5628 #undef SYSCTL_ADD_T4_REG64 5629 5630 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 5631 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 5632 &pi->stats.name, desc) 5633 5634 /* We get these from port_stats and they may be stale by upto 1s */ 5635 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 5636 "# drops due to buffer-group 0 overflows"); 5637 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 5638 "# drops due to buffer-group 1 overflows"); 5639 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 5640 "# drops due to buffer-group 2 overflows"); 5641 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 5642 "# drops due to buffer-group 3 overflows"); 5643 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 5644 "# of buffer-group 0 truncated packets"); 5645 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 5646 "# of buffer-group 1 truncated packets"); 5647 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 5648 "# of buffer-group 2 truncated packets"); 5649 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 5650 "# of buffer-group 3 truncated packets"); 5651 5652 #undef SYSCTL_ADD_T4_PORTSTAT 5653 } 5654 5655 static int 5656 sysctl_int_array(SYSCTL_HANDLER_ARGS) 5657 { 5658 int rc, *i, space = 0; 5659 struct sbuf sb; 5660 5661 sbuf_new_for_sysctl(&sb, NULL, 64, req); 5662 for (i = arg1; arg2; arg2 -= sizeof(int), i++) { 5663 if (space) 5664 sbuf_printf(&sb, " "); 5665 sbuf_printf(&sb, "%d", *i); 5666 space = 1; 5667 } 5668 rc = sbuf_finish(&sb); 5669 sbuf_delete(&sb); 5670 return (rc); 5671 } 5672 5673 static int 5674 sysctl_bitfield(SYSCTL_HANDLER_ARGS) 5675 { 5676 int rc; 5677 struct sbuf *sb; 5678 5679 rc = sysctl_wire_old_buffer(req, 0); 5680 if (rc != 0) 5681 return(rc); 5682 5683 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5684 if (sb == NULL) 5685 return (ENOMEM); 5686 5687 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 5688 rc = sbuf_finish(sb); 5689 sbuf_delete(sb); 5690 5691 return (rc); 5692 } 5693 5694 static int 5695 sysctl_btphy(SYSCTL_HANDLER_ARGS) 5696 { 5697 struct port_info *pi = arg1; 5698 int op = arg2; 5699 struct adapter *sc = pi->adapter; 5700 u_int v; 5701 int rc; 5702 5703 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, "t4btt"); 5704 if (rc) 5705 return (rc); 5706 /* XXX: magic numbers */ 5707 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, 5708 &v); 5709 end_synchronized_op(sc, 0); 5710 if (rc) 5711 return (rc); 5712 if (op == 0) 5713 v /= 256; 5714 5715 rc = sysctl_handle_int(oidp, &v, 0, req); 5716 return (rc); 5717 } 5718 5719 static int 5720 sysctl_noflowq(SYSCTL_HANDLER_ARGS) 5721 { 5722 struct vi_info *vi = arg1; 5723 int rc, val; 5724 5725 val = vi->rsrv_noflowq; 5726 rc = sysctl_handle_int(oidp, &val, 0, req); 5727 if (rc != 0 || req->newptr == NULL) 5728 return (rc); 5729 5730 if ((val >= 1) && (vi->ntxq > 1)) 5731 vi->rsrv_noflowq = 1; 5732 else 5733 vi->rsrv_noflowq = 0; 5734 5735 return (rc); 5736 } 5737 5738 static int 5739 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 5740 { 5741 struct vi_info *vi = arg1; 5742 struct adapter *sc = vi->pi->adapter; 5743 int idx, rc, i; 5744 struct sge_rxq *rxq; 5745 #ifdef TCP_OFFLOAD 5746 struct sge_ofld_rxq *ofld_rxq; 5747 #endif 5748 uint8_t v; 5749 5750 idx = vi->tmr_idx; 5751 5752 rc = sysctl_handle_int(oidp, &idx, 0, req); 5753 if (rc != 0 || req->newptr == NULL) 5754 return (rc); 5755 5756 if (idx < 0 || idx >= SGE_NTIMERS) 5757 return (EINVAL); 5758 5759 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5760 "t4tmr"); 5761 if (rc) 5762 return (rc); 5763 5764 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(vi->pktc_idx != -1); 5765 for_each_rxq(vi, i, rxq) { 5766 #ifdef atomic_store_rel_8 5767 atomic_store_rel_8(&rxq->iq.intr_params, v); 5768 #else 5769 rxq->iq.intr_params = v; 5770 #endif 5771 } 5772 #ifdef TCP_OFFLOAD 5773 for_each_ofld_rxq(vi, i, ofld_rxq) { 5774 #ifdef atomic_store_rel_8 5775 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); 5776 #else 5777 ofld_rxq->iq.intr_params = v; 5778 #endif 5779 } 5780 #endif 5781 vi->tmr_idx = idx; 5782 5783 end_synchronized_op(sc, LOCK_HELD); 5784 return (0); 5785 } 5786 5787 static int 5788 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 5789 { 5790 struct vi_info *vi = arg1; 5791 struct adapter *sc = vi->pi->adapter; 5792 int idx, rc; 5793 5794 idx = vi->pktc_idx; 5795 5796 rc = sysctl_handle_int(oidp, &idx, 0, req); 5797 if (rc != 0 || req->newptr == NULL) 5798 return (rc); 5799 5800 if (idx < -1 || idx >= SGE_NCOUNTERS) 5801 return (EINVAL); 5802 5803 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5804 "t4pktc"); 5805 if (rc) 5806 return (rc); 5807 5808 if (vi->flags & VI_INIT_DONE) 5809 rc = EBUSY; /* cannot be changed once the queues are created */ 5810 else 5811 vi->pktc_idx = idx; 5812 5813 end_synchronized_op(sc, LOCK_HELD); 5814 return (rc); 5815 } 5816 5817 static int 5818 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 5819 { 5820 struct vi_info *vi = arg1; 5821 struct adapter *sc = vi->pi->adapter; 5822 int qsize, rc; 5823 5824 qsize = vi->qsize_rxq; 5825 5826 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5827 if (rc != 0 || req->newptr == NULL) 5828 return (rc); 5829 5830 if (qsize < 128 || (qsize & 7)) 5831 return (EINVAL); 5832 5833 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5834 "t4rxqs"); 5835 if (rc) 5836 return (rc); 5837 5838 if (vi->flags & VI_INIT_DONE) 5839 rc = EBUSY; /* cannot be changed once the queues are created */ 5840 else 5841 vi->qsize_rxq = qsize; 5842 5843 end_synchronized_op(sc, LOCK_HELD); 5844 return (rc); 5845 } 5846 5847 static int 5848 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 5849 { 5850 struct vi_info *vi = arg1; 5851 struct adapter *sc = vi->pi->adapter; 5852 int qsize, rc; 5853 5854 qsize = vi->qsize_txq; 5855 5856 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5857 if (rc != 0 || req->newptr == NULL) 5858 return (rc); 5859 5860 if (qsize < 128 || qsize > 65536) 5861 return (EINVAL); 5862 5863 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5864 "t4txqs"); 5865 if (rc) 5866 return (rc); 5867 5868 if (vi->flags & VI_INIT_DONE) 5869 rc = EBUSY; /* cannot be changed once the queues are created */ 5870 else 5871 vi->qsize_txq = qsize; 5872 5873 end_synchronized_op(sc, LOCK_HELD); 5874 return (rc); 5875 } 5876 5877 static int 5878 sysctl_pause_settings(SYSCTL_HANDLER_ARGS) 5879 { 5880 struct port_info *pi = arg1; 5881 struct adapter *sc = pi->adapter; 5882 struct link_config *lc = &pi->link_cfg; 5883 int rc; 5884 5885 if (req->newptr == NULL) { 5886 struct sbuf *sb; 5887 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX"; 5888 5889 rc = sysctl_wire_old_buffer(req, 0); 5890 if (rc != 0) 5891 return(rc); 5892 5893 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5894 if (sb == NULL) 5895 return (ENOMEM); 5896 5897 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits); 5898 rc = sbuf_finish(sb); 5899 sbuf_delete(sb); 5900 } else { 5901 char s[2]; 5902 int n; 5903 5904 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX)); 5905 s[1] = 0; 5906 5907 rc = sysctl_handle_string(oidp, s, sizeof(s), req); 5908 if (rc != 0) 5909 return(rc); 5910 5911 if (s[1] != 0) 5912 return (EINVAL); 5913 if (s[0] < '0' || s[0] > '9') 5914 return (EINVAL); /* not a number */ 5915 n = s[0] - '0'; 5916 if (n & ~(PAUSE_TX | PAUSE_RX)) 5917 return (EINVAL); /* some other bit is set too */ 5918 5919 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, 5920 "t4PAUSE"); 5921 if (rc) 5922 return (rc); 5923 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) { 5924 int link_ok = lc->link_ok; 5925 5926 lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX); 5927 lc->requested_fc |= n; 5928 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc); 5929 lc->link_ok = link_ok; /* restore */ 5930 } 5931 end_synchronized_op(sc, 0); 5932 } 5933 5934 return (rc); 5935 } 5936 5937 static int 5938 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 5939 { 5940 struct adapter *sc = arg1; 5941 int reg = arg2; 5942 uint64_t val; 5943 5944 val = t4_read_reg64(sc, reg); 5945 5946 return (sysctl_handle_64(oidp, &val, 0, req)); 5947 } 5948 5949 static int 5950 sysctl_temperature(SYSCTL_HANDLER_ARGS) 5951 { 5952 struct adapter *sc = arg1; 5953 int rc, t; 5954 uint32_t param, val; 5955 5956 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); 5957 if (rc) 5958 return (rc); 5959 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 5960 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 5961 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); 5962 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 5963 end_synchronized_op(sc, 0); 5964 if (rc) 5965 return (rc); 5966 5967 /* unknown is returned as 0 but we display -1 in that case */ 5968 t = val == 0 ? -1 : val; 5969 5970 rc = sysctl_handle_int(oidp, &t, 0, req); 5971 return (rc); 5972 } 5973 5974 #ifdef SBUF_DRAIN 5975 static int 5976 sysctl_cctrl(SYSCTL_HANDLER_ARGS) 5977 { 5978 struct adapter *sc = arg1; 5979 struct sbuf *sb; 5980 int rc, i; 5981 uint16_t incr[NMTUS][NCCTRL_WIN]; 5982 static const char *dec_fac[] = { 5983 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 5984 "0.9375" 5985 }; 5986 5987 rc = sysctl_wire_old_buffer(req, 0); 5988 if (rc != 0) 5989 return (rc); 5990 5991 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5992 if (sb == NULL) 5993 return (ENOMEM); 5994 5995 t4_read_cong_tbl(sc, incr); 5996 5997 for (i = 0; i < NCCTRL_WIN; ++i) { 5998 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 5999 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 6000 incr[5][i], incr[6][i], incr[7][i]); 6001 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 6002 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 6003 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 6004 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 6005 } 6006 6007 rc = sbuf_finish(sb); 6008 sbuf_delete(sb); 6009 6010 return (rc); 6011 } 6012 6013 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 6014 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 6015 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 6016 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 6017 }; 6018 6019 static int 6020 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 6021 { 6022 struct adapter *sc = arg1; 6023 struct sbuf *sb; 6024 int rc, i, n, qid = arg2; 6025 uint32_t *buf, *p; 6026 char *qtype; 6027 u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5; 6028 6029 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 6030 ("%s: bad qid %d\n", __func__, qid)); 6031 6032 if (qid < CIM_NUM_IBQ) { 6033 /* inbound queue */ 6034 qtype = "IBQ"; 6035 n = 4 * CIM_IBQ_SIZE; 6036 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 6037 rc = t4_read_cim_ibq(sc, qid, buf, n); 6038 } else { 6039 /* outbound queue */ 6040 qtype = "OBQ"; 6041 qid -= CIM_NUM_IBQ; 6042 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 6043 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 6044 rc = t4_read_cim_obq(sc, qid, buf, n); 6045 } 6046 6047 if (rc < 0) { 6048 rc = -rc; 6049 goto done; 6050 } 6051 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 6052 6053 rc = sysctl_wire_old_buffer(req, 0); 6054 if (rc != 0) 6055 goto done; 6056 6057 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 6058 if (sb == NULL) { 6059 rc = ENOMEM; 6060 goto done; 6061 } 6062 6063 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 6064 for (i = 0, p = buf; i < n; i += 16, p += 4) 6065 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 6066 p[2], p[3]); 6067 6068 rc = sbuf_finish(sb); 6069 sbuf_delete(sb); 6070 done: 6071 free(buf, M_CXGBE); 6072 return (rc); 6073 } 6074 6075 static int 6076 sysctl_cim_la(SYSCTL_HANDLER_ARGS) 6077 { 6078 struct adapter *sc = arg1; 6079 u_int cfg; 6080 struct sbuf *sb; 6081 uint32_t *buf, *p; 6082 int rc; 6083 6084 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 6085 if (rc != 0) 6086 return (rc); 6087 6088 rc = sysctl_wire_old_buffer(req, 0); 6089 if (rc != 0) 6090 return (rc); 6091 6092 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6093 if (sb == NULL) 6094 return (ENOMEM); 6095 6096 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 6097 M_ZERO | M_WAITOK); 6098 6099 rc = -t4_cim_read_la(sc, buf, NULL); 6100 if (rc != 0) 6101 goto done; 6102 6103 sbuf_printf(sb, "Status Data PC%s", 6104 cfg & F_UPDBGLACAPTPCONLY ? "" : 6105 " LS0Stat LS0Addr LS0Data"); 6106 6107 KASSERT((sc->params.cim_la_size & 7) == 0, 6108 ("%s: p will walk off the end of buf", __func__)); 6109 6110 for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) { 6111 if (cfg & F_UPDBGLACAPTPCONLY) { 6112 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 6113 p[6], p[7]); 6114 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 6115 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 6116 p[4] & 0xff, p[5] >> 8); 6117 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 6118 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 6119 p[1] & 0xf, p[2] >> 4); 6120 } else { 6121 sbuf_printf(sb, 6122 "\n %02x %x%07x %x%07x %08x %08x " 6123 "%08x%08x%08x%08x", 6124 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 6125 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 6126 p[6], p[7]); 6127 } 6128 } 6129 6130 rc = sbuf_finish(sb); 6131 sbuf_delete(sb); 6132 done: 6133 free(buf, M_CXGBE); 6134 return (rc); 6135 } 6136 6137 static int 6138 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) 6139 { 6140 struct adapter *sc = arg1; 6141 u_int i; 6142 struct sbuf *sb; 6143 uint32_t *buf, *p; 6144 int rc; 6145 6146 rc = sysctl_wire_old_buffer(req, 0); 6147 if (rc != 0) 6148 return (rc); 6149 6150 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6151 if (sb == NULL) 6152 return (ENOMEM); 6153 6154 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, 6155 M_ZERO | M_WAITOK); 6156 6157 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); 6158 p = buf; 6159 6160 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 6161 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], 6162 p[1], p[0]); 6163 } 6164 6165 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); 6166 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 6167 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", 6168 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, 6169 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, 6170 (p[1] >> 2) | ((p[2] & 3) << 30), 6171 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, 6172 p[0] & 1); 6173 } 6174 6175 rc = sbuf_finish(sb); 6176 sbuf_delete(sb); 6177 free(buf, M_CXGBE); 6178 return (rc); 6179 } 6180 6181 static int 6182 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) 6183 { 6184 struct adapter *sc = arg1; 6185 u_int i; 6186 struct sbuf *sb; 6187 uint32_t *buf, *p; 6188 int rc; 6189 6190 rc = sysctl_wire_old_buffer(req, 0); 6191 if (rc != 0) 6192 return (rc); 6193 6194 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6195 if (sb == NULL) 6196 return (ENOMEM); 6197 6198 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, 6199 M_ZERO | M_WAITOK); 6200 6201 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); 6202 p = buf; 6203 6204 sbuf_printf(sb, "Cntl ID DataBE Addr Data"); 6205 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 6206 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", 6207 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, 6208 p[4], p[3], p[2], p[1], p[0]); 6209 } 6210 6211 sbuf_printf(sb, "\n\nCntl ID Data"); 6212 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 6213 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", 6214 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); 6215 } 6216 6217 rc = sbuf_finish(sb); 6218 sbuf_delete(sb); 6219 free(buf, M_CXGBE); 6220 return (rc); 6221 } 6222 6223 static int 6224 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 6225 { 6226 struct adapter *sc = arg1; 6227 struct sbuf *sb; 6228 int rc, i; 6229 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 6230 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 6231 uint16_t thres[CIM_NUM_IBQ]; 6232 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 6233 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 6234 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 6235 6236 if (is_t4(sc)) { 6237 cim_num_obq = CIM_NUM_OBQ; 6238 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 6239 obq_rdaddr = A_UP_OBQ_0_REALADDR; 6240 } else { 6241 cim_num_obq = CIM_NUM_OBQ_T5; 6242 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 6243 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 6244 } 6245 nq = CIM_NUM_IBQ + cim_num_obq; 6246 6247 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 6248 if (rc == 0) 6249 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 6250 if (rc != 0) 6251 return (rc); 6252 6253 t4_read_cimq_cfg(sc, base, size, thres); 6254 6255 rc = sysctl_wire_old_buffer(req, 0); 6256 if (rc != 0) 6257 return (rc); 6258 6259 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 6260 if (sb == NULL) 6261 return (ENOMEM); 6262 6263 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 6264 6265 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 6266 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 6267 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 6268 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 6269 G_QUEREMFLITS(p[2]) * 16); 6270 for ( ; i < nq; i++, p += 4, wr += 2) 6271 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 6272 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 6273 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 6274 G_QUEREMFLITS(p[2]) * 16); 6275 6276 rc = sbuf_finish(sb); 6277 sbuf_delete(sb); 6278 6279 return (rc); 6280 } 6281 6282 static int 6283 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 6284 { 6285 struct adapter *sc = arg1; 6286 struct sbuf *sb; 6287 int rc; 6288 struct tp_cpl_stats stats; 6289 6290 rc = sysctl_wire_old_buffer(req, 0); 6291 if (rc != 0) 6292 return (rc); 6293 6294 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6295 if (sb == NULL) 6296 return (ENOMEM); 6297 6298 t4_tp_get_cpl_stats(sc, &stats); 6299 6300 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6301 "channel 3\n"); 6302 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 6303 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 6304 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 6305 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 6306 6307 rc = sbuf_finish(sb); 6308 sbuf_delete(sb); 6309 6310 return (rc); 6311 } 6312 6313 static int 6314 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 6315 { 6316 struct adapter *sc = arg1; 6317 struct sbuf *sb; 6318 int rc; 6319 struct tp_usm_stats stats; 6320 6321 rc = sysctl_wire_old_buffer(req, 0); 6322 if (rc != 0) 6323 return(rc); 6324 6325 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6326 if (sb == NULL) 6327 return (ENOMEM); 6328 6329 t4_get_usm_stats(sc, &stats); 6330 6331 sbuf_printf(sb, "Frames: %u\n", stats.frames); 6332 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 6333 sbuf_printf(sb, "Drops: %u", stats.drops); 6334 6335 rc = sbuf_finish(sb); 6336 sbuf_delete(sb); 6337 6338 return (rc); 6339 } 6340 6341 const char *devlog_level_strings[] = { 6342 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 6343 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 6344 [FW_DEVLOG_LEVEL_ERR] = "ERR", 6345 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 6346 [FW_DEVLOG_LEVEL_INFO] = "INFO", 6347 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 6348 }; 6349 6350 const char *devlog_facility_strings[] = { 6351 [FW_DEVLOG_FACILITY_CORE] = "CORE", 6352 [FW_DEVLOG_FACILITY_CF] = "CF", 6353 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 6354 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 6355 [FW_DEVLOG_FACILITY_RES] = "RES", 6356 [FW_DEVLOG_FACILITY_HW] = "HW", 6357 [FW_DEVLOG_FACILITY_FLR] = "FLR", 6358 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 6359 [FW_DEVLOG_FACILITY_PHY] = "PHY", 6360 [FW_DEVLOG_FACILITY_MAC] = "MAC", 6361 [FW_DEVLOG_FACILITY_PORT] = "PORT", 6362 [FW_DEVLOG_FACILITY_VI] = "VI", 6363 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 6364 [FW_DEVLOG_FACILITY_ACL] = "ACL", 6365 [FW_DEVLOG_FACILITY_TM] = "TM", 6366 [FW_DEVLOG_FACILITY_QFC] = "QFC", 6367 [FW_DEVLOG_FACILITY_DCB] = "DCB", 6368 [FW_DEVLOG_FACILITY_ETH] = "ETH", 6369 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 6370 [FW_DEVLOG_FACILITY_RI] = "RI", 6371 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 6372 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 6373 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 6374 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 6375 }; 6376 6377 static int 6378 sysctl_devlog(SYSCTL_HANDLER_ARGS) 6379 { 6380 struct adapter *sc = arg1; 6381 struct devlog_params *dparams = &sc->params.devlog; 6382 struct fw_devlog_e *buf, *e; 6383 int i, j, rc, nentries, first = 0, m; 6384 struct sbuf *sb; 6385 uint64_t ftstamp = UINT64_MAX; 6386 6387 if (dparams->start == 0) { 6388 dparams->memtype = FW_MEMTYPE_EDC0; 6389 dparams->start = 0x84000; 6390 dparams->size = 32768; 6391 } 6392 6393 nentries = dparams->size / sizeof(struct fw_devlog_e); 6394 6395 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 6396 if (buf == NULL) 6397 return (ENOMEM); 6398 6399 m = fwmtype_to_hwmtype(dparams->memtype); 6400 rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf); 6401 if (rc != 0) 6402 goto done; 6403 6404 for (i = 0; i < nentries; i++) { 6405 e = &buf[i]; 6406 6407 if (e->timestamp == 0) 6408 break; /* end */ 6409 6410 e->timestamp = be64toh(e->timestamp); 6411 e->seqno = be32toh(e->seqno); 6412 for (j = 0; j < 8; j++) 6413 e->params[j] = be32toh(e->params[j]); 6414 6415 if (e->timestamp < ftstamp) { 6416 ftstamp = e->timestamp; 6417 first = i; 6418 } 6419 } 6420 6421 if (buf[first].timestamp == 0) 6422 goto done; /* nothing in the log */ 6423 6424 rc = sysctl_wire_old_buffer(req, 0); 6425 if (rc != 0) 6426 goto done; 6427 6428 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6429 if (sb == NULL) { 6430 rc = ENOMEM; 6431 goto done; 6432 } 6433 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 6434 "Seq#", "Tstamp", "Level", "Facility", "Message"); 6435 6436 i = first; 6437 do { 6438 e = &buf[i]; 6439 if (e->timestamp == 0) 6440 break; /* end */ 6441 6442 sbuf_printf(sb, "%10d %15ju %8s %8s ", 6443 e->seqno, e->timestamp, 6444 (e->level < nitems(devlog_level_strings) ? 6445 devlog_level_strings[e->level] : "UNKNOWN"), 6446 (e->facility < nitems(devlog_facility_strings) ? 6447 devlog_facility_strings[e->facility] : "UNKNOWN")); 6448 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 6449 e->params[2], e->params[3], e->params[4], 6450 e->params[5], e->params[6], e->params[7]); 6451 6452 if (++i == nentries) 6453 i = 0; 6454 } while (i != first); 6455 6456 rc = sbuf_finish(sb); 6457 sbuf_delete(sb); 6458 done: 6459 free(buf, M_CXGBE); 6460 return (rc); 6461 } 6462 6463 static int 6464 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 6465 { 6466 struct adapter *sc = arg1; 6467 struct sbuf *sb; 6468 int rc; 6469 struct tp_fcoe_stats stats[4]; 6470 6471 rc = sysctl_wire_old_buffer(req, 0); 6472 if (rc != 0) 6473 return (rc); 6474 6475 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6476 if (sb == NULL) 6477 return (ENOMEM); 6478 6479 t4_get_fcoe_stats(sc, 0, &stats[0]); 6480 t4_get_fcoe_stats(sc, 1, &stats[1]); 6481 t4_get_fcoe_stats(sc, 2, &stats[2]); 6482 t4_get_fcoe_stats(sc, 3, &stats[3]); 6483 6484 sbuf_printf(sb, " channel 0 channel 1 " 6485 "channel 2 channel 3\n"); 6486 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 6487 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 6488 stats[3].octetsDDP); 6489 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 6490 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 6491 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 6492 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 6493 stats[3].framesDrop); 6494 6495 rc = sbuf_finish(sb); 6496 sbuf_delete(sb); 6497 6498 return (rc); 6499 } 6500 6501 static int 6502 sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 6503 { 6504 struct adapter *sc = arg1; 6505 struct sbuf *sb; 6506 int rc, i; 6507 unsigned int map, kbps, ipg, mode; 6508 unsigned int pace_tab[NTX_SCHED]; 6509 6510 rc = sysctl_wire_old_buffer(req, 0); 6511 if (rc != 0) 6512 return (rc); 6513 6514 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6515 if (sb == NULL) 6516 return (ENOMEM); 6517 6518 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 6519 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 6520 t4_read_pace_tbl(sc, pace_tab); 6521 6522 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 6523 "Class IPG (0.1 ns) Flow IPG (us)"); 6524 6525 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 6526 t4_get_tx_sched(sc, i, &kbps, &ipg); 6527 sbuf_printf(sb, "\n %u %-5s %u ", i, 6528 (mode & (1 << i)) ? "flow" : "class", map & 3); 6529 if (kbps) 6530 sbuf_printf(sb, "%9u ", kbps); 6531 else 6532 sbuf_printf(sb, " disabled "); 6533 6534 if (ipg) 6535 sbuf_printf(sb, "%13u ", ipg); 6536 else 6537 sbuf_printf(sb, " disabled "); 6538 6539 if (pace_tab[i]) 6540 sbuf_printf(sb, "%10u", pace_tab[i]); 6541 else 6542 sbuf_printf(sb, " disabled"); 6543 } 6544 6545 rc = sbuf_finish(sb); 6546 sbuf_delete(sb); 6547 6548 return (rc); 6549 } 6550 6551 static int 6552 sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 6553 { 6554 struct adapter *sc = arg1; 6555 struct sbuf *sb; 6556 int rc, i, j; 6557 uint64_t *p0, *p1; 6558 struct lb_port_stats s[2]; 6559 static const char *stat_name[] = { 6560 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 6561 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 6562 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 6563 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 6564 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 6565 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 6566 "BG2FramesTrunc:", "BG3FramesTrunc:" 6567 }; 6568 6569 rc = sysctl_wire_old_buffer(req, 0); 6570 if (rc != 0) 6571 return (rc); 6572 6573 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6574 if (sb == NULL) 6575 return (ENOMEM); 6576 6577 memset(s, 0, sizeof(s)); 6578 6579 for (i = 0; i < 4; i += 2) { 6580 t4_get_lb_stats(sc, i, &s[0]); 6581 t4_get_lb_stats(sc, i + 1, &s[1]); 6582 6583 p0 = &s[0].octets; 6584 p1 = &s[1].octets; 6585 sbuf_printf(sb, "%s Loopback %u" 6586 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 6587 6588 for (j = 0; j < nitems(stat_name); j++) 6589 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 6590 *p0++, *p1++); 6591 } 6592 6593 rc = sbuf_finish(sb); 6594 sbuf_delete(sb); 6595 6596 return (rc); 6597 } 6598 6599 static int 6600 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) 6601 { 6602 int rc = 0; 6603 struct port_info *pi = arg1; 6604 struct sbuf *sb; 6605 static const char *linkdnreasons[] = { 6606 "non-specific", "remote fault", "autoneg failed", "reserved3", 6607 "PHY overheated", "unknown", "rx los", "reserved7" 6608 }; 6609 6610 rc = sysctl_wire_old_buffer(req, 0); 6611 if (rc != 0) 6612 return(rc); 6613 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); 6614 if (sb == NULL) 6615 return (ENOMEM); 6616 6617 if (pi->linkdnrc < 0) 6618 sbuf_printf(sb, "n/a"); 6619 else if (pi->linkdnrc < nitems(linkdnreasons)) 6620 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]); 6621 else 6622 sbuf_printf(sb, "%d", pi->linkdnrc); 6623 6624 rc = sbuf_finish(sb); 6625 sbuf_delete(sb); 6626 6627 return (rc); 6628 } 6629 6630 struct mem_desc { 6631 unsigned int base; 6632 unsigned int limit; 6633 unsigned int idx; 6634 }; 6635 6636 static int 6637 mem_desc_cmp(const void *a, const void *b) 6638 { 6639 return ((const struct mem_desc *)a)->base - 6640 ((const struct mem_desc *)b)->base; 6641 } 6642 6643 static void 6644 mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 6645 unsigned int to) 6646 { 6647 unsigned int size; 6648 6649 size = to - from + 1; 6650 if (size == 0) 6651 return; 6652 6653 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 6654 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 6655 } 6656 6657 static int 6658 sysctl_meminfo(SYSCTL_HANDLER_ARGS) 6659 { 6660 struct adapter *sc = arg1; 6661 struct sbuf *sb; 6662 int rc, i, n; 6663 uint32_t lo, hi, used, alloc; 6664 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 6665 static const char *region[] = { 6666 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 6667 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 6668 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 6669 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 6670 "RQUDP region:", "PBL region:", "TXPBL region:", 6671 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 6672 "On-chip queues:" 6673 }; 6674 struct mem_desc avail[4]; 6675 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 6676 struct mem_desc *md = mem; 6677 6678 rc = sysctl_wire_old_buffer(req, 0); 6679 if (rc != 0) 6680 return (rc); 6681 6682 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6683 if (sb == NULL) 6684 return (ENOMEM); 6685 6686 for (i = 0; i < nitems(mem); i++) { 6687 mem[i].limit = 0; 6688 mem[i].idx = i; 6689 } 6690 6691 /* Find and sort the populated memory ranges */ 6692 i = 0; 6693 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 6694 if (lo & F_EDRAM0_ENABLE) { 6695 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 6696 avail[i].base = G_EDRAM0_BASE(hi) << 20; 6697 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 6698 avail[i].idx = 0; 6699 i++; 6700 } 6701 if (lo & F_EDRAM1_ENABLE) { 6702 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 6703 avail[i].base = G_EDRAM1_BASE(hi) << 20; 6704 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 6705 avail[i].idx = 1; 6706 i++; 6707 } 6708 if (lo & F_EXT_MEM_ENABLE) { 6709 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 6710 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 6711 avail[i].limit = avail[i].base + 6712 (G_EXT_MEM_SIZE(hi) << 20); 6713 avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */ 6714 i++; 6715 } 6716 if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) { 6717 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 6718 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 6719 avail[i].limit = avail[i].base + 6720 (G_EXT_MEM1_SIZE(hi) << 20); 6721 avail[i].idx = 4; 6722 i++; 6723 } 6724 if (!i) /* no memory available */ 6725 return 0; 6726 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 6727 6728 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 6729 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 6730 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 6731 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 6732 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 6733 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 6734 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 6735 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 6736 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 6737 6738 /* the next few have explicit upper bounds */ 6739 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 6740 md->limit = md->base - 1 + 6741 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 6742 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 6743 md++; 6744 6745 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 6746 md->limit = md->base - 1 + 6747 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 6748 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 6749 md++; 6750 6751 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6752 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 6753 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 6754 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 6755 } else { 6756 md->base = 0; 6757 md->idx = nitems(region); /* hide it */ 6758 } 6759 md++; 6760 6761 #define ulp_region(reg) \ 6762 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 6763 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 6764 6765 ulp_region(RX_ISCSI); 6766 ulp_region(RX_TDDP); 6767 ulp_region(TX_TPT); 6768 ulp_region(RX_STAG); 6769 ulp_region(RX_RQ); 6770 ulp_region(RX_RQUDP); 6771 ulp_region(RX_PBL); 6772 ulp_region(TX_PBL); 6773 #undef ulp_region 6774 6775 md->base = 0; 6776 md->idx = nitems(region); 6777 if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) { 6778 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); 6779 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc, 6780 A_SGE_DBVFIFO_SIZE))) << 2) - 1; 6781 } 6782 md++; 6783 6784 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 6785 md->limit = md->base + sc->tids.ntids - 1; 6786 md++; 6787 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 6788 md->limit = md->base + sc->tids.ntids - 1; 6789 md++; 6790 6791 md->base = sc->vres.ocq.start; 6792 if (sc->vres.ocq.size) 6793 md->limit = md->base + sc->vres.ocq.size - 1; 6794 else 6795 md->idx = nitems(region); /* hide it */ 6796 md++; 6797 6798 /* add any address-space holes, there can be up to 3 */ 6799 for (n = 0; n < i - 1; n++) 6800 if (avail[n].limit < avail[n + 1].base) 6801 (md++)->base = avail[n].limit; 6802 if (avail[n].limit) 6803 (md++)->base = avail[n].limit; 6804 6805 n = md - mem; 6806 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 6807 6808 for (lo = 0; lo < i; lo++) 6809 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 6810 avail[lo].limit - 1); 6811 6812 sbuf_printf(sb, "\n"); 6813 for (i = 0; i < n; i++) { 6814 if (mem[i].idx >= nitems(region)) 6815 continue; /* skip holes */ 6816 if (!mem[i].limit) 6817 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 6818 mem_region_show(sb, region[mem[i].idx], mem[i].base, 6819 mem[i].limit); 6820 } 6821 6822 sbuf_printf(sb, "\n"); 6823 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 6824 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 6825 mem_region_show(sb, "uP RAM:", lo, hi); 6826 6827 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 6828 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 6829 mem_region_show(sb, "uP Extmem2:", lo, hi); 6830 6831 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 6832 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 6833 G_PMRXMAXPAGE(lo), 6834 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 6835 (lo & F_PMRXNUMCHN) ? 2 : 1); 6836 6837 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 6838 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 6839 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 6840 G_PMTXMAXPAGE(lo), 6841 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 6842 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 6843 sbuf_printf(sb, "%u p-structs\n", 6844 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 6845 6846 for (i = 0; i < 4; i++) { 6847 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 6848 if (is_t4(sc)) { 6849 used = G_USED(lo); 6850 alloc = G_ALLOC(lo); 6851 } else { 6852 used = G_T5_USED(lo); 6853 alloc = G_T5_ALLOC(lo); 6854 } 6855 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 6856 i, used, alloc); 6857 } 6858 for (i = 0; i < 4; i++) { 6859 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 6860 if (is_t4(sc)) { 6861 used = G_USED(lo); 6862 alloc = G_ALLOC(lo); 6863 } else { 6864 used = G_T5_USED(lo); 6865 alloc = G_T5_ALLOC(lo); 6866 } 6867 sbuf_printf(sb, 6868 "\nLoopback %d using %u pages out of %u allocated", 6869 i, used, alloc); 6870 } 6871 6872 rc = sbuf_finish(sb); 6873 sbuf_delete(sb); 6874 6875 return (rc); 6876 } 6877 6878 static inline void 6879 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) 6880 { 6881 *mask = x | y; 6882 y = htobe64(y); 6883 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); 6884 } 6885 6886 static int 6887 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) 6888 { 6889 struct adapter *sc = arg1; 6890 struct sbuf *sb; 6891 int rc, i, n; 6892 6893 rc = sysctl_wire_old_buffer(req, 0); 6894 if (rc != 0) 6895 return (rc); 6896 6897 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6898 if (sb == NULL) 6899 return (ENOMEM); 6900 6901 sbuf_printf(sb, 6902 "Idx Ethernet address Mask Vld Ports PF" 6903 " VF Replication P0 P1 P2 P3 ML"); 6904 n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES : 6905 NUM_MPS_T5_CLS_SRAM_L_INSTANCES; 6906 for (i = 0; i < n; i++) { 6907 uint64_t tcamx, tcamy, mask; 6908 uint32_t cls_lo, cls_hi; 6909 uint8_t addr[ETHER_ADDR_LEN]; 6910 6911 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); 6912 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); 6913 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 6914 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 6915 6916 if (tcamx & tcamy) 6917 continue; 6918 6919 tcamxy2valmask(tcamx, tcamy, addr, &mask); 6920 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" 6921 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], 6922 addr[3], addr[4], addr[5], (uintmax_t)mask, 6923 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', 6924 G_PORTMAP(cls_hi), G_PF(cls_lo), 6925 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); 6926 6927 if (cls_lo & F_REPLICATE) { 6928 struct fw_ldst_cmd ldst_cmd; 6929 6930 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 6931 ldst_cmd.op_to_addrspace = 6932 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 6933 F_FW_CMD_REQUEST | F_FW_CMD_READ | 6934 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 6935 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 6936 ldst_cmd.u.mps.rplc.fid_idx = 6937 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 6938 V_FW_LDST_CMD_IDX(i)); 6939 6940 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 6941 "t4mps"); 6942 if (rc) 6943 break; 6944 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 6945 sizeof(ldst_cmd), &ldst_cmd); 6946 end_synchronized_op(sc, 0); 6947 6948 if (rc != 0) { 6949 sbuf_printf(sb, 6950 " ------------ error %3u ------------", rc); 6951 rc = 0; 6952 } else { 6953 sbuf_printf(sb, " %08x %08x %08x %08x", 6954 be32toh(ldst_cmd.u.mps.rplc.rplc127_96), 6955 be32toh(ldst_cmd.u.mps.rplc.rplc95_64), 6956 be32toh(ldst_cmd.u.mps.rplc.rplc63_32), 6957 be32toh(ldst_cmd.u.mps.rplc.rplc31_0)); 6958 } 6959 } else 6960 sbuf_printf(sb, "%36s", ""); 6961 6962 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), 6963 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), 6964 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); 6965 } 6966 6967 if (rc) 6968 (void) sbuf_finish(sb); 6969 else 6970 rc = sbuf_finish(sb); 6971 sbuf_delete(sb); 6972 6973 return (rc); 6974 } 6975 6976 static int 6977 sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 6978 { 6979 struct adapter *sc = arg1; 6980 struct sbuf *sb; 6981 int rc; 6982 uint16_t mtus[NMTUS]; 6983 6984 rc = sysctl_wire_old_buffer(req, 0); 6985 if (rc != 0) 6986 return (rc); 6987 6988 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6989 if (sb == NULL) 6990 return (ENOMEM); 6991 6992 t4_read_mtu_tbl(sc, mtus, NULL); 6993 6994 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 6995 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 6996 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 6997 mtus[14], mtus[15]); 6998 6999 rc = sbuf_finish(sb); 7000 sbuf_delete(sb); 7001 7002 return (rc); 7003 } 7004 7005 static int 7006 sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 7007 { 7008 struct adapter *sc = arg1; 7009 struct sbuf *sb; 7010 int rc, i; 7011 uint32_t cnt[PM_NSTATS]; 7012 uint64_t cyc[PM_NSTATS]; 7013 static const char *rx_stats[] = { 7014 "Read:", "Write bypass:", "Write mem:", "Flush:" 7015 }; 7016 static const char *tx_stats[] = { 7017 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:" 7018 }; 7019 7020 rc = sysctl_wire_old_buffer(req, 0); 7021 if (rc != 0) 7022 return (rc); 7023 7024 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7025 if (sb == NULL) 7026 return (ENOMEM); 7027 7028 t4_pmtx_get_stats(sc, cnt, cyc); 7029 sbuf_printf(sb, " Tx pcmds Tx bytes"); 7030 for (i = 0; i < ARRAY_SIZE(tx_stats); i++) 7031 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i], 7032 cyc[i]); 7033 7034 t4_pmrx_get_stats(sc, cnt, cyc); 7035 sbuf_printf(sb, "\n Rx pcmds Rx bytes"); 7036 for (i = 0; i < ARRAY_SIZE(rx_stats); i++) 7037 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i], 7038 cyc[i]); 7039 7040 rc = sbuf_finish(sb); 7041 sbuf_delete(sb); 7042 7043 return (rc); 7044 } 7045 7046 static int 7047 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 7048 { 7049 struct adapter *sc = arg1; 7050 struct sbuf *sb; 7051 int rc; 7052 struct tp_rdma_stats stats; 7053 7054 rc = sysctl_wire_old_buffer(req, 0); 7055 if (rc != 0) 7056 return (rc); 7057 7058 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7059 if (sb == NULL) 7060 return (ENOMEM); 7061 7062 t4_tp_get_rdma_stats(sc, &stats); 7063 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 7064 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 7065 7066 rc = sbuf_finish(sb); 7067 sbuf_delete(sb); 7068 7069 return (rc); 7070 } 7071 7072 static int 7073 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 7074 { 7075 struct adapter *sc = arg1; 7076 struct sbuf *sb; 7077 int rc; 7078 struct tp_tcp_stats v4, v6; 7079 7080 rc = sysctl_wire_old_buffer(req, 0); 7081 if (rc != 0) 7082 return (rc); 7083 7084 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7085 if (sb == NULL) 7086 return (ENOMEM); 7087 7088 t4_tp_get_tcp_stats(sc, &v4, &v6); 7089 sbuf_printf(sb, 7090 " IP IPv6\n"); 7091 sbuf_printf(sb, "OutRsts: %20u %20u\n", 7092 v4.tcpOutRsts, v6.tcpOutRsts); 7093 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 7094 v4.tcpInSegs, v6.tcpInSegs); 7095 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 7096 v4.tcpOutSegs, v6.tcpOutSegs); 7097 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 7098 v4.tcpRetransSegs, v6.tcpRetransSegs); 7099 7100 rc = sbuf_finish(sb); 7101 sbuf_delete(sb); 7102 7103 return (rc); 7104 } 7105 7106 static int 7107 sysctl_tids(SYSCTL_HANDLER_ARGS) 7108 { 7109 struct adapter *sc = arg1; 7110 struct sbuf *sb; 7111 int rc; 7112 struct tid_info *t = &sc->tids; 7113 7114 rc = sysctl_wire_old_buffer(req, 0); 7115 if (rc != 0) 7116 return (rc); 7117 7118 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7119 if (sb == NULL) 7120 return (ENOMEM); 7121 7122 if (t->natids) { 7123 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 7124 t->atids_in_use); 7125 } 7126 7127 if (t->ntids) { 7128 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 7129 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 7130 7131 if (b) { 7132 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 7133 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 7134 t->ntids - 1); 7135 } else { 7136 sbuf_printf(sb, "TID range: %u-%u", 7137 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 7138 t->ntids - 1); 7139 } 7140 } else 7141 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 7142 sbuf_printf(sb, ", in use: %u\n", 7143 atomic_load_acq_int(&t->tids_in_use)); 7144 } 7145 7146 if (t->nstids) { 7147 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 7148 t->stid_base + t->nstids - 1, t->stids_in_use); 7149 } 7150 7151 if (t->nftids) { 7152 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 7153 t->ftid_base + t->nftids - 1); 7154 } 7155 7156 if (t->netids) { 7157 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base, 7158 t->etid_base + t->netids - 1); 7159 } 7160 7161 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 7162 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 7163 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 7164 7165 rc = sbuf_finish(sb); 7166 sbuf_delete(sb); 7167 7168 return (rc); 7169 } 7170 7171 static int 7172 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 7173 { 7174 struct adapter *sc = arg1; 7175 struct sbuf *sb; 7176 int rc; 7177 struct tp_err_stats stats; 7178 7179 rc = sysctl_wire_old_buffer(req, 0); 7180 if (rc != 0) 7181 return (rc); 7182 7183 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7184 if (sb == NULL) 7185 return (ENOMEM); 7186 7187 t4_tp_get_err_stats(sc, &stats); 7188 7189 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 7190 "channel 3\n"); 7191 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 7192 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 7193 stats.macInErrs[3]); 7194 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 7195 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 7196 stats.hdrInErrs[3]); 7197 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 7198 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 7199 stats.tcpInErrs[3]); 7200 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 7201 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 7202 stats.tcp6InErrs[3]); 7203 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 7204 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 7205 stats.tnlCongDrops[3]); 7206 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 7207 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 7208 stats.tnlTxDrops[3]); 7209 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 7210 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 7211 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 7212 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 7213 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 7214 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 7215 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 7216 stats.ofldNoNeigh, stats.ofldCongDefer); 7217 7218 rc = sbuf_finish(sb); 7219 sbuf_delete(sb); 7220 7221 return (rc); 7222 } 7223 7224 struct field_desc { 7225 const char *name; 7226 u_int start; 7227 u_int width; 7228 }; 7229 7230 static void 7231 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) 7232 { 7233 char buf[32]; 7234 int line_size = 0; 7235 7236 while (f->name) { 7237 uint64_t mask = (1ULL << f->width) - 1; 7238 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, 7239 ((uintmax_t)v >> f->start) & mask); 7240 7241 if (line_size + len >= 79) { 7242 line_size = 8; 7243 sbuf_printf(sb, "\n "); 7244 } 7245 sbuf_printf(sb, "%s ", buf); 7246 line_size += len + 1; 7247 f++; 7248 } 7249 sbuf_printf(sb, "\n"); 7250 } 7251 7252 static struct field_desc tp_la0[] = { 7253 { "RcfOpCodeOut", 60, 4 }, 7254 { "State", 56, 4 }, 7255 { "WcfState", 52, 4 }, 7256 { "RcfOpcSrcOut", 50, 2 }, 7257 { "CRxError", 49, 1 }, 7258 { "ERxError", 48, 1 }, 7259 { "SanityFailed", 47, 1 }, 7260 { "SpuriousMsg", 46, 1 }, 7261 { "FlushInputMsg", 45, 1 }, 7262 { "FlushInputCpl", 44, 1 }, 7263 { "RssUpBit", 43, 1 }, 7264 { "RssFilterHit", 42, 1 }, 7265 { "Tid", 32, 10 }, 7266 { "InitTcb", 31, 1 }, 7267 { "LineNumber", 24, 7 }, 7268 { "Emsg", 23, 1 }, 7269 { "EdataOut", 22, 1 }, 7270 { "Cmsg", 21, 1 }, 7271 { "CdataOut", 20, 1 }, 7272 { "EreadPdu", 19, 1 }, 7273 { "CreadPdu", 18, 1 }, 7274 { "TunnelPkt", 17, 1 }, 7275 { "RcfPeerFin", 16, 1 }, 7276 { "RcfReasonOut", 12, 4 }, 7277 { "TxCchannel", 10, 2 }, 7278 { "RcfTxChannel", 8, 2 }, 7279 { "RxEchannel", 6, 2 }, 7280 { "RcfRxChannel", 5, 1 }, 7281 { "RcfDataOutSrdy", 4, 1 }, 7282 { "RxDvld", 3, 1 }, 7283 { "RxOoDvld", 2, 1 }, 7284 { "RxCongestion", 1, 1 }, 7285 { "TxCongestion", 0, 1 }, 7286 { NULL } 7287 }; 7288 7289 static struct field_desc tp_la1[] = { 7290 { "CplCmdIn", 56, 8 }, 7291 { "CplCmdOut", 48, 8 }, 7292 { "ESynOut", 47, 1 }, 7293 { "EAckOut", 46, 1 }, 7294 { "EFinOut", 45, 1 }, 7295 { "ERstOut", 44, 1 }, 7296 { "SynIn", 43, 1 }, 7297 { "AckIn", 42, 1 }, 7298 { "FinIn", 41, 1 }, 7299 { "RstIn", 40, 1 }, 7300 { "DataIn", 39, 1 }, 7301 { "DataInVld", 38, 1 }, 7302 { "PadIn", 37, 1 }, 7303 { "RxBufEmpty", 36, 1 }, 7304 { "RxDdp", 35, 1 }, 7305 { "RxFbCongestion", 34, 1 }, 7306 { "TxFbCongestion", 33, 1 }, 7307 { "TxPktSumSrdy", 32, 1 }, 7308 { "RcfUlpType", 28, 4 }, 7309 { "Eread", 27, 1 }, 7310 { "Ebypass", 26, 1 }, 7311 { "Esave", 25, 1 }, 7312 { "Static0", 24, 1 }, 7313 { "Cread", 23, 1 }, 7314 { "Cbypass", 22, 1 }, 7315 { "Csave", 21, 1 }, 7316 { "CPktOut", 20, 1 }, 7317 { "RxPagePoolFull", 18, 2 }, 7318 { "RxLpbkPkt", 17, 1 }, 7319 { "TxLpbkPkt", 16, 1 }, 7320 { "RxVfValid", 15, 1 }, 7321 { "SynLearned", 14, 1 }, 7322 { "SetDelEntry", 13, 1 }, 7323 { "SetInvEntry", 12, 1 }, 7324 { "CpcmdDvld", 11, 1 }, 7325 { "CpcmdSave", 10, 1 }, 7326 { "RxPstructsFull", 8, 2 }, 7327 { "EpcmdDvld", 7, 1 }, 7328 { "EpcmdFlush", 6, 1 }, 7329 { "EpcmdTrimPrefix", 5, 1 }, 7330 { "EpcmdTrimPostfix", 4, 1 }, 7331 { "ERssIp4Pkt", 3, 1 }, 7332 { "ERssIp6Pkt", 2, 1 }, 7333 { "ERssTcpUdpPkt", 1, 1 }, 7334 { "ERssFceFipPkt", 0, 1 }, 7335 { NULL } 7336 }; 7337 7338 static struct field_desc tp_la2[] = { 7339 { "CplCmdIn", 56, 8 }, 7340 { "MpsVfVld", 55, 1 }, 7341 { "MpsPf", 52, 3 }, 7342 { "MpsVf", 44, 8 }, 7343 { "SynIn", 43, 1 }, 7344 { "AckIn", 42, 1 }, 7345 { "FinIn", 41, 1 }, 7346 { "RstIn", 40, 1 }, 7347 { "DataIn", 39, 1 }, 7348 { "DataInVld", 38, 1 }, 7349 { "PadIn", 37, 1 }, 7350 { "RxBufEmpty", 36, 1 }, 7351 { "RxDdp", 35, 1 }, 7352 { "RxFbCongestion", 34, 1 }, 7353 { "TxFbCongestion", 33, 1 }, 7354 { "TxPktSumSrdy", 32, 1 }, 7355 { "RcfUlpType", 28, 4 }, 7356 { "Eread", 27, 1 }, 7357 { "Ebypass", 26, 1 }, 7358 { "Esave", 25, 1 }, 7359 { "Static0", 24, 1 }, 7360 { "Cread", 23, 1 }, 7361 { "Cbypass", 22, 1 }, 7362 { "Csave", 21, 1 }, 7363 { "CPktOut", 20, 1 }, 7364 { "RxPagePoolFull", 18, 2 }, 7365 { "RxLpbkPkt", 17, 1 }, 7366 { "TxLpbkPkt", 16, 1 }, 7367 { "RxVfValid", 15, 1 }, 7368 { "SynLearned", 14, 1 }, 7369 { "SetDelEntry", 13, 1 }, 7370 { "SetInvEntry", 12, 1 }, 7371 { "CpcmdDvld", 11, 1 }, 7372 { "CpcmdSave", 10, 1 }, 7373 { "RxPstructsFull", 8, 2 }, 7374 { "EpcmdDvld", 7, 1 }, 7375 { "EpcmdFlush", 6, 1 }, 7376 { "EpcmdTrimPrefix", 5, 1 }, 7377 { "EpcmdTrimPostfix", 4, 1 }, 7378 { "ERssIp4Pkt", 3, 1 }, 7379 { "ERssIp6Pkt", 2, 1 }, 7380 { "ERssTcpUdpPkt", 1, 1 }, 7381 { "ERssFceFipPkt", 0, 1 }, 7382 { NULL } 7383 }; 7384 7385 static void 7386 tp_la_show(struct sbuf *sb, uint64_t *p, int idx) 7387 { 7388 7389 field_desc_show(sb, *p, tp_la0); 7390 } 7391 7392 static void 7393 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) 7394 { 7395 7396 if (idx) 7397 sbuf_printf(sb, "\n"); 7398 field_desc_show(sb, p[0], tp_la0); 7399 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 7400 field_desc_show(sb, p[1], tp_la0); 7401 } 7402 7403 static void 7404 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) 7405 { 7406 7407 if (idx) 7408 sbuf_printf(sb, "\n"); 7409 field_desc_show(sb, p[0], tp_la0); 7410 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 7411 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); 7412 } 7413 7414 static int 7415 sysctl_tp_la(SYSCTL_HANDLER_ARGS) 7416 { 7417 struct adapter *sc = arg1; 7418 struct sbuf *sb; 7419 uint64_t *buf, *p; 7420 int rc; 7421 u_int i, inc; 7422 void (*show_func)(struct sbuf *, uint64_t *, int); 7423 7424 rc = sysctl_wire_old_buffer(req, 0); 7425 if (rc != 0) 7426 return (rc); 7427 7428 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7429 if (sb == NULL) 7430 return (ENOMEM); 7431 7432 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); 7433 7434 t4_tp_read_la(sc, buf, NULL); 7435 p = buf; 7436 7437 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { 7438 case 2: 7439 inc = 2; 7440 show_func = tp_la_show2; 7441 break; 7442 case 3: 7443 inc = 2; 7444 show_func = tp_la_show3; 7445 break; 7446 default: 7447 inc = 1; 7448 show_func = tp_la_show; 7449 } 7450 7451 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) 7452 (*show_func)(sb, p, i); 7453 7454 rc = sbuf_finish(sb); 7455 sbuf_delete(sb); 7456 free(buf, M_CXGBE); 7457 return (rc); 7458 } 7459 7460 static int 7461 sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 7462 { 7463 struct adapter *sc = arg1; 7464 struct sbuf *sb; 7465 int rc; 7466 u64 nrate[NCHAN], orate[NCHAN]; 7467 7468 rc = sysctl_wire_old_buffer(req, 0); 7469 if (rc != 0) 7470 return (rc); 7471 7472 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7473 if (sb == NULL) 7474 return (ENOMEM); 7475 7476 t4_get_chan_txrate(sc, nrate, orate); 7477 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 7478 "channel 3\n"); 7479 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 7480 nrate[0], nrate[1], nrate[2], nrate[3]); 7481 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 7482 orate[0], orate[1], orate[2], orate[3]); 7483 7484 rc = sbuf_finish(sb); 7485 sbuf_delete(sb); 7486 7487 return (rc); 7488 } 7489 7490 static int 7491 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) 7492 { 7493 struct adapter *sc = arg1; 7494 struct sbuf *sb; 7495 uint32_t *buf, *p; 7496 int rc, i; 7497 7498 rc = sysctl_wire_old_buffer(req, 0); 7499 if (rc != 0) 7500 return (rc); 7501 7502 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7503 if (sb == NULL) 7504 return (ENOMEM); 7505 7506 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, 7507 M_ZERO | M_WAITOK); 7508 7509 t4_ulprx_read_la(sc, buf); 7510 p = buf; 7511 7512 sbuf_printf(sb, " Pcmd Type Message" 7513 " Data"); 7514 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { 7515 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", 7516 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); 7517 } 7518 7519 rc = sbuf_finish(sb); 7520 sbuf_delete(sb); 7521 free(buf, M_CXGBE); 7522 return (rc); 7523 } 7524 7525 static int 7526 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 7527 { 7528 struct adapter *sc = arg1; 7529 struct sbuf *sb; 7530 int rc, v; 7531 7532 rc = sysctl_wire_old_buffer(req, 0); 7533 if (rc != 0) 7534 return (rc); 7535 7536 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7537 if (sb == NULL) 7538 return (ENOMEM); 7539 7540 v = t4_read_reg(sc, A_SGE_STAT_CFG); 7541 if (G_STATSOURCE_T5(v) == 7) { 7542 if (G_STATMODE(v) == 0) { 7543 sbuf_printf(sb, "total %d, incomplete %d", 7544 t4_read_reg(sc, A_SGE_STAT_TOTAL), 7545 t4_read_reg(sc, A_SGE_STAT_MATCH)); 7546 } else if (G_STATMODE(v) == 1) { 7547 sbuf_printf(sb, "total %d, data overflow %d", 7548 t4_read_reg(sc, A_SGE_STAT_TOTAL), 7549 t4_read_reg(sc, A_SGE_STAT_MATCH)); 7550 } 7551 } 7552 rc = sbuf_finish(sb); 7553 sbuf_delete(sb); 7554 7555 return (rc); 7556 } 7557 #endif 7558 7559 static uint32_t 7560 fconf_to_mode(uint32_t fconf) 7561 { 7562 uint32_t mode; 7563 7564 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 7565 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 7566 7567 if (fconf & F_FRAGMENTATION) 7568 mode |= T4_FILTER_IP_FRAGMENT; 7569 7570 if (fconf & F_MPSHITTYPE) 7571 mode |= T4_FILTER_MPS_HIT_TYPE; 7572 7573 if (fconf & F_MACMATCH) 7574 mode |= T4_FILTER_MAC_IDX; 7575 7576 if (fconf & F_ETHERTYPE) 7577 mode |= T4_FILTER_ETH_TYPE; 7578 7579 if (fconf & F_PROTOCOL) 7580 mode |= T4_FILTER_IP_PROTO; 7581 7582 if (fconf & F_TOS) 7583 mode |= T4_FILTER_IP_TOS; 7584 7585 if (fconf & F_VLAN) 7586 mode |= T4_FILTER_VLAN; 7587 7588 if (fconf & F_VNIC_ID) 7589 mode |= T4_FILTER_VNIC; 7590 7591 if (fconf & F_PORT) 7592 mode |= T4_FILTER_PORT; 7593 7594 if (fconf & F_FCOE) 7595 mode |= T4_FILTER_FCoE; 7596 7597 return (mode); 7598 } 7599 7600 static uint32_t 7601 mode_to_fconf(uint32_t mode) 7602 { 7603 uint32_t fconf = 0; 7604 7605 if (mode & T4_FILTER_IP_FRAGMENT) 7606 fconf |= F_FRAGMENTATION; 7607 7608 if (mode & T4_FILTER_MPS_HIT_TYPE) 7609 fconf |= F_MPSHITTYPE; 7610 7611 if (mode & T4_FILTER_MAC_IDX) 7612 fconf |= F_MACMATCH; 7613 7614 if (mode & T4_FILTER_ETH_TYPE) 7615 fconf |= F_ETHERTYPE; 7616 7617 if (mode & T4_FILTER_IP_PROTO) 7618 fconf |= F_PROTOCOL; 7619 7620 if (mode & T4_FILTER_IP_TOS) 7621 fconf |= F_TOS; 7622 7623 if (mode & T4_FILTER_VLAN) 7624 fconf |= F_VLAN; 7625 7626 if (mode & T4_FILTER_VNIC) 7627 fconf |= F_VNIC_ID; 7628 7629 if (mode & T4_FILTER_PORT) 7630 fconf |= F_PORT; 7631 7632 if (mode & T4_FILTER_FCoE) 7633 fconf |= F_FCOE; 7634 7635 return (fconf); 7636 } 7637 7638 static uint32_t 7639 fspec_to_fconf(struct t4_filter_specification *fs) 7640 { 7641 uint32_t fconf = 0; 7642 7643 if (fs->val.frag || fs->mask.frag) 7644 fconf |= F_FRAGMENTATION; 7645 7646 if (fs->val.matchtype || fs->mask.matchtype) 7647 fconf |= F_MPSHITTYPE; 7648 7649 if (fs->val.macidx || fs->mask.macidx) 7650 fconf |= F_MACMATCH; 7651 7652 if (fs->val.ethtype || fs->mask.ethtype) 7653 fconf |= F_ETHERTYPE; 7654 7655 if (fs->val.proto || fs->mask.proto) 7656 fconf |= F_PROTOCOL; 7657 7658 if (fs->val.tos || fs->mask.tos) 7659 fconf |= F_TOS; 7660 7661 if (fs->val.vlan_vld || fs->mask.vlan_vld) 7662 fconf |= F_VLAN; 7663 7664 if (fs->val.vnic_vld || fs->mask.vnic_vld) 7665 fconf |= F_VNIC_ID; 7666 7667 if (fs->val.iport || fs->mask.iport) 7668 fconf |= F_PORT; 7669 7670 if (fs->val.fcoe || fs->mask.fcoe) 7671 fconf |= F_FCOE; 7672 7673 return (fconf); 7674 } 7675 7676 static int 7677 get_filter_mode(struct adapter *sc, uint32_t *mode) 7678 { 7679 int rc; 7680 uint32_t fconf; 7681 7682 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7683 "t4getfm"); 7684 if (rc) 7685 return (rc); 7686 7687 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 7688 A_TP_VLAN_PRI_MAP); 7689 7690 if (sc->params.tp.vlan_pri_map != fconf) { 7691 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 7692 device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map, 7693 fconf); 7694 } 7695 7696 *mode = fconf_to_mode(fconf); 7697 7698 end_synchronized_op(sc, LOCK_HELD); 7699 return (0); 7700 } 7701 7702 static int 7703 set_filter_mode(struct adapter *sc, uint32_t mode) 7704 { 7705 uint32_t fconf; 7706 int rc; 7707 7708 fconf = mode_to_fconf(mode); 7709 7710 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7711 "t4setfm"); 7712 if (rc) 7713 return (rc); 7714 7715 if (sc->tids.ftids_in_use > 0) { 7716 rc = EBUSY; 7717 goto done; 7718 } 7719 7720 #ifdef TCP_OFFLOAD 7721 if (uld_active(sc, ULD_TOM)) { 7722 rc = EBUSY; 7723 goto done; 7724 } 7725 #endif 7726 7727 rc = -t4_set_filter_mode(sc, fconf); 7728 done: 7729 end_synchronized_op(sc, LOCK_HELD); 7730 return (rc); 7731 } 7732 7733 static inline uint64_t 7734 get_filter_hits(struct adapter *sc, uint32_t fid) 7735 { 7736 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 7737 uint64_t hits; 7738 7739 memwin_info(sc, 0, &mw_base, NULL); 7740 off = position_memwin(sc, 0, 7741 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 7742 if (is_t4(sc)) { 7743 hits = t4_read_reg64(sc, mw_base + off + 16); 7744 hits = be64toh(hits); 7745 } else { 7746 hits = t4_read_reg(sc, mw_base + off + 24); 7747 hits = be32toh(hits); 7748 } 7749 7750 return (hits); 7751 } 7752 7753 static int 7754 get_filter(struct adapter *sc, struct t4_filter *t) 7755 { 7756 int i, rc, nfilters = sc->tids.nftids; 7757 struct filter_entry *f; 7758 7759 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7760 "t4getf"); 7761 if (rc) 7762 return (rc); 7763 7764 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 7765 t->idx >= nfilters) { 7766 t->idx = 0xffffffff; 7767 goto done; 7768 } 7769 7770 f = &sc->tids.ftid_tab[t->idx]; 7771 for (i = t->idx; i < nfilters; i++, f++) { 7772 if (f->valid) { 7773 t->idx = i; 7774 t->l2tidx = f->l2t ? f->l2t->idx : 0; 7775 t->smtidx = f->smtidx; 7776 if (f->fs.hitcnts) 7777 t->hits = get_filter_hits(sc, t->idx); 7778 else 7779 t->hits = UINT64_MAX; 7780 t->fs = f->fs; 7781 7782 goto done; 7783 } 7784 } 7785 7786 t->idx = 0xffffffff; 7787 done: 7788 end_synchronized_op(sc, LOCK_HELD); 7789 return (0); 7790 } 7791 7792 static int 7793 set_filter(struct adapter *sc, struct t4_filter *t) 7794 { 7795 unsigned int nfilters, nports; 7796 struct filter_entry *f; 7797 int i, rc; 7798 7799 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 7800 if (rc) 7801 return (rc); 7802 7803 nfilters = sc->tids.nftids; 7804 nports = sc->params.nports; 7805 7806 if (nfilters == 0) { 7807 rc = ENOTSUP; 7808 goto done; 7809 } 7810 7811 if (!(sc->flags & FULL_INIT_DONE)) { 7812 rc = EAGAIN; 7813 goto done; 7814 } 7815 7816 if (t->idx >= nfilters) { 7817 rc = EINVAL; 7818 goto done; 7819 } 7820 7821 /* Validate against the global filter mode */ 7822 if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) != 7823 sc->params.tp.vlan_pri_map) { 7824 rc = E2BIG; 7825 goto done; 7826 } 7827 7828 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 7829 rc = EINVAL; 7830 goto done; 7831 } 7832 7833 if (t->fs.val.iport >= nports) { 7834 rc = EINVAL; 7835 goto done; 7836 } 7837 7838 /* Can't specify an iq if not steering to it */ 7839 if (!t->fs.dirsteer && t->fs.iq) { 7840 rc = EINVAL; 7841 goto done; 7842 } 7843 7844 /* IPv6 filter idx must be 4 aligned */ 7845 if (t->fs.type == 1 && 7846 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 7847 rc = EINVAL; 7848 goto done; 7849 } 7850 7851 if (sc->tids.ftid_tab == NULL) { 7852 KASSERT(sc->tids.ftids_in_use == 0, 7853 ("%s: no memory allocated but filters_in_use > 0", 7854 __func__)); 7855 7856 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 7857 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 7858 if (sc->tids.ftid_tab == NULL) { 7859 rc = ENOMEM; 7860 goto done; 7861 } 7862 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 7863 } 7864 7865 for (i = 0; i < 4; i++) { 7866 f = &sc->tids.ftid_tab[t->idx + i]; 7867 7868 if (f->pending || f->valid) { 7869 rc = EBUSY; 7870 goto done; 7871 } 7872 if (f->locked) { 7873 rc = EPERM; 7874 goto done; 7875 } 7876 7877 if (t->fs.type == 0) 7878 break; 7879 } 7880 7881 f = &sc->tids.ftid_tab[t->idx]; 7882 f->fs = t->fs; 7883 7884 rc = set_filter_wr(sc, t->idx); 7885 done: 7886 end_synchronized_op(sc, 0); 7887 7888 if (rc == 0) { 7889 mtx_lock(&sc->tids.ftid_lock); 7890 for (;;) { 7891 if (f->pending == 0) { 7892 rc = f->valid ? 0 : EIO; 7893 break; 7894 } 7895 7896 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7897 PCATCH, "t4setfw", 0)) { 7898 rc = EINPROGRESS; 7899 break; 7900 } 7901 } 7902 mtx_unlock(&sc->tids.ftid_lock); 7903 } 7904 return (rc); 7905 } 7906 7907 static int 7908 del_filter(struct adapter *sc, struct t4_filter *t) 7909 { 7910 unsigned int nfilters; 7911 struct filter_entry *f; 7912 int rc; 7913 7914 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 7915 if (rc) 7916 return (rc); 7917 7918 nfilters = sc->tids.nftids; 7919 7920 if (nfilters == 0) { 7921 rc = ENOTSUP; 7922 goto done; 7923 } 7924 7925 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 7926 t->idx >= nfilters) { 7927 rc = EINVAL; 7928 goto done; 7929 } 7930 7931 if (!(sc->flags & FULL_INIT_DONE)) { 7932 rc = EAGAIN; 7933 goto done; 7934 } 7935 7936 f = &sc->tids.ftid_tab[t->idx]; 7937 7938 if (f->pending) { 7939 rc = EBUSY; 7940 goto done; 7941 } 7942 if (f->locked) { 7943 rc = EPERM; 7944 goto done; 7945 } 7946 7947 if (f->valid) { 7948 t->fs = f->fs; /* extra info for the caller */ 7949 rc = del_filter_wr(sc, t->idx); 7950 } 7951 7952 done: 7953 end_synchronized_op(sc, 0); 7954 7955 if (rc == 0) { 7956 mtx_lock(&sc->tids.ftid_lock); 7957 for (;;) { 7958 if (f->pending == 0) { 7959 rc = f->valid ? EIO : 0; 7960 break; 7961 } 7962 7963 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7964 PCATCH, "t4delfw", 0)) { 7965 rc = EINPROGRESS; 7966 break; 7967 } 7968 } 7969 mtx_unlock(&sc->tids.ftid_lock); 7970 } 7971 7972 return (rc); 7973 } 7974 7975 static void 7976 clear_filter(struct filter_entry *f) 7977 { 7978 if (f->l2t) 7979 t4_l2t_release(f->l2t); 7980 7981 bzero(f, sizeof (*f)); 7982 } 7983 7984 static int 7985 set_filter_wr(struct adapter *sc, int fidx) 7986 { 7987 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7988 struct fw_filter_wr *fwr; 7989 unsigned int ftid; 7990 struct wrq_cookie cookie; 7991 7992 ASSERT_SYNCHRONIZED_OP(sc); 7993 7994 if (f->fs.newdmac || f->fs.newvlan) { 7995 /* This filter needs an L2T entry; allocate one. */ 7996 f->l2t = t4_l2t_alloc_switching(sc->l2t); 7997 if (f->l2t == NULL) 7998 return (EAGAIN); 7999 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 8000 f->fs.dmac)) { 8001 t4_l2t_release(f->l2t); 8002 f->l2t = NULL; 8003 return (ENOMEM); 8004 } 8005 } 8006 8007 ftid = sc->tids.ftid_base + fidx; 8008 8009 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 8010 if (fwr == NULL) 8011 return (ENOMEM); 8012 bzero(fwr, sizeof(*fwr)); 8013 8014 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 8015 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 8016 fwr->tid_to_iq = 8017 htobe32(V_FW_FILTER_WR_TID(ftid) | 8018 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 8019 V_FW_FILTER_WR_NOREPLY(0) | 8020 V_FW_FILTER_WR_IQ(f->fs.iq)); 8021 fwr->del_filter_to_l2tix = 8022 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 8023 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 8024 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 8025 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 8026 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 8027 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 8028 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 8029 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 8030 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 8031 f->fs.newvlan == VLAN_REWRITE) | 8032 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 8033 f->fs.newvlan == VLAN_REWRITE) | 8034 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 8035 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 8036 V_FW_FILTER_WR_PRIO(f->fs.prio) | 8037 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 8038 fwr->ethtype = htobe16(f->fs.val.ethtype); 8039 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 8040 fwr->frag_to_ovlan_vldm = 8041 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 8042 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 8043 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 8044 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 8045 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 8046 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 8047 fwr->smac_sel = 0; 8048 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 8049 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 8050 fwr->maci_to_matchtypem = 8051 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 8052 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 8053 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 8054 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 8055 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 8056 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 8057 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 8058 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 8059 fwr->ptcl = f->fs.val.proto; 8060 fwr->ptclm = f->fs.mask.proto; 8061 fwr->ttyp = f->fs.val.tos; 8062 fwr->ttypm = f->fs.mask.tos; 8063 fwr->ivlan = htobe16(f->fs.val.vlan); 8064 fwr->ivlanm = htobe16(f->fs.mask.vlan); 8065 fwr->ovlan = htobe16(f->fs.val.vnic); 8066 fwr->ovlanm = htobe16(f->fs.mask.vnic); 8067 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 8068 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 8069 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 8070 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 8071 fwr->lp = htobe16(f->fs.val.dport); 8072 fwr->lpm = htobe16(f->fs.mask.dport); 8073 fwr->fp = htobe16(f->fs.val.sport); 8074 fwr->fpm = htobe16(f->fs.mask.sport); 8075 if (f->fs.newsmac) 8076 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 8077 8078 f->pending = 1; 8079 sc->tids.ftids_in_use++; 8080 8081 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 8082 return (0); 8083 } 8084 8085 static int 8086 del_filter_wr(struct adapter *sc, int fidx) 8087 { 8088 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 8089 struct fw_filter_wr *fwr; 8090 unsigned int ftid; 8091 struct wrq_cookie cookie; 8092 8093 ftid = sc->tids.ftid_base + fidx; 8094 8095 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 8096 if (fwr == NULL) 8097 return (ENOMEM); 8098 bzero(fwr, sizeof (*fwr)); 8099 8100 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 8101 8102 f->pending = 1; 8103 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 8104 return (0); 8105 } 8106 8107 int 8108 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 8109 { 8110 struct adapter *sc = iq->adapter; 8111 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 8112 unsigned int idx = GET_TID(rpl); 8113 unsigned int rc; 8114 struct filter_entry *f; 8115 8116 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 8117 rss->opcode)); 8118 8119 if (is_ftid(sc, idx)) { 8120 8121 idx -= sc->tids.ftid_base; 8122 f = &sc->tids.ftid_tab[idx]; 8123 rc = G_COOKIE(rpl->cookie); 8124 8125 mtx_lock(&sc->tids.ftid_lock); 8126 if (rc == FW_FILTER_WR_FLT_ADDED) { 8127 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 8128 __func__, idx)); 8129 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 8130 f->pending = 0; /* asynchronous setup completed */ 8131 f->valid = 1; 8132 } else { 8133 if (rc != FW_FILTER_WR_FLT_DELETED) { 8134 /* Add or delete failed, display an error */ 8135 log(LOG_ERR, 8136 "filter %u setup failed with error %u\n", 8137 idx, rc); 8138 } 8139 8140 clear_filter(f); 8141 sc->tids.ftids_in_use--; 8142 } 8143 wakeup(&sc->tids.ftid_tab); 8144 mtx_unlock(&sc->tids.ftid_lock); 8145 } 8146 8147 return (0); 8148 } 8149 8150 static int 8151 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 8152 { 8153 int rc; 8154 8155 if (cntxt->cid > M_CTXTQID) 8156 return (EINVAL); 8157 8158 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 8159 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 8160 return (EINVAL); 8161 8162 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 8163 if (rc) 8164 return (rc); 8165 8166 if (sc->flags & FW_OK) { 8167 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 8168 &cntxt->data[0]); 8169 if (rc == 0) 8170 goto done; 8171 } 8172 8173 /* 8174 * Read via firmware failed or wasn't even attempted. Read directly via 8175 * the backdoor. 8176 */ 8177 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 8178 done: 8179 end_synchronized_op(sc, 0); 8180 return (rc); 8181 } 8182 8183 static int 8184 load_fw(struct adapter *sc, struct t4_data *fw) 8185 { 8186 int rc; 8187 uint8_t *fw_data; 8188 8189 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 8190 if (rc) 8191 return (rc); 8192 8193 if (sc->flags & FULL_INIT_DONE) { 8194 rc = EBUSY; 8195 goto done; 8196 } 8197 8198 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 8199 if (fw_data == NULL) { 8200 rc = ENOMEM; 8201 goto done; 8202 } 8203 8204 rc = copyin(fw->data, fw_data, fw->len); 8205 if (rc == 0) 8206 rc = -t4_load_fw(sc, fw_data, fw->len); 8207 8208 free(fw_data, M_CXGBE); 8209 done: 8210 end_synchronized_op(sc, 0); 8211 return (rc); 8212 } 8213 8214 static int 8215 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 8216 { 8217 uint32_t addr, off, remaining, i, n; 8218 uint32_t *buf, *b; 8219 uint32_t mw_base, mw_aperture; 8220 int rc; 8221 uint8_t *dst; 8222 8223 rc = validate_mem_range(sc, mr->addr, mr->len); 8224 if (rc != 0) 8225 return (rc); 8226 8227 memwin_info(sc, win, &mw_base, &mw_aperture); 8228 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 8229 addr = mr->addr; 8230 remaining = mr->len; 8231 dst = (void *)mr->data; 8232 8233 while (remaining) { 8234 off = position_memwin(sc, win, addr); 8235 8236 /* number of bytes that we'll copy in the inner loop */ 8237 n = min(remaining, mw_aperture - off); 8238 for (i = 0; i < n; i += 4) 8239 *b++ = t4_read_reg(sc, mw_base + off + i); 8240 8241 rc = copyout(buf, dst, n); 8242 if (rc != 0) 8243 break; 8244 8245 b = buf; 8246 dst += n; 8247 remaining -= n; 8248 addr += n; 8249 } 8250 8251 free(buf, M_CXGBE); 8252 return (rc); 8253 } 8254 8255 static int 8256 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 8257 { 8258 int rc; 8259 8260 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 8261 return (EINVAL); 8262 8263 if (i2cd->len > sizeof(i2cd->data)) 8264 return (EFBIG); 8265 8266 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 8267 if (rc) 8268 return (rc); 8269 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 8270 i2cd->offset, i2cd->len, &i2cd->data[0]); 8271 end_synchronized_op(sc, 0); 8272 8273 return (rc); 8274 } 8275 8276 static int 8277 in_range(int val, int lo, int hi) 8278 { 8279 8280 return (val < 0 || (val <= hi && val >= lo)); 8281 } 8282 8283 static int 8284 set_sched_class(struct adapter *sc, struct t4_sched_params *p) 8285 { 8286 int fw_subcmd, fw_type, rc; 8287 8288 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc"); 8289 if (rc) 8290 return (rc); 8291 8292 if (!(sc->flags & FULL_INIT_DONE)) { 8293 rc = EAGAIN; 8294 goto done; 8295 } 8296 8297 /* 8298 * Translate the cxgbetool parameters into T4 firmware parameters. (The 8299 * sub-command and type are in common locations.) 8300 */ 8301 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG) 8302 fw_subcmd = FW_SCHED_SC_CONFIG; 8303 else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS) 8304 fw_subcmd = FW_SCHED_SC_PARAMS; 8305 else { 8306 rc = EINVAL; 8307 goto done; 8308 } 8309 if (p->type == SCHED_CLASS_TYPE_PACKET) 8310 fw_type = FW_SCHED_TYPE_PKTSCHED; 8311 else { 8312 rc = EINVAL; 8313 goto done; 8314 } 8315 8316 if (fw_subcmd == FW_SCHED_SC_CONFIG) { 8317 /* Vet our parameters ..*/ 8318 if (p->u.config.minmax < 0) { 8319 rc = EINVAL; 8320 goto done; 8321 } 8322 8323 /* And pass the request to the firmware ...*/ 8324 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1); 8325 goto done; 8326 } 8327 8328 if (fw_subcmd == FW_SCHED_SC_PARAMS) { 8329 int fw_level; 8330 int fw_mode; 8331 int fw_rateunit; 8332 int fw_ratemode; 8333 8334 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL) 8335 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL; 8336 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) 8337 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR; 8338 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) 8339 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL; 8340 else { 8341 rc = EINVAL; 8342 goto done; 8343 } 8344 8345 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS) 8346 fw_mode = FW_SCHED_PARAMS_MODE_CLASS; 8347 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW) 8348 fw_mode = FW_SCHED_PARAMS_MODE_FLOW; 8349 else { 8350 rc = EINVAL; 8351 goto done; 8352 } 8353 8354 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS) 8355 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 8356 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS) 8357 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE; 8358 else { 8359 rc = EINVAL; 8360 goto done; 8361 } 8362 8363 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL) 8364 fw_ratemode = FW_SCHED_PARAMS_RATE_REL; 8365 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS) 8366 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS; 8367 else { 8368 rc = EINVAL; 8369 goto done; 8370 } 8371 8372 /* Vet our parameters ... */ 8373 if (!in_range(p->u.params.channel, 0, 3) || 8374 !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) || 8375 !in_range(p->u.params.minrate, 0, 10000000) || 8376 !in_range(p->u.params.maxrate, 0, 10000000) || 8377 !in_range(p->u.params.weight, 0, 100)) { 8378 rc = ERANGE; 8379 goto done; 8380 } 8381 8382 /* 8383 * Translate any unset parameters into the firmware's 8384 * nomenclature and/or fail the call if the parameters 8385 * are required ... 8386 */ 8387 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 || 8388 p->u.params.channel < 0 || p->u.params.cl < 0) { 8389 rc = EINVAL; 8390 goto done; 8391 } 8392 if (p->u.params.minrate < 0) 8393 p->u.params.minrate = 0; 8394 if (p->u.params.maxrate < 0) { 8395 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 8396 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 8397 rc = EINVAL; 8398 goto done; 8399 } else 8400 p->u.params.maxrate = 0; 8401 } 8402 if (p->u.params.weight < 0) { 8403 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) { 8404 rc = EINVAL; 8405 goto done; 8406 } else 8407 p->u.params.weight = 0; 8408 } 8409 if (p->u.params.pktsize < 0) { 8410 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 8411 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 8412 rc = EINVAL; 8413 goto done; 8414 } else 8415 p->u.params.pktsize = 0; 8416 } 8417 8418 /* See what the firmware thinks of the request ... */ 8419 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode, 8420 fw_rateunit, fw_ratemode, p->u.params.channel, 8421 p->u.params.cl, p->u.params.minrate, p->u.params.maxrate, 8422 p->u.params.weight, p->u.params.pktsize, 1); 8423 goto done; 8424 } 8425 8426 rc = EINVAL; 8427 done: 8428 end_synchronized_op(sc, 0); 8429 return (rc); 8430 } 8431 8432 static int 8433 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p) 8434 { 8435 struct port_info *pi = NULL; 8436 struct vi_info *vi; 8437 struct sge_txq *txq; 8438 uint32_t fw_mnem, fw_queue, fw_class; 8439 int i, rc; 8440 8441 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq"); 8442 if (rc) 8443 return (rc); 8444 8445 if (!(sc->flags & FULL_INIT_DONE)) { 8446 rc = EAGAIN; 8447 goto done; 8448 } 8449 8450 if (p->port >= sc->params.nports) { 8451 rc = EINVAL; 8452 goto done; 8453 } 8454 8455 /* XXX: Only supported for the main VI. */ 8456 pi = sc->port[p->port]; 8457 vi = &pi->vi[0]; 8458 if (!in_range(p->queue, 0, vi->ntxq - 1) || !in_range(p->cl, 0, 7)) { 8459 rc = EINVAL; 8460 goto done; 8461 } 8462 8463 /* 8464 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX 8465 * Scheduling Class in this case). 8466 */ 8467 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 8468 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH)); 8469 fw_class = p->cl < 0 ? 0xffffffff : p->cl; 8470 8471 /* 8472 * If op.queue is non-negative, then we're only changing the scheduling 8473 * on a single specified TX queue. 8474 */ 8475 if (p->queue >= 0) { 8476 txq = &sc->sge.txq[vi->first_txq + p->queue]; 8477 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 8478 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 8479 &fw_class); 8480 goto done; 8481 } 8482 8483 /* 8484 * Change the scheduling on all the TX queues for the 8485 * interface. 8486 */ 8487 for_each_txq(vi, i, txq) { 8488 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 8489 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 8490 &fw_class); 8491 if (rc) 8492 goto done; 8493 } 8494 8495 rc = 0; 8496 done: 8497 end_synchronized_op(sc, 0); 8498 return (rc); 8499 } 8500 8501 int 8502 t4_os_find_pci_capability(struct adapter *sc, int cap) 8503 { 8504 int i; 8505 8506 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 8507 } 8508 8509 int 8510 t4_os_pci_save_state(struct adapter *sc) 8511 { 8512 device_t dev; 8513 struct pci_devinfo *dinfo; 8514 8515 dev = sc->dev; 8516 dinfo = device_get_ivars(dev); 8517 8518 pci_cfg_save(dev, dinfo, 0); 8519 return (0); 8520 } 8521 8522 int 8523 t4_os_pci_restore_state(struct adapter *sc) 8524 { 8525 device_t dev; 8526 struct pci_devinfo *dinfo; 8527 8528 dev = sc->dev; 8529 dinfo = device_get_ivars(dev); 8530 8531 pci_cfg_restore(dev, dinfo); 8532 return (0); 8533 } 8534 8535 void 8536 t4_os_portmod_changed(const struct adapter *sc, int idx) 8537 { 8538 struct port_info *pi = sc->port[idx]; 8539 struct vi_info *vi; 8540 struct ifnet *ifp; 8541 int v; 8542 static const char *mod_str[] = { 8543 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 8544 }; 8545 8546 for_each_vi(pi, v, vi) { 8547 build_medialist(pi, &vi->media); 8548 } 8549 8550 ifp = pi->vi[0].ifp; 8551 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 8552 if_printf(ifp, "transceiver unplugged.\n"); 8553 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 8554 if_printf(ifp, "unknown transceiver inserted.\n"); 8555 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 8556 if_printf(ifp, "unsupported transceiver inserted.\n"); 8557 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 8558 if_printf(ifp, "%s transceiver inserted.\n", 8559 mod_str[pi->mod_type]); 8560 } else { 8561 if_printf(ifp, "transceiver (type %d) inserted.\n", 8562 pi->mod_type); 8563 } 8564 } 8565 8566 void 8567 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason) 8568 { 8569 struct port_info *pi = sc->port[idx]; 8570 struct vi_info *vi; 8571 struct ifnet *ifp; 8572 int v; 8573 8574 if (link_stat) 8575 pi->linkdnrc = -1; 8576 else { 8577 if (reason >= 0) 8578 pi->linkdnrc = reason; 8579 } 8580 for_each_vi(pi, v, vi) { 8581 ifp = vi->ifp; 8582 if (ifp == NULL) 8583 continue; 8584 8585 if (link_stat) { 8586 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 8587 if_link_state_change(ifp, LINK_STATE_UP); 8588 } else { 8589 if_link_state_change(ifp, LINK_STATE_DOWN); 8590 } 8591 } 8592 } 8593 8594 void 8595 t4_iterate(void (*func)(struct adapter *, void *), void *arg) 8596 { 8597 struct adapter *sc; 8598 8599 sx_slock(&t4_list_lock); 8600 SLIST_FOREACH(sc, &t4_list, link) { 8601 /* 8602 * func should not make any assumptions about what state sc is 8603 * in - the only guarantee is that sc->sc_lock is a valid lock. 8604 */ 8605 func(sc, arg); 8606 } 8607 sx_sunlock(&t4_list_lock); 8608 } 8609 8610 static int 8611 t4_open(struct cdev *dev, int flags, int type, struct thread *td) 8612 { 8613 return (0); 8614 } 8615 8616 static int 8617 t4_close(struct cdev *dev, int flags, int type, struct thread *td) 8618 { 8619 return (0); 8620 } 8621 8622 static int 8623 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 8624 struct thread *td) 8625 { 8626 int rc; 8627 struct adapter *sc = dev->si_drv1; 8628 8629 rc = priv_check(td, PRIV_DRIVER); 8630 if (rc != 0) 8631 return (rc); 8632 8633 switch (cmd) { 8634 case CHELSIO_T4_GETREG: { 8635 struct t4_reg *edata = (struct t4_reg *)data; 8636 8637 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8638 return (EFAULT); 8639 8640 if (edata->size == 4) 8641 edata->val = t4_read_reg(sc, edata->addr); 8642 else if (edata->size == 8) 8643 edata->val = t4_read_reg64(sc, edata->addr); 8644 else 8645 return (EINVAL); 8646 8647 break; 8648 } 8649 case CHELSIO_T4_SETREG: { 8650 struct t4_reg *edata = (struct t4_reg *)data; 8651 8652 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8653 return (EFAULT); 8654 8655 if (edata->size == 4) { 8656 if (edata->val & 0xffffffff00000000) 8657 return (EINVAL); 8658 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 8659 } else if (edata->size == 8) 8660 t4_write_reg64(sc, edata->addr, edata->val); 8661 else 8662 return (EINVAL); 8663 break; 8664 } 8665 case CHELSIO_T4_REGDUMP: { 8666 struct t4_regdump *regs = (struct t4_regdump *)data; 8667 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 8668 uint8_t *buf; 8669 8670 if (regs->len < reglen) { 8671 regs->len = reglen; /* hint to the caller */ 8672 return (ENOBUFS); 8673 } 8674 8675 regs->len = reglen; 8676 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 8677 t4_get_regs(sc, regs, buf); 8678 rc = copyout(buf, regs->data, reglen); 8679 free(buf, M_CXGBE); 8680 break; 8681 } 8682 case CHELSIO_T4_GET_FILTER_MODE: 8683 rc = get_filter_mode(sc, (uint32_t *)data); 8684 break; 8685 case CHELSIO_T4_SET_FILTER_MODE: 8686 rc = set_filter_mode(sc, *(uint32_t *)data); 8687 break; 8688 case CHELSIO_T4_GET_FILTER: 8689 rc = get_filter(sc, (struct t4_filter *)data); 8690 break; 8691 case CHELSIO_T4_SET_FILTER: 8692 rc = set_filter(sc, (struct t4_filter *)data); 8693 break; 8694 case CHELSIO_T4_DEL_FILTER: 8695 rc = del_filter(sc, (struct t4_filter *)data); 8696 break; 8697 case CHELSIO_T4_GET_SGE_CONTEXT: 8698 rc = get_sge_context(sc, (struct t4_sge_context *)data); 8699 break; 8700 case CHELSIO_T4_LOAD_FW: 8701 rc = load_fw(sc, (struct t4_data *)data); 8702 break; 8703 case CHELSIO_T4_GET_MEM: 8704 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 8705 break; 8706 case CHELSIO_T4_GET_I2C: 8707 rc = read_i2c(sc, (struct t4_i2c_data *)data); 8708 break; 8709 case CHELSIO_T4_CLEAR_STATS: { 8710 int i, v; 8711 u_int port_id = *(uint32_t *)data; 8712 struct port_info *pi; 8713 struct vi_info *vi; 8714 8715 if (port_id >= sc->params.nports) 8716 return (EINVAL); 8717 pi = sc->port[port_id]; 8718 8719 /* MAC stats */ 8720 t4_clr_port_stats(sc, pi->tx_chan); 8721 pi->tx_parse_error = 0; 8722 mtx_lock(&sc->regwin_lock); 8723 for_each_vi(pi, v, vi) { 8724 if (vi->flags & VI_INIT_DONE) 8725 t4_clr_vi_stats(sc, vi->viid); 8726 } 8727 mtx_unlock(&sc->regwin_lock); 8728 8729 /* 8730 * Since this command accepts a port, clear stats for 8731 * all VIs on this port. 8732 */ 8733 for_each_vi(pi, v, vi) { 8734 if (vi->flags & VI_INIT_DONE) { 8735 struct sge_rxq *rxq; 8736 struct sge_txq *txq; 8737 struct sge_wrq *wrq; 8738 8739 if (vi->flags & VI_NETMAP) 8740 continue; 8741 8742 for_each_rxq(vi, i, rxq) { 8743 #if defined(INET) || defined(INET6) 8744 rxq->lro.lro_queued = 0; 8745 rxq->lro.lro_flushed = 0; 8746 #endif 8747 rxq->rxcsum = 0; 8748 rxq->vlan_extraction = 0; 8749 } 8750 8751 for_each_txq(vi, i, txq) { 8752 txq->txcsum = 0; 8753 txq->tso_wrs = 0; 8754 txq->vlan_insertion = 0; 8755 txq->imm_wrs = 0; 8756 txq->sgl_wrs = 0; 8757 txq->txpkt_wrs = 0; 8758 txq->txpkts0_wrs = 0; 8759 txq->txpkts1_wrs = 0; 8760 txq->txpkts0_pkts = 0; 8761 txq->txpkts1_pkts = 0; 8762 mp_ring_reset_stats(txq->r); 8763 } 8764 8765 #ifdef TCP_OFFLOAD 8766 /* nothing to clear for each ofld_rxq */ 8767 8768 for_each_ofld_txq(vi, i, wrq) { 8769 wrq->tx_wrs_direct = 0; 8770 wrq->tx_wrs_copied = 0; 8771 } 8772 #endif 8773 8774 if (IS_MAIN_VI(vi)) { 8775 wrq = &sc->sge.ctrlq[pi->port_id]; 8776 wrq->tx_wrs_direct = 0; 8777 wrq->tx_wrs_copied = 0; 8778 } 8779 } 8780 } 8781 break; 8782 } 8783 case CHELSIO_T4_SCHED_CLASS: 8784 rc = set_sched_class(sc, (struct t4_sched_params *)data); 8785 break; 8786 case CHELSIO_T4_SCHED_QUEUE: 8787 rc = set_sched_queue(sc, (struct t4_sched_queue *)data); 8788 break; 8789 case CHELSIO_T4_GET_TRACER: 8790 rc = t4_get_tracer(sc, (struct t4_tracer *)data); 8791 break; 8792 case CHELSIO_T4_SET_TRACER: 8793 rc = t4_set_tracer(sc, (struct t4_tracer *)data); 8794 break; 8795 default: 8796 rc = EINVAL; 8797 } 8798 8799 return (rc); 8800 } 8801 8802 #ifdef TCP_OFFLOAD 8803 void 8804 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask, 8805 const unsigned int *pgsz_order) 8806 { 8807 struct vi_info *vi = ifp->if_softc; 8808 struct adapter *sc = vi->pi->adapter; 8809 8810 t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask); 8811 t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) | 8812 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) | 8813 V_HPZ3(pgsz_order[3])); 8814 } 8815 8816 static int 8817 toe_capability(struct vi_info *vi, int enable) 8818 { 8819 int rc; 8820 struct port_info *pi = vi->pi; 8821 struct adapter *sc = pi->adapter; 8822 8823 ASSERT_SYNCHRONIZED_OP(sc); 8824 8825 if (!is_offload(sc)) 8826 return (ENODEV); 8827 8828 if (enable) { 8829 if ((vi->ifp->if_capenable & IFCAP_TOE) != 0) { 8830 /* TOE is already enabled. */ 8831 return (0); 8832 } 8833 8834 /* 8835 * We need the port's queues around so that we're able to send 8836 * and receive CPLs to/from the TOE even if the ifnet for this 8837 * port has never been UP'd administratively. 8838 */ 8839 if (!(vi->flags & VI_INIT_DONE)) { 8840 rc = cxgbe_init_synchronized(vi); 8841 if (rc) 8842 return (rc); 8843 } 8844 if (!(pi->vi[0].flags & VI_INIT_DONE)) { 8845 rc = cxgbe_init_synchronized(&pi->vi[0]); 8846 if (rc) 8847 return (rc); 8848 } 8849 8850 if (isset(&sc->offload_map, pi->port_id)) { 8851 /* TOE is enabled on another VI of this port. */ 8852 pi->uld_vis++; 8853 return (0); 8854 } 8855 8856 if (!uld_active(sc, ULD_TOM)) { 8857 rc = t4_activate_uld(sc, ULD_TOM); 8858 if (rc == EAGAIN) { 8859 log(LOG_WARNING, 8860 "You must kldload t4_tom.ko before trying " 8861 "to enable TOE on a cxgbe interface.\n"); 8862 } 8863 if (rc != 0) 8864 return (rc); 8865 KASSERT(sc->tom_softc != NULL, 8866 ("%s: TOM activated but softc NULL", __func__)); 8867 KASSERT(uld_active(sc, ULD_TOM), 8868 ("%s: TOM activated but flag not set", __func__)); 8869 } 8870 8871 /* Activate iWARP and iSCSI too, if the modules are loaded. */ 8872 if (!uld_active(sc, ULD_IWARP)) 8873 (void) t4_activate_uld(sc, ULD_IWARP); 8874 if (!uld_active(sc, ULD_ISCSI)) 8875 (void) t4_activate_uld(sc, ULD_ISCSI); 8876 8877 pi->uld_vis++; 8878 setbit(&sc->offload_map, pi->port_id); 8879 } else { 8880 pi->uld_vis--; 8881 8882 if (!isset(&sc->offload_map, pi->port_id) || pi->uld_vis > 0) 8883 return (0); 8884 8885 KASSERT(uld_active(sc, ULD_TOM), 8886 ("%s: TOM never initialized?", __func__)); 8887 clrbit(&sc->offload_map, pi->port_id); 8888 } 8889 8890 return (0); 8891 } 8892 8893 /* 8894 * Add an upper layer driver to the global list. 8895 */ 8896 int 8897 t4_register_uld(struct uld_info *ui) 8898 { 8899 int rc = 0; 8900 struct uld_info *u; 8901 8902 sx_xlock(&t4_uld_list_lock); 8903 SLIST_FOREACH(u, &t4_uld_list, link) { 8904 if (u->uld_id == ui->uld_id) { 8905 rc = EEXIST; 8906 goto done; 8907 } 8908 } 8909 8910 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 8911 ui->refcount = 0; 8912 done: 8913 sx_xunlock(&t4_uld_list_lock); 8914 return (rc); 8915 } 8916 8917 int 8918 t4_unregister_uld(struct uld_info *ui) 8919 { 8920 int rc = EINVAL; 8921 struct uld_info *u; 8922 8923 sx_xlock(&t4_uld_list_lock); 8924 8925 SLIST_FOREACH(u, &t4_uld_list, link) { 8926 if (u == ui) { 8927 if (ui->refcount > 0) { 8928 rc = EBUSY; 8929 goto done; 8930 } 8931 8932 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 8933 rc = 0; 8934 goto done; 8935 } 8936 } 8937 done: 8938 sx_xunlock(&t4_uld_list_lock); 8939 return (rc); 8940 } 8941 8942 int 8943 t4_activate_uld(struct adapter *sc, int id) 8944 { 8945 int rc; 8946 struct uld_info *ui; 8947 8948 ASSERT_SYNCHRONIZED_OP(sc); 8949 8950 if (id < 0 || id > ULD_MAX) 8951 return (EINVAL); 8952 rc = EAGAIN; /* kldoad the module with this ULD and try again. */ 8953 8954 sx_slock(&t4_uld_list_lock); 8955 8956 SLIST_FOREACH(ui, &t4_uld_list, link) { 8957 if (ui->uld_id == id) { 8958 if (!(sc->flags & FULL_INIT_DONE)) { 8959 rc = adapter_full_init(sc); 8960 if (rc != 0) 8961 break; 8962 } 8963 8964 rc = ui->activate(sc); 8965 if (rc == 0) { 8966 setbit(&sc->active_ulds, id); 8967 ui->refcount++; 8968 } 8969 break; 8970 } 8971 } 8972 8973 sx_sunlock(&t4_uld_list_lock); 8974 8975 return (rc); 8976 } 8977 8978 int 8979 t4_deactivate_uld(struct adapter *sc, int id) 8980 { 8981 int rc; 8982 struct uld_info *ui; 8983 8984 ASSERT_SYNCHRONIZED_OP(sc); 8985 8986 if (id < 0 || id > ULD_MAX) 8987 return (EINVAL); 8988 rc = ENXIO; 8989 8990 sx_slock(&t4_uld_list_lock); 8991 8992 SLIST_FOREACH(ui, &t4_uld_list, link) { 8993 if (ui->uld_id == id) { 8994 rc = ui->deactivate(sc); 8995 if (rc == 0) { 8996 clrbit(&sc->active_ulds, id); 8997 ui->refcount--; 8998 } 8999 break; 9000 } 9001 } 9002 9003 sx_sunlock(&t4_uld_list_lock); 9004 9005 return (rc); 9006 } 9007 9008 int 9009 uld_active(struct adapter *sc, int uld_id) 9010 { 9011 9012 MPASS(uld_id >= 0 && uld_id <= ULD_MAX); 9013 9014 return (isset(&sc->active_ulds, uld_id)); 9015 } 9016 #endif 9017 9018 /* 9019 * Come up with reasonable defaults for some of the tunables, provided they're 9020 * not set by the user (in which case we'll use the values as is). 9021 */ 9022 static void 9023 tweak_tunables(void) 9024 { 9025 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 9026 9027 if (t4_ntxq10g < 1) { 9028 #ifdef RSS 9029 t4_ntxq10g = rss_getnumbuckets(); 9030 #else 9031 t4_ntxq10g = min(nc, NTXQ_10G); 9032 #endif 9033 } 9034 9035 if (t4_ntxq1g < 1) { 9036 #ifdef RSS 9037 /* XXX: way too many for 1GbE? */ 9038 t4_ntxq1g = rss_getnumbuckets(); 9039 #else 9040 t4_ntxq1g = min(nc, NTXQ_1G); 9041 #endif 9042 } 9043 9044 if (t4_nrxq10g < 1) { 9045 #ifdef RSS 9046 t4_nrxq10g = rss_getnumbuckets(); 9047 #else 9048 t4_nrxq10g = min(nc, NRXQ_10G); 9049 #endif 9050 } 9051 9052 if (t4_nrxq1g < 1) { 9053 #ifdef RSS 9054 /* XXX: way too many for 1GbE? */ 9055 t4_nrxq1g = rss_getnumbuckets(); 9056 #else 9057 t4_nrxq1g = min(nc, NRXQ_1G); 9058 #endif 9059 } 9060 9061 #ifdef TCP_OFFLOAD 9062 if (t4_nofldtxq10g < 1) 9063 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 9064 9065 if (t4_nofldtxq1g < 1) 9066 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 9067 9068 if (t4_nofldrxq10g < 1) 9069 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 9070 9071 if (t4_nofldrxq1g < 1) 9072 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 9073 9074 if (t4_toecaps_allowed == -1) 9075 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 9076 #else 9077 if (t4_toecaps_allowed == -1) 9078 t4_toecaps_allowed = 0; 9079 #endif 9080 9081 #ifdef DEV_NETMAP 9082 if (t4_nnmtxq10g < 1) 9083 t4_nnmtxq10g = min(nc, NNMTXQ_10G); 9084 9085 if (t4_nnmtxq1g < 1) 9086 t4_nnmtxq1g = min(nc, NNMTXQ_1G); 9087 9088 if (t4_nnmrxq10g < 1) 9089 t4_nnmrxq10g = min(nc, NNMRXQ_10G); 9090 9091 if (t4_nnmrxq1g < 1) 9092 t4_nnmrxq1g = min(nc, NNMRXQ_1G); 9093 #endif 9094 9095 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 9096 t4_tmr_idx_10g = TMR_IDX_10G; 9097 9098 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 9099 t4_pktc_idx_10g = PKTC_IDX_10G; 9100 9101 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 9102 t4_tmr_idx_1g = TMR_IDX_1G; 9103 9104 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 9105 t4_pktc_idx_1g = PKTC_IDX_1G; 9106 9107 if (t4_qsize_txq < 128) 9108 t4_qsize_txq = 128; 9109 9110 if (t4_qsize_rxq < 128) 9111 t4_qsize_rxq = 128; 9112 while (t4_qsize_rxq & 7) 9113 t4_qsize_rxq++; 9114 9115 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 9116 } 9117 9118 static struct sx mlu; /* mod load unload */ 9119 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload"); 9120 9121 static int 9122 mod_event(module_t mod, int cmd, void *arg) 9123 { 9124 int rc = 0; 9125 static int loaded = 0; 9126 9127 switch (cmd) { 9128 case MOD_LOAD: 9129 sx_xlock(&mlu); 9130 if (loaded++ == 0) { 9131 t4_sge_modload(); 9132 sx_init(&t4_list_lock, "T4/T5 adapters"); 9133 SLIST_INIT(&t4_list); 9134 #ifdef TCP_OFFLOAD 9135 sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); 9136 SLIST_INIT(&t4_uld_list); 9137 #endif 9138 t4_tracer_modload(); 9139 tweak_tunables(); 9140 } 9141 sx_xunlock(&mlu); 9142 break; 9143 9144 case MOD_UNLOAD: 9145 sx_xlock(&mlu); 9146 if (--loaded == 0) { 9147 int tries; 9148 9149 sx_slock(&t4_list_lock); 9150 if (!SLIST_EMPTY(&t4_list)) { 9151 rc = EBUSY; 9152 sx_sunlock(&t4_list_lock); 9153 goto done_unload; 9154 } 9155 #ifdef TCP_OFFLOAD 9156 sx_slock(&t4_uld_list_lock); 9157 if (!SLIST_EMPTY(&t4_uld_list)) { 9158 rc = EBUSY; 9159 sx_sunlock(&t4_uld_list_lock); 9160 sx_sunlock(&t4_list_lock); 9161 goto done_unload; 9162 } 9163 #endif 9164 tries = 0; 9165 while (tries++ < 5 && t4_sge_extfree_refs() != 0) { 9166 uprintf("%ju clusters with custom free routine " 9167 "still is use.\n", t4_sge_extfree_refs()); 9168 pause("t4unload", 2 * hz); 9169 } 9170 #ifdef TCP_OFFLOAD 9171 sx_sunlock(&t4_uld_list_lock); 9172 #endif 9173 sx_sunlock(&t4_list_lock); 9174 9175 if (t4_sge_extfree_refs() == 0) { 9176 t4_tracer_modunload(); 9177 #ifdef TCP_OFFLOAD 9178 sx_destroy(&t4_uld_list_lock); 9179 #endif 9180 sx_destroy(&t4_list_lock); 9181 t4_sge_modunload(); 9182 loaded = 0; 9183 } else { 9184 rc = EBUSY; 9185 loaded++; /* undo earlier decrement */ 9186 } 9187 } 9188 done_unload: 9189 sx_xunlock(&mlu); 9190 break; 9191 } 9192 9193 return (rc); 9194 } 9195 9196 static devclass_t t4_devclass, t5_devclass; 9197 static devclass_t cxgbe_devclass, cxl_devclass; 9198 static devclass_t vcxgbe_devclass, vcxl_devclass; 9199 9200 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 9201 MODULE_VERSION(t4nex, 1); 9202 MODULE_DEPEND(t4nex, firmware, 1, 1, 1); 9203 #ifdef DEV_NETMAP 9204 MODULE_DEPEND(t4nex, netmap, 1, 1, 1); 9205 #endif /* DEV_NETMAP */ 9206 9207 9208 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 9209 MODULE_VERSION(t5nex, 1); 9210 MODULE_DEPEND(t5nex, firmware, 1, 1, 1); 9211 #ifdef DEV_NETMAP 9212 MODULE_DEPEND(t5nex, netmap, 1, 1, 1); 9213 #endif /* DEV_NETMAP */ 9214 9215 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 9216 MODULE_VERSION(cxgbe, 1); 9217 9218 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 9219 MODULE_VERSION(cxl, 1); 9220 9221 DRIVER_MODULE(vcxgbe, cxgbe, vcxgbe_driver, vcxgbe_devclass, 0, 0); 9222 MODULE_VERSION(vcxgbe, 1); 9223 9224 DRIVER_MODULE(vcxl, cxl, vcxl_driver, vcxl_devclass, 0, 0); 9225 MODULE_VERSION(vcxl, 1); 9226