1 /*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_inet.h" 32 #include "opt_inet6.h" 33 #include "opt_rss.h" 34 35 #include <sys/param.h> 36 #include <sys/conf.h> 37 #include <sys/priv.h> 38 #include <sys/kernel.h> 39 #include <sys/bus.h> 40 #include <sys/module.h> 41 #include <sys/malloc.h> 42 #include <sys/queue.h> 43 #include <sys/taskqueue.h> 44 #include <sys/pciio.h> 45 #include <dev/pci/pcireg.h> 46 #include <dev/pci/pcivar.h> 47 #include <dev/pci/pci_private.h> 48 #include <sys/firmware.h> 49 #include <sys/sbuf.h> 50 #include <sys/smp.h> 51 #include <sys/socket.h> 52 #include <sys/sockio.h> 53 #include <sys/sysctl.h> 54 #include <net/ethernet.h> 55 #include <net/if.h> 56 #include <net/if_types.h> 57 #include <net/if_dl.h> 58 #include <net/if_vlan_var.h> 59 #ifdef RSS 60 #include <net/rss_config.h> 61 #endif 62 #if defined(__i386__) || defined(__amd64__) 63 #include <vm/vm.h> 64 #include <vm/pmap.h> 65 #endif 66 67 #include "common/common.h" 68 #include "common/t4_msg.h" 69 #include "common/t4_regs.h" 70 #include "common/t4_regs_values.h" 71 #include "t4_ioctl.h" 72 #include "t4_l2t.h" 73 #include "t4_mp_ring.h" 74 75 /* T4 bus driver interface */ 76 static int t4_probe(device_t); 77 static int t4_attach(device_t); 78 static int t4_detach(device_t); 79 static device_method_t t4_methods[] = { 80 DEVMETHOD(device_probe, t4_probe), 81 DEVMETHOD(device_attach, t4_attach), 82 DEVMETHOD(device_detach, t4_detach), 83 84 DEVMETHOD_END 85 }; 86 static driver_t t4_driver = { 87 "t4nex", 88 t4_methods, 89 sizeof(struct adapter) 90 }; 91 92 93 /* T4 port (cxgbe) interface */ 94 static int cxgbe_probe(device_t); 95 static int cxgbe_attach(device_t); 96 static int cxgbe_detach(device_t); 97 static device_method_t cxgbe_methods[] = { 98 DEVMETHOD(device_probe, cxgbe_probe), 99 DEVMETHOD(device_attach, cxgbe_attach), 100 DEVMETHOD(device_detach, cxgbe_detach), 101 { 0, 0 } 102 }; 103 static driver_t cxgbe_driver = { 104 "cxgbe", 105 cxgbe_methods, 106 sizeof(struct port_info) 107 }; 108 109 /* T4 VI (vcxgbe) interface */ 110 static int vcxgbe_probe(device_t); 111 static int vcxgbe_attach(device_t); 112 static int vcxgbe_detach(device_t); 113 static device_method_t vcxgbe_methods[] = { 114 DEVMETHOD(device_probe, vcxgbe_probe), 115 DEVMETHOD(device_attach, vcxgbe_attach), 116 DEVMETHOD(device_detach, vcxgbe_detach), 117 { 0, 0 } 118 }; 119 static driver_t vcxgbe_driver = { 120 "vcxgbe", 121 vcxgbe_methods, 122 sizeof(struct vi_info) 123 }; 124 125 static d_ioctl_t t4_ioctl; 126 static d_open_t t4_open; 127 static d_close_t t4_close; 128 129 static struct cdevsw t4_cdevsw = { 130 .d_version = D_VERSION, 131 .d_flags = 0, 132 .d_open = t4_open, 133 .d_close = t4_close, 134 .d_ioctl = t4_ioctl, 135 .d_name = "t4nex", 136 }; 137 138 /* T5 bus driver interface */ 139 static int t5_probe(device_t); 140 static device_method_t t5_methods[] = { 141 DEVMETHOD(device_probe, t5_probe), 142 DEVMETHOD(device_attach, t4_attach), 143 DEVMETHOD(device_detach, t4_detach), 144 145 DEVMETHOD_END 146 }; 147 static driver_t t5_driver = { 148 "t5nex", 149 t5_methods, 150 sizeof(struct adapter) 151 }; 152 153 154 /* T5 port (cxl) interface */ 155 static driver_t cxl_driver = { 156 "cxl", 157 cxgbe_methods, 158 sizeof(struct port_info) 159 }; 160 161 /* T5 VI (vcxl) interface */ 162 static driver_t vcxl_driver = { 163 "vcxl", 164 vcxgbe_methods, 165 sizeof(struct vi_info) 166 }; 167 168 static struct cdevsw t5_cdevsw = { 169 .d_version = D_VERSION, 170 .d_flags = 0, 171 .d_open = t4_open, 172 .d_close = t4_close, 173 .d_ioctl = t4_ioctl, 174 .d_name = "t5nex", 175 }; 176 177 /* ifnet + media interface */ 178 static void cxgbe_init(void *); 179 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 180 static int cxgbe_transmit(struct ifnet *, struct mbuf *); 181 static void cxgbe_qflush(struct ifnet *); 182 static int cxgbe_media_change(struct ifnet *); 183 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 184 185 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 186 187 /* 188 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 189 * then ADAPTER_LOCK, then t4_uld_list_lock. 190 */ 191 static struct sx t4_list_lock; 192 SLIST_HEAD(, adapter) t4_list; 193 #ifdef TCP_OFFLOAD 194 static struct sx t4_uld_list_lock; 195 SLIST_HEAD(, uld_info) t4_uld_list; 196 #endif 197 198 /* 199 * Tunables. See tweak_tunables() too. 200 * 201 * Each tunable is set to a default value here if it's known at compile-time. 202 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 203 * provide a reasonable default when the driver is loaded. 204 * 205 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 206 * T5 are under hw.cxl. 207 */ 208 209 /* 210 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 211 */ 212 #define NTXQ_10G 16 213 static int t4_ntxq10g = -1; 214 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 215 216 #define NRXQ_10G 8 217 static int t4_nrxq10g = -1; 218 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 219 220 #define NTXQ_1G 4 221 static int t4_ntxq1g = -1; 222 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 223 224 #define NRXQ_1G 2 225 static int t4_nrxq1g = -1; 226 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 227 228 static int t4_rsrv_noflowq = 0; 229 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq); 230 231 #ifdef TCP_OFFLOAD 232 #define NOFLDTXQ_10G 8 233 static int t4_nofldtxq10g = -1; 234 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 235 236 #define NOFLDRXQ_10G 2 237 static int t4_nofldrxq10g = -1; 238 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 239 240 #define NOFLDTXQ_1G 2 241 static int t4_nofldtxq1g = -1; 242 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 243 244 #define NOFLDRXQ_1G 1 245 static int t4_nofldrxq1g = -1; 246 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 247 #endif 248 249 #ifdef DEV_NETMAP 250 #define NNMTXQ_10G 2 251 static int t4_nnmtxq10g = -1; 252 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g); 253 254 #define NNMRXQ_10G 2 255 static int t4_nnmrxq10g = -1; 256 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g); 257 258 #define NNMTXQ_1G 1 259 static int t4_nnmtxq1g = -1; 260 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g); 261 262 #define NNMRXQ_1G 1 263 static int t4_nnmrxq1g = -1; 264 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g); 265 #endif 266 267 /* 268 * Holdoff parameters for 10G and 1G ports. 269 */ 270 #define TMR_IDX_10G 1 271 static int t4_tmr_idx_10g = TMR_IDX_10G; 272 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 273 274 #define PKTC_IDX_10G (-1) 275 static int t4_pktc_idx_10g = PKTC_IDX_10G; 276 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 277 278 #define TMR_IDX_1G 1 279 static int t4_tmr_idx_1g = TMR_IDX_1G; 280 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 281 282 #define PKTC_IDX_1G (-1) 283 static int t4_pktc_idx_1g = PKTC_IDX_1G; 284 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 285 286 /* 287 * Size (# of entries) of each tx and rx queue. 288 */ 289 static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 290 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 291 292 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 293 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 294 295 /* 296 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 297 */ 298 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 299 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 300 301 /* 302 * Configuration file. 303 */ 304 #define DEFAULT_CF "default" 305 #define FLASH_CF "flash" 306 #define UWIRE_CF "uwire" 307 #define FPGA_CF "fpga" 308 static char t4_cfg_file[32] = DEFAULT_CF; 309 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 310 311 /* 312 * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively). 313 * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them. 314 * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water 315 * mark or when signalled to do so, 0 to never emit PAUSE. 316 */ 317 static int t4_pause_settings = PAUSE_TX | PAUSE_RX; 318 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings); 319 320 /* 321 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 322 * encouraged respectively). 323 */ 324 static unsigned int t4_fw_install = 1; 325 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 326 327 /* 328 * ASIC features that will be used. Disable the ones you don't want so that the 329 * chip resources aren't wasted on features that will not be used. 330 */ 331 static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 332 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 333 334 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 335 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 336 337 static int t4_toecaps_allowed = -1; 338 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 339 340 static int t4_rdmacaps_allowed = 0; 341 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 342 343 static int t4_iscsicaps_allowed = 0; 344 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 345 346 static int t4_fcoecaps_allowed = 0; 347 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 348 349 static int t5_write_combine = 0; 350 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 351 352 static int t4_num_vis = 1; 353 TUNABLE_INT("hw.cxgbe.num_vis", &t4_num_vis); 354 355 /* Functions used by extra VIs to obtain unique MAC addresses for each VI. */ 356 static int vi_mac_funcs[] = { 357 FW_VI_FUNC_OFLD, 358 FW_VI_FUNC_IWARP, 359 FW_VI_FUNC_OPENISCSI, 360 FW_VI_FUNC_OPENFCOE, 361 FW_VI_FUNC_FOISCSI, 362 FW_VI_FUNC_FOFCOE, 363 }; 364 365 struct intrs_and_queues { 366 uint16_t intr_type; /* INTx, MSI, or MSI-X */ 367 uint16_t nirq; /* Total # of vectors */ 368 uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */ 369 uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */ 370 uint16_t ntxq10g; /* # of NIC txq's for each 10G port */ 371 uint16_t nrxq10g; /* # of NIC rxq's for each 10G port */ 372 uint16_t ntxq1g; /* # of NIC txq's for each 1G port */ 373 uint16_t nrxq1g; /* # of NIC rxq's for each 1G port */ 374 uint16_t rsrv_noflowq; /* Flag whether to reserve queue 0 */ 375 #ifdef TCP_OFFLOAD 376 uint16_t nofldtxq10g; /* # of TOE txq's for each 10G port */ 377 uint16_t nofldrxq10g; /* # of TOE rxq's for each 10G port */ 378 uint16_t nofldtxq1g; /* # of TOE txq's for each 1G port */ 379 uint16_t nofldrxq1g; /* # of TOE rxq's for each 1G port */ 380 #endif 381 #ifdef DEV_NETMAP 382 uint16_t nnmtxq10g; /* # of netmap txq's for each 10G port */ 383 uint16_t nnmrxq10g; /* # of netmap rxq's for each 10G port */ 384 uint16_t nnmtxq1g; /* # of netmap txq's for each 1G port */ 385 uint16_t nnmrxq1g; /* # of netmap rxq's for each 1G port */ 386 #endif 387 }; 388 389 struct filter_entry { 390 uint32_t valid:1; /* filter allocated and valid */ 391 uint32_t locked:1; /* filter is administratively locked */ 392 uint32_t pending:1; /* filter action is pending firmware reply */ 393 uint32_t smtidx:8; /* Source MAC Table index for smac */ 394 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 395 396 struct t4_filter_specification fs; 397 }; 398 399 static int map_bars_0_and_4(struct adapter *); 400 static int map_bar_2(struct adapter *); 401 static void setup_memwin(struct adapter *); 402 static int validate_mem_range(struct adapter *, uint32_t, int); 403 static int fwmtype_to_hwmtype(int); 404 static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 405 uint32_t *); 406 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 407 static uint32_t position_memwin(struct adapter *, int, uint32_t); 408 static int cfg_itype_and_nqueues(struct adapter *, int, int, int, 409 struct intrs_and_queues *); 410 static int prep_firmware(struct adapter *); 411 static int partition_resources(struct adapter *, const struct firmware *, 412 const char *); 413 static int get_params__pre_init(struct adapter *); 414 static int get_params__post_init(struct adapter *); 415 static int set_params__post_init(struct adapter *); 416 static void t4_set_desc(struct adapter *); 417 static void build_medialist(struct port_info *, struct ifmedia *); 418 static int cxgbe_init_synchronized(struct vi_info *); 419 static int cxgbe_uninit_synchronized(struct vi_info *); 420 static int setup_intr_handlers(struct adapter *); 421 static void quiesce_txq(struct adapter *, struct sge_txq *); 422 static void quiesce_wrq(struct adapter *, struct sge_wrq *); 423 static void quiesce_iq(struct adapter *, struct sge_iq *); 424 static void quiesce_fl(struct adapter *, struct sge_fl *); 425 static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 426 driver_intr_t *, void *, char *); 427 static int t4_free_irq(struct adapter *, struct irq *); 428 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 429 unsigned int); 430 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 431 static void vi_refresh_stats(struct adapter *, struct vi_info *); 432 static void cxgbe_refresh_stats(struct adapter *, struct port_info *); 433 static void cxgbe_tick(void *); 434 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 435 static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 436 struct mbuf *); 437 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 438 static int fw_msg_not_handled(struct adapter *, const __be64 *); 439 static void t4_sysctls(struct adapter *); 440 static void cxgbe_sysctls(struct port_info *); 441 static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 442 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 443 static int sysctl_btphy(SYSCTL_HANDLER_ARGS); 444 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); 445 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 446 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 447 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 448 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 449 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS); 450 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 451 static int sysctl_temperature(SYSCTL_HANDLER_ARGS); 452 #ifdef SBUF_DRAIN 453 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 454 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 455 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 456 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); 457 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); 458 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 459 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 460 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 461 static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 462 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 463 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 464 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 465 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); 466 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 467 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); 468 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 469 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 470 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 471 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 472 static int sysctl_tids(SYSCTL_HANDLER_ARGS); 473 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 474 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 475 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 476 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 477 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 478 #endif 479 static uint32_t fconf_to_mode(uint32_t); 480 static uint32_t mode_to_fconf(uint32_t); 481 static uint32_t fspec_to_fconf(struct t4_filter_specification *); 482 static int get_filter_mode(struct adapter *, uint32_t *); 483 static int set_filter_mode(struct adapter *, uint32_t); 484 static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 485 static int get_filter(struct adapter *, struct t4_filter *); 486 static int set_filter(struct adapter *, struct t4_filter *); 487 static int del_filter(struct adapter *, struct t4_filter *); 488 static void clear_filter(struct filter_entry *); 489 static int set_filter_wr(struct adapter *, int); 490 static int del_filter_wr(struct adapter *, int); 491 static int get_sge_context(struct adapter *, struct t4_sge_context *); 492 static int load_fw(struct adapter *, struct t4_data *); 493 static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 494 static int read_i2c(struct adapter *, struct t4_i2c_data *); 495 static int set_sched_class(struct adapter *, struct t4_sched_params *); 496 static int set_sched_queue(struct adapter *, struct t4_sched_queue *); 497 #ifdef TCP_OFFLOAD 498 static int toe_capability(struct vi_info *, int); 499 #endif 500 static int mod_event(module_t, int, void *); 501 502 struct { 503 uint16_t device; 504 char *desc; 505 } t4_pciids[] = { 506 {0xa000, "Chelsio Terminator 4 FPGA"}, 507 {0x4400, "Chelsio T440-dbg"}, 508 {0x4401, "Chelsio T420-CR"}, 509 {0x4402, "Chelsio T422-CR"}, 510 {0x4403, "Chelsio T440-CR"}, 511 {0x4404, "Chelsio T420-BCH"}, 512 {0x4405, "Chelsio T440-BCH"}, 513 {0x4406, "Chelsio T440-CH"}, 514 {0x4407, "Chelsio T420-SO"}, 515 {0x4408, "Chelsio T420-CX"}, 516 {0x4409, "Chelsio T420-BT"}, 517 {0x440a, "Chelsio T404-BT"}, 518 {0x440e, "Chelsio T440-LP-CR"}, 519 }, t5_pciids[] = { 520 {0xb000, "Chelsio Terminator 5 FPGA"}, 521 {0x5400, "Chelsio T580-dbg"}, 522 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ 523 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ 524 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ 525 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ 526 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ 527 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ 528 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ 529 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ 530 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 531 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ 532 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ 533 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ 534 {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */ 535 #ifdef notyet 536 {0x5404, "Chelsio T520-BCH"}, 537 {0x5405, "Chelsio T540-BCH"}, 538 {0x5406, "Chelsio T540-CH"}, 539 {0x5408, "Chelsio T520-CX"}, 540 {0x540b, "Chelsio B520-SR"}, 541 {0x540c, "Chelsio B504-BT"}, 542 {0x540f, "Chelsio Amsterdam"}, 543 {0x5413, "Chelsio T580-CHR"}, 544 #endif 545 }; 546 547 #ifdef TCP_OFFLOAD 548 /* 549 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 550 * exactly the same for both rxq and ofld_rxq. 551 */ 552 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 553 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 554 #endif 555 556 /* No easy way to include t4_msg.h before adapter.h so we check this way */ 557 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 558 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 559 560 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); 561 562 static int 563 t4_probe(device_t dev) 564 { 565 int i; 566 uint16_t v = pci_get_vendor(dev); 567 uint16_t d = pci_get_device(dev); 568 uint8_t f = pci_get_function(dev); 569 570 if (v != PCI_VENDOR_ID_CHELSIO) 571 return (ENXIO); 572 573 /* Attach only to PF0 of the FPGA */ 574 if (d == 0xa000 && f != 0) 575 return (ENXIO); 576 577 for (i = 0; i < nitems(t4_pciids); i++) { 578 if (d == t4_pciids[i].device) { 579 device_set_desc(dev, t4_pciids[i].desc); 580 return (BUS_PROBE_DEFAULT); 581 } 582 } 583 584 return (ENXIO); 585 } 586 587 static int 588 t5_probe(device_t dev) 589 { 590 int i; 591 uint16_t v = pci_get_vendor(dev); 592 uint16_t d = pci_get_device(dev); 593 uint8_t f = pci_get_function(dev); 594 595 if (v != PCI_VENDOR_ID_CHELSIO) 596 return (ENXIO); 597 598 /* Attach only to PF0 of the FPGA */ 599 if (d == 0xb000 && f != 0) 600 return (ENXIO); 601 602 for (i = 0; i < nitems(t5_pciids); i++) { 603 if (d == t5_pciids[i].device) { 604 device_set_desc(dev, t5_pciids[i].desc); 605 return (BUS_PROBE_DEFAULT); 606 } 607 } 608 609 return (ENXIO); 610 } 611 612 static void 613 t5_attribute_workaround(device_t dev) 614 { 615 device_t root_port; 616 uint32_t v; 617 618 /* 619 * The T5 chips do not properly echo the No Snoop and Relaxed 620 * Ordering attributes when replying to a TLP from a Root 621 * Port. As a workaround, find the parent Root Port and 622 * disable No Snoop and Relaxed Ordering. Note that this 623 * affects all devices under this root port. 624 */ 625 root_port = pci_find_pcie_root_port(dev); 626 if (root_port == NULL) { 627 device_printf(dev, "Unable to find parent root port\n"); 628 return; 629 } 630 631 v = pcie_adjust_config(root_port, PCIER_DEVICE_CTL, 632 PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE, 0, 2); 633 if ((v & (PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE)) != 634 0) 635 device_printf(dev, "Disabled No Snoop/Relaxed Ordering on %s\n", 636 device_get_nameunit(root_port)); 637 } 638 639 static int 640 t4_attach(device_t dev) 641 { 642 struct adapter *sc; 643 int rc = 0, i, j, n10g, n1g, rqidx, tqidx; 644 struct intrs_and_queues iaq; 645 struct sge *s; 646 #ifdef TCP_OFFLOAD 647 int ofld_rqidx, ofld_tqidx; 648 #endif 649 #ifdef DEV_NETMAP 650 int nm_rqidx, nm_tqidx; 651 #endif 652 int num_vis; 653 654 sc = device_get_softc(dev); 655 sc->dev = dev; 656 TUNABLE_INT_FETCH("hw.cxgbe.debug_flags", &sc->debug_flags); 657 658 if ((pci_get_device(dev) & 0xff00) == 0x5400) 659 t5_attribute_workaround(dev); 660 pci_enable_busmaster(dev); 661 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 662 uint32_t v; 663 664 pci_set_max_read_req(dev, 4096); 665 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 666 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 667 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 668 669 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5); 670 } 671 672 sc->traceq = -1; 673 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); 674 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", 675 device_get_nameunit(dev)); 676 677 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 678 device_get_nameunit(dev)); 679 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 680 sx_xlock(&t4_list_lock); 681 SLIST_INSERT_HEAD(&t4_list, sc, link); 682 sx_xunlock(&t4_list_lock); 683 684 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 685 TAILQ_INIT(&sc->sfl); 686 callout_init_mtx(&sc->sfl_callout, &sc->sfl_lock, 0); 687 688 mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF); 689 690 rc = map_bars_0_and_4(sc); 691 if (rc != 0) 692 goto done; /* error message displayed already */ 693 694 /* 695 * This is the real PF# to which we're attaching. Works from within PCI 696 * passthrough environments too, where pci_get_function() could return a 697 * different PF# depending on the passthrough configuration. We need to 698 * use the real PF# in all our communication with the firmware. 699 */ 700 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 701 sc->mbox = sc->pf; 702 703 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 704 sc->an_handler = an_not_handled; 705 for (i = 0; i < nitems(sc->cpl_handler); i++) 706 sc->cpl_handler[i] = cpl_not_handled; 707 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 708 sc->fw_msg_handler[i] = fw_msg_not_handled; 709 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 710 t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt); 711 t4_register_cpl_handler(sc, CPL_T5_TRACE_PKT, t5_trace_pkt); 712 t4_init_sge_cpl_handlers(sc); 713 714 /* Prepare the adapter for operation */ 715 rc = -t4_prep_adapter(sc); 716 if (rc != 0) { 717 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 718 goto done; 719 } 720 721 /* 722 * Do this really early, with the memory windows set up even before the 723 * character device. The userland tool's register i/o and mem read 724 * will work even in "recovery mode". 725 */ 726 setup_memwin(sc); 727 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 728 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 729 device_get_nameunit(dev)); 730 if (sc->cdev == NULL) 731 device_printf(dev, "failed to create nexus char device.\n"); 732 else 733 sc->cdev->si_drv1 = sc; 734 735 /* Go no further if recovery mode has been requested. */ 736 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 737 device_printf(dev, "recovery mode.\n"); 738 goto done; 739 } 740 741 #if defined(__i386__) 742 if ((cpu_feature & CPUID_CX8) == 0) { 743 device_printf(dev, "64 bit atomics not available.\n"); 744 rc = ENOTSUP; 745 goto done; 746 } 747 #endif 748 749 /* Prepare the firmware for operation */ 750 rc = prep_firmware(sc); 751 if (rc != 0) 752 goto done; /* error message displayed already */ 753 754 rc = get_params__post_init(sc); 755 if (rc != 0) 756 goto done; /* error message displayed already */ 757 758 rc = set_params__post_init(sc); 759 if (rc != 0) 760 goto done; /* error message displayed already */ 761 762 rc = map_bar_2(sc); 763 if (rc != 0) 764 goto done; /* error message displayed already */ 765 766 rc = t4_create_dma_tag(sc); 767 if (rc != 0) 768 goto done; /* error message displayed already */ 769 770 /* 771 * Number of VIs to create per-port. The first VI is the 772 * "main" regular VI for the port. The second VI is used for 773 * netmap if present, and any remaining VIs are used for 774 * additional virtual interfaces. 775 * 776 * Limit the number of VIs per port to the number of available 777 * MAC addresses per port. 778 */ 779 if (t4_num_vis >= 1) 780 num_vis = t4_num_vis; 781 else 782 num_vis = 1; 783 #ifdef DEV_NETMAP 784 num_vis++; 785 #endif 786 if (num_vis > nitems(vi_mac_funcs)) { 787 num_vis = nitems(vi_mac_funcs); 788 device_printf(dev, "Number of VIs limited to %d\n", num_vis); 789 } 790 791 /* 792 * First pass over all the ports - allocate VIs and initialize some 793 * basic parameters like mac address, port type, etc. We also figure 794 * out whether a port is 10G or 1G and use that information when 795 * calculating how many interrupts to attempt to allocate. 796 */ 797 n10g = n1g = 0; 798 for_each_port(sc, i) { 799 struct port_info *pi; 800 struct vi_info *vi; 801 802 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 803 sc->port[i] = pi; 804 805 /* These must be set before t4_port_init */ 806 pi->adapter = sc; 807 pi->port_id = i; 808 pi->nvi = num_vis; 809 pi->vi = malloc(sizeof(struct vi_info) * num_vis, M_CXGBE, 810 M_ZERO | M_WAITOK); 811 812 /* 813 * Allocate the "main" VI and initialize parameters 814 * like mac addr. 815 */ 816 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 817 if (rc != 0) { 818 device_printf(dev, "unable to initialize port %d: %d\n", 819 i, rc); 820 free(pi->vi, M_CXGBE); 821 free(pi, M_CXGBE); 822 sc->port[i] = NULL; 823 goto done; 824 } 825 826 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX); 827 pi->link_cfg.requested_fc |= t4_pause_settings; 828 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX); 829 pi->link_cfg.fc |= t4_pause_settings; 830 831 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 832 if (rc != 0) { 833 device_printf(dev, "port %d l1cfg failed: %d\n", i, rc); 834 free(pi->vi, M_CXGBE); 835 free(pi, M_CXGBE); 836 sc->port[i] = NULL; 837 goto done; 838 } 839 840 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 841 device_get_nameunit(dev), i); 842 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 843 sc->chan_map[pi->tx_chan] = i; 844 845 if (is_10G_port(pi) || is_40G_port(pi)) { 846 n10g++; 847 for_each_vi(pi, j, vi) { 848 vi->tmr_idx = t4_tmr_idx_10g; 849 vi->pktc_idx = t4_pktc_idx_10g; 850 } 851 } else { 852 n1g++; 853 for_each_vi(pi, j, vi) { 854 vi->tmr_idx = t4_tmr_idx_1g; 855 vi->pktc_idx = t4_pktc_idx_1g; 856 } 857 } 858 859 pi->linkdnrc = -1; 860 861 for_each_vi(pi, j, vi) { 862 vi->qsize_rxq = t4_qsize_rxq; 863 vi->qsize_txq = t4_qsize_txq; 864 vi->pi = pi; 865 } 866 867 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 868 if (pi->dev == NULL) { 869 device_printf(dev, 870 "failed to add device for port %d.\n", i); 871 rc = ENXIO; 872 goto done; 873 } 874 pi->vi[0].dev = pi->dev; 875 device_set_softc(pi->dev, pi); 876 } 877 878 /* 879 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 880 */ 881 #ifdef DEV_NETMAP 882 num_vis--; 883 #endif 884 rc = cfg_itype_and_nqueues(sc, n10g, n1g, num_vis, &iaq); 885 if (rc != 0) 886 goto done; /* error message displayed already */ 887 888 sc->intr_type = iaq.intr_type; 889 sc->intr_count = iaq.nirq; 890 891 s = &sc->sge; 892 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 893 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 894 if (num_vis > 1) { 895 s->nrxq += (n10g + n1g) * (num_vis - 1); 896 s->ntxq += (n10g + n1g) * (num_vis - 1); 897 } 898 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 899 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 900 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 901 #ifdef TCP_OFFLOAD 902 if (is_offload(sc)) { 903 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 904 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 905 if (num_vis > 1) { 906 s->nofldrxq += (n10g + n1g) * (num_vis - 1); 907 s->nofldtxq += (n10g + n1g) * (num_vis - 1); 908 } 909 s->neq += s->nofldtxq + s->nofldrxq; 910 s->niq += s->nofldrxq; 911 912 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 913 M_CXGBE, M_ZERO | M_WAITOK); 914 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 915 M_CXGBE, M_ZERO | M_WAITOK); 916 } 917 #endif 918 #ifdef DEV_NETMAP 919 s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g; 920 s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g; 921 s->neq += s->nnmtxq + s->nnmrxq; 922 s->niq += s->nnmrxq; 923 924 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq), 925 M_CXGBE, M_ZERO | M_WAITOK); 926 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq), 927 M_CXGBE, M_ZERO | M_WAITOK); 928 #endif 929 930 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 931 M_ZERO | M_WAITOK); 932 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 933 M_ZERO | M_WAITOK); 934 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 935 M_ZERO | M_WAITOK); 936 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 937 M_ZERO | M_WAITOK); 938 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 939 M_ZERO | M_WAITOK); 940 941 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 942 M_ZERO | M_WAITOK); 943 944 t4_init_l2t(sc, M_WAITOK); 945 946 /* 947 * Second pass over the ports. This time we know the number of rx and 948 * tx queues that each port should get. 949 */ 950 rqidx = tqidx = 0; 951 #ifdef TCP_OFFLOAD 952 ofld_rqidx = ofld_tqidx = 0; 953 #endif 954 #ifdef DEV_NETMAP 955 nm_rqidx = nm_tqidx = 0; 956 #endif 957 for_each_port(sc, i) { 958 struct port_info *pi = sc->port[i]; 959 struct vi_info *vi; 960 961 if (pi == NULL) 962 continue; 963 964 for_each_vi(pi, j, vi) { 965 #ifdef DEV_NETMAP 966 if (j == 1) { 967 vi->flags |= VI_NETMAP | INTR_RXQ; 968 vi->first_rxq = nm_rqidx; 969 vi->first_txq = nm_tqidx; 970 if (is_10G_port(pi) || is_40G_port(pi)) { 971 vi->nrxq = iaq.nnmrxq10g; 972 vi->ntxq = iaq.nnmtxq10g; 973 } else { 974 vi->nrxq = iaq.nnmrxq1g; 975 vi->ntxq = iaq.nnmtxq1g; 976 } 977 nm_rqidx += vi->nrxq; 978 nm_tqidx += vi->ntxq; 979 continue; 980 } 981 #endif 982 983 vi->first_rxq = rqidx; 984 vi->first_txq = tqidx; 985 if (is_10G_port(pi) || is_40G_port(pi)) { 986 vi->flags |= iaq.intr_flags_10g & INTR_RXQ; 987 vi->nrxq = j == 0 ? iaq.nrxq10g : 1; 988 vi->ntxq = j == 0 ? iaq.ntxq10g : 1; 989 } else { 990 vi->flags |= iaq.intr_flags_1g & INTR_RXQ; 991 vi->nrxq = j == 0 ? iaq.nrxq1g : 1; 992 vi->ntxq = j == 0 ? iaq.ntxq1g : 1; 993 } 994 995 if (vi->ntxq > 1) 996 vi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0; 997 else 998 vi->rsrv_noflowq = 0; 999 1000 rqidx += vi->nrxq; 1001 tqidx += vi->ntxq; 1002 1003 #ifdef TCP_OFFLOAD 1004 if (!is_offload(sc)) 1005 continue; 1006 vi->first_ofld_rxq = ofld_rqidx; 1007 vi->first_ofld_txq = ofld_tqidx; 1008 if (is_10G_port(pi) || is_40G_port(pi)) { 1009 vi->flags |= iaq.intr_flags_10g & INTR_OFLD_RXQ; 1010 vi->nofldrxq = j == 0 ? iaq.nofldrxq10g : 1; 1011 vi->nofldtxq = j == 0 ? iaq.nofldtxq10g : 1; 1012 } else { 1013 vi->flags |= iaq.intr_flags_1g & INTR_OFLD_RXQ; 1014 vi->nofldrxq = j == 0 ? iaq.nofldrxq1g : 1; 1015 vi->nofldtxq = j == 0 ? iaq.nofldtxq1g : 1; 1016 } 1017 ofld_rqidx += vi->nofldrxq; 1018 ofld_tqidx += vi->nofldtxq; 1019 #endif 1020 } 1021 } 1022 1023 rc = setup_intr_handlers(sc); 1024 if (rc != 0) { 1025 device_printf(dev, 1026 "failed to setup interrupt handlers: %d\n", rc); 1027 goto done; 1028 } 1029 1030 rc = bus_generic_attach(dev); 1031 if (rc != 0) { 1032 device_printf(dev, 1033 "failed to attach all child ports: %d\n", rc); 1034 goto done; 1035 } 1036 1037 device_printf(dev, 1038 "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 1039 sc->params.pci.speed, sc->params.pci.width, sc->params.nports, 1040 sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" : 1041 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 1042 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 1043 1044 t4_set_desc(sc); 1045 1046 done: 1047 if (rc != 0 && sc->cdev) { 1048 /* cdev was created and so cxgbetool works; recover that way. */ 1049 device_printf(dev, 1050 "error during attach, adapter is now in recovery mode.\n"); 1051 rc = 0; 1052 } 1053 1054 if (rc != 0) 1055 t4_detach(dev); 1056 else 1057 t4_sysctls(sc); 1058 1059 return (rc); 1060 } 1061 1062 /* 1063 * Idempotent 1064 */ 1065 static int 1066 t4_detach(device_t dev) 1067 { 1068 struct adapter *sc; 1069 struct port_info *pi; 1070 int i, rc; 1071 1072 sc = device_get_softc(dev); 1073 1074 if (sc->flags & FULL_INIT_DONE) 1075 t4_intr_disable(sc); 1076 1077 if (sc->cdev) { 1078 destroy_dev(sc->cdev); 1079 sc->cdev = NULL; 1080 } 1081 1082 rc = bus_generic_detach(dev); 1083 if (rc) { 1084 device_printf(dev, 1085 "failed to detach child devices: %d\n", rc); 1086 return (rc); 1087 } 1088 1089 for (i = 0; i < sc->intr_count; i++) 1090 t4_free_irq(sc, &sc->irq[i]); 1091 1092 for (i = 0; i < MAX_NPORTS; i++) { 1093 pi = sc->port[i]; 1094 if (pi) { 1095 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->vi[0].viid); 1096 if (pi->dev) 1097 device_delete_child(dev, pi->dev); 1098 1099 mtx_destroy(&pi->pi_lock); 1100 free(pi->vi, M_CXGBE); 1101 free(pi, M_CXGBE); 1102 } 1103 } 1104 1105 if (sc->flags & FULL_INIT_DONE) 1106 adapter_full_uninit(sc); 1107 1108 if (sc->flags & FW_OK) 1109 t4_fw_bye(sc, sc->mbox); 1110 1111 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 1112 pci_release_msi(dev); 1113 1114 if (sc->regs_res) 1115 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 1116 sc->regs_res); 1117 1118 if (sc->udbs_res) 1119 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 1120 sc->udbs_res); 1121 1122 if (sc->msix_res) 1123 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 1124 sc->msix_res); 1125 1126 if (sc->l2t) 1127 t4_free_l2t(sc->l2t); 1128 1129 #ifdef TCP_OFFLOAD 1130 free(sc->sge.ofld_rxq, M_CXGBE); 1131 free(sc->sge.ofld_txq, M_CXGBE); 1132 #endif 1133 #ifdef DEV_NETMAP 1134 free(sc->sge.nm_rxq, M_CXGBE); 1135 free(sc->sge.nm_txq, M_CXGBE); 1136 #endif 1137 free(sc->irq, M_CXGBE); 1138 free(sc->sge.rxq, M_CXGBE); 1139 free(sc->sge.txq, M_CXGBE); 1140 free(sc->sge.ctrlq, M_CXGBE); 1141 free(sc->sge.iqmap, M_CXGBE); 1142 free(sc->sge.eqmap, M_CXGBE); 1143 free(sc->tids.ftid_tab, M_CXGBE); 1144 t4_destroy_dma_tag(sc); 1145 if (mtx_initialized(&sc->sc_lock)) { 1146 sx_xlock(&t4_list_lock); 1147 SLIST_REMOVE(&t4_list, sc, adapter, link); 1148 sx_xunlock(&t4_list_lock); 1149 mtx_destroy(&sc->sc_lock); 1150 } 1151 1152 callout_drain(&sc->sfl_callout); 1153 if (mtx_initialized(&sc->tids.ftid_lock)) 1154 mtx_destroy(&sc->tids.ftid_lock); 1155 if (mtx_initialized(&sc->sfl_lock)) 1156 mtx_destroy(&sc->sfl_lock); 1157 if (mtx_initialized(&sc->ifp_lock)) 1158 mtx_destroy(&sc->ifp_lock); 1159 if (mtx_initialized(&sc->regwin_lock)) 1160 mtx_destroy(&sc->regwin_lock); 1161 1162 bzero(sc, sizeof(*sc)); 1163 1164 return (0); 1165 } 1166 1167 static int 1168 cxgbe_probe(device_t dev) 1169 { 1170 char buf[128]; 1171 struct port_info *pi = device_get_softc(dev); 1172 1173 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 1174 device_set_desc_copy(dev, buf); 1175 1176 return (BUS_PROBE_DEFAULT); 1177 } 1178 1179 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 1180 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 1181 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS) 1182 #define T4_CAP_ENABLE (T4_CAP) 1183 1184 static int 1185 cxgbe_vi_attach(device_t dev, struct vi_info *vi) 1186 { 1187 struct ifnet *ifp; 1188 struct sbuf *sb; 1189 1190 vi->xact_addr_filt = -1; 1191 callout_init(&vi->tick, 1); 1192 1193 /* Allocate an ifnet and set it up */ 1194 ifp = if_alloc(IFT_ETHER); 1195 if (ifp == NULL) { 1196 device_printf(dev, "Cannot allocate ifnet\n"); 1197 return (ENOMEM); 1198 } 1199 vi->ifp = ifp; 1200 ifp->if_softc = vi; 1201 1202 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1203 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1204 1205 ifp->if_init = cxgbe_init; 1206 ifp->if_ioctl = cxgbe_ioctl; 1207 ifp->if_transmit = cxgbe_transmit; 1208 ifp->if_qflush = cxgbe_qflush; 1209 ifp->if_get_counter = cxgbe_get_counter; 1210 1211 ifp->if_capabilities = T4_CAP; 1212 #ifdef TCP_OFFLOAD 1213 if (vi->nofldrxq != 0) 1214 ifp->if_capabilities |= IFCAP_TOE; 1215 #endif 1216 ifp->if_capenable = T4_CAP_ENABLE; 1217 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1218 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1219 1220 ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1221 ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS; 1222 ifp->if_hw_tsomaxsegsize = 65536; 1223 1224 /* Initialize ifmedia for this VI */ 1225 ifmedia_init(&vi->media, IFM_IMASK, cxgbe_media_change, 1226 cxgbe_media_status); 1227 build_medialist(vi->pi, &vi->media); 1228 1229 vi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 1230 EVENTHANDLER_PRI_ANY); 1231 1232 ether_ifattach(ifp, vi->hw_addr); 1233 1234 sb = sbuf_new_auto(); 1235 sbuf_printf(sb, "%d txq, %d rxq (NIC)", vi->ntxq, vi->nrxq); 1236 #ifdef TCP_OFFLOAD 1237 if (ifp->if_capabilities & IFCAP_TOE) 1238 sbuf_printf(sb, "; %d txq, %d rxq (TOE)", 1239 vi->nofldtxq, vi->nofldrxq); 1240 #endif 1241 sbuf_finish(sb); 1242 device_printf(dev, "%s\n", sbuf_data(sb)); 1243 sbuf_delete(sb); 1244 1245 vi_sysctls(vi); 1246 1247 return (0); 1248 } 1249 1250 static int 1251 cxgbe_attach(device_t dev) 1252 { 1253 struct port_info *pi = device_get_softc(dev); 1254 struct vi_info *vi; 1255 int i, rc; 1256 1257 callout_init_mtx(&pi->tick, &pi->pi_lock, 0); 1258 1259 rc = cxgbe_vi_attach(dev, &pi->vi[0]); 1260 if (rc) 1261 return (rc); 1262 1263 for_each_vi(pi, i, vi) { 1264 if (i == 0) 1265 continue; 1266 #ifdef DEV_NETMAP 1267 if (vi->flags & VI_NETMAP) { 1268 /* 1269 * media handled here to keep 1270 * implementation private to this file 1271 */ 1272 ifmedia_init(&vi->media, IFM_IMASK, cxgbe_media_change, 1273 cxgbe_media_status); 1274 build_medialist(pi, &vi->media); 1275 vi->dev = device_add_child(dev, is_t4(pi->adapter) ? 1276 "ncxgbe" : "ncxl", device_get_unit(dev)); 1277 } else 1278 #endif 1279 vi->dev = device_add_child(dev, is_t4(pi->adapter) ? 1280 "vcxgbe" : "vcxl", -1); 1281 if (vi->dev == NULL) { 1282 device_printf(dev, "failed to add VI %d\n", i); 1283 continue; 1284 } 1285 device_set_softc(vi->dev, vi); 1286 } 1287 1288 cxgbe_sysctls(pi); 1289 1290 bus_generic_attach(dev); 1291 1292 return (0); 1293 } 1294 1295 static void 1296 cxgbe_vi_detach(struct vi_info *vi) 1297 { 1298 struct ifnet *ifp = vi->ifp; 1299 1300 ether_ifdetach(ifp); 1301 1302 if (vi->vlan_c) 1303 EVENTHANDLER_DEREGISTER(vlan_config, vi->vlan_c); 1304 1305 /* Let detach proceed even if these fail. */ 1306 cxgbe_uninit_synchronized(vi); 1307 callout_drain(&vi->tick); 1308 vi_full_uninit(vi); 1309 1310 ifmedia_removeall(&vi->media); 1311 if_free(vi->ifp); 1312 vi->ifp = NULL; 1313 } 1314 1315 static int 1316 cxgbe_detach(device_t dev) 1317 { 1318 struct port_info *pi = device_get_softc(dev); 1319 struct adapter *sc = pi->adapter; 1320 int rc; 1321 1322 /* Detach the extra VIs first. */ 1323 rc = bus_generic_detach(dev); 1324 if (rc) 1325 return (rc); 1326 device_delete_children(dev); 1327 1328 doom_vi(sc, &pi->vi[0]); 1329 1330 if (pi->flags & HAS_TRACEQ) { 1331 sc->traceq = -1; /* cloner should not create ifnet */ 1332 t4_tracer_port_detach(sc); 1333 } 1334 1335 cxgbe_vi_detach(&pi->vi[0]); 1336 callout_drain(&pi->tick); 1337 1338 end_synchronized_op(sc, 0); 1339 1340 return (0); 1341 } 1342 1343 static void 1344 cxgbe_init(void *arg) 1345 { 1346 struct vi_info *vi = arg; 1347 struct adapter *sc = vi->pi->adapter; 1348 1349 if (begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4init") != 0) 1350 return; 1351 cxgbe_init_synchronized(vi); 1352 end_synchronized_op(sc, 0); 1353 } 1354 1355 static int 1356 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1357 { 1358 int rc = 0, mtu, flags, can_sleep; 1359 struct vi_info *vi = ifp->if_softc; 1360 struct adapter *sc = vi->pi->adapter; 1361 struct ifreq *ifr = (struct ifreq *)data; 1362 uint32_t mask; 1363 1364 switch (cmd) { 1365 case SIOCSIFMTU: 1366 mtu = ifr->ifr_mtu; 1367 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1368 return (EINVAL); 1369 1370 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4mtu"); 1371 if (rc) 1372 return (rc); 1373 ifp->if_mtu = mtu; 1374 if (vi->flags & VI_INIT_DONE) { 1375 t4_update_fl_bufsize(ifp); 1376 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1377 rc = update_mac_settings(ifp, XGMAC_MTU); 1378 } 1379 end_synchronized_op(sc, 0); 1380 break; 1381 1382 case SIOCSIFFLAGS: 1383 can_sleep = 0; 1384 redo_sifflags: 1385 rc = begin_synchronized_op(sc, vi, 1386 can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg"); 1387 if (rc) 1388 return (rc); 1389 1390 if (ifp->if_flags & IFF_UP) { 1391 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1392 flags = vi->if_flags; 1393 if ((ifp->if_flags ^ flags) & 1394 (IFF_PROMISC | IFF_ALLMULTI)) { 1395 if (can_sleep == 1) { 1396 end_synchronized_op(sc, 0); 1397 can_sleep = 0; 1398 goto redo_sifflags; 1399 } 1400 rc = update_mac_settings(ifp, 1401 XGMAC_PROMISC | XGMAC_ALLMULTI); 1402 } 1403 } else { 1404 if (can_sleep == 0) { 1405 end_synchronized_op(sc, LOCK_HELD); 1406 can_sleep = 1; 1407 goto redo_sifflags; 1408 } 1409 rc = cxgbe_init_synchronized(vi); 1410 } 1411 vi->if_flags = ifp->if_flags; 1412 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1413 if (can_sleep == 0) { 1414 end_synchronized_op(sc, LOCK_HELD); 1415 can_sleep = 1; 1416 goto redo_sifflags; 1417 } 1418 rc = cxgbe_uninit_synchronized(vi); 1419 } 1420 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD); 1421 break; 1422 1423 case SIOCADDMULTI: 1424 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1425 rc = begin_synchronized_op(sc, vi, HOLD_LOCK, "t4multi"); 1426 if (rc) 1427 return (rc); 1428 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1429 rc = update_mac_settings(ifp, XGMAC_MCADDRS); 1430 end_synchronized_op(sc, LOCK_HELD); 1431 break; 1432 1433 case SIOCSIFCAP: 1434 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4cap"); 1435 if (rc) 1436 return (rc); 1437 1438 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1439 if (mask & IFCAP_TXCSUM) { 1440 ifp->if_capenable ^= IFCAP_TXCSUM; 1441 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1442 1443 if (IFCAP_TSO4 & ifp->if_capenable && 1444 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1445 ifp->if_capenable &= ~IFCAP_TSO4; 1446 if_printf(ifp, 1447 "tso4 disabled due to -txcsum.\n"); 1448 } 1449 } 1450 if (mask & IFCAP_TXCSUM_IPV6) { 1451 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1452 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1453 1454 if (IFCAP_TSO6 & ifp->if_capenable && 1455 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1456 ifp->if_capenable &= ~IFCAP_TSO6; 1457 if_printf(ifp, 1458 "tso6 disabled due to -txcsum6.\n"); 1459 } 1460 } 1461 if (mask & IFCAP_RXCSUM) 1462 ifp->if_capenable ^= IFCAP_RXCSUM; 1463 if (mask & IFCAP_RXCSUM_IPV6) 1464 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1465 1466 /* 1467 * Note that we leave CSUM_TSO alone (it is always set). The 1468 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1469 * sending a TSO request our way, so it's sufficient to toggle 1470 * IFCAP_TSOx only. 1471 */ 1472 if (mask & IFCAP_TSO4) { 1473 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1474 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1475 if_printf(ifp, "enable txcsum first.\n"); 1476 rc = EAGAIN; 1477 goto fail; 1478 } 1479 ifp->if_capenable ^= IFCAP_TSO4; 1480 } 1481 if (mask & IFCAP_TSO6) { 1482 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1483 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1484 if_printf(ifp, "enable txcsum6 first.\n"); 1485 rc = EAGAIN; 1486 goto fail; 1487 } 1488 ifp->if_capenable ^= IFCAP_TSO6; 1489 } 1490 if (mask & IFCAP_LRO) { 1491 #if defined(INET) || defined(INET6) 1492 int i; 1493 struct sge_rxq *rxq; 1494 1495 ifp->if_capenable ^= IFCAP_LRO; 1496 for_each_rxq(vi, i, rxq) { 1497 if (ifp->if_capenable & IFCAP_LRO) 1498 rxq->iq.flags |= IQ_LRO_ENABLED; 1499 else 1500 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1501 } 1502 #endif 1503 } 1504 #ifdef TCP_OFFLOAD 1505 if (mask & IFCAP_TOE) { 1506 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1507 1508 rc = toe_capability(vi, enable); 1509 if (rc != 0) 1510 goto fail; 1511 1512 ifp->if_capenable ^= mask; 1513 } 1514 #endif 1515 if (mask & IFCAP_VLAN_HWTAGGING) { 1516 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1517 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1518 rc = update_mac_settings(ifp, XGMAC_VLANEX); 1519 } 1520 if (mask & IFCAP_VLAN_MTU) { 1521 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1522 1523 /* Need to find out how to disable auto-mtu-inflation */ 1524 } 1525 if (mask & IFCAP_VLAN_HWTSO) 1526 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1527 if (mask & IFCAP_VLAN_HWCSUM) 1528 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1529 1530 #ifdef VLAN_CAPABILITIES 1531 VLAN_CAPABILITIES(ifp); 1532 #endif 1533 fail: 1534 end_synchronized_op(sc, 0); 1535 break; 1536 1537 case SIOCSIFMEDIA: 1538 case SIOCGIFMEDIA: 1539 ifmedia_ioctl(ifp, ifr, &vi->media, cmd); 1540 break; 1541 1542 case SIOCGI2C: { 1543 struct ifi2creq i2c; 1544 1545 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); 1546 if (rc != 0) 1547 break; 1548 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) { 1549 rc = EPERM; 1550 break; 1551 } 1552 if (i2c.len > sizeof(i2c.data)) { 1553 rc = EINVAL; 1554 break; 1555 } 1556 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4i2c"); 1557 if (rc) 1558 return (rc); 1559 rc = -t4_i2c_rd(sc, sc->mbox, vi->pi->port_id, i2c.dev_addr, 1560 i2c.offset, i2c.len, &i2c.data[0]); 1561 end_synchronized_op(sc, 0); 1562 if (rc == 0) 1563 rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); 1564 break; 1565 } 1566 1567 default: 1568 rc = ether_ioctl(ifp, cmd, data); 1569 } 1570 1571 return (rc); 1572 } 1573 1574 static int 1575 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1576 { 1577 struct vi_info *vi = ifp->if_softc; 1578 struct port_info *pi = vi->pi; 1579 struct adapter *sc = pi->adapter; 1580 struct sge_txq *txq; 1581 void *items[1]; 1582 int rc; 1583 1584 M_ASSERTPKTHDR(m); 1585 MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */ 1586 1587 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1588 m_freem(m); 1589 return (ENETDOWN); 1590 } 1591 1592 rc = parse_pkt(&m); 1593 if (__predict_false(rc != 0)) { 1594 MPASS(m == NULL); /* was freed already */ 1595 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */ 1596 return (rc); 1597 } 1598 1599 /* Select a txq. */ 1600 txq = &sc->sge.txq[vi->first_txq]; 1601 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 1602 txq += ((m->m_pkthdr.flowid % (vi->ntxq - vi->rsrv_noflowq)) + 1603 vi->rsrv_noflowq); 1604 1605 items[0] = m; 1606 rc = mp_ring_enqueue(txq->r, items, 1, 4096); 1607 if (__predict_false(rc != 0)) 1608 m_freem(m); 1609 1610 return (rc); 1611 } 1612 1613 static void 1614 cxgbe_qflush(struct ifnet *ifp) 1615 { 1616 struct vi_info *vi = ifp->if_softc; 1617 struct sge_txq *txq; 1618 int i; 1619 1620 /* queues do not exist if !VI_INIT_DONE. */ 1621 if (vi->flags & VI_INIT_DONE) { 1622 for_each_txq(vi, i, txq) { 1623 TXQ_LOCK(txq); 1624 txq->eq.flags &= ~EQ_ENABLED; 1625 TXQ_UNLOCK(txq); 1626 while (!mp_ring_is_idle(txq->r)) { 1627 mp_ring_check_drainage(txq->r, 0); 1628 pause("qflush", 1); 1629 } 1630 } 1631 } 1632 if_qflush(ifp); 1633 } 1634 1635 static uint64_t 1636 vi_get_counter(struct ifnet *ifp, ift_counter c) 1637 { 1638 struct vi_info *vi = ifp->if_softc; 1639 struct fw_vi_stats_vf *s = &vi->stats; 1640 1641 vi_refresh_stats(vi->pi->adapter, vi); 1642 1643 switch (c) { 1644 case IFCOUNTER_IPACKETS: 1645 return (s->rx_bcast_frames + s->rx_mcast_frames + 1646 s->rx_ucast_frames); 1647 case IFCOUNTER_IERRORS: 1648 return (s->rx_err_frames); 1649 case IFCOUNTER_OPACKETS: 1650 return (s->tx_bcast_frames + s->tx_mcast_frames + 1651 s->tx_ucast_frames + s->tx_offload_frames); 1652 case IFCOUNTER_OERRORS: 1653 return (s->tx_drop_frames); 1654 case IFCOUNTER_IBYTES: 1655 return (s->rx_bcast_bytes + s->rx_mcast_bytes + 1656 s->rx_ucast_bytes); 1657 case IFCOUNTER_OBYTES: 1658 return (s->tx_bcast_bytes + s->tx_mcast_bytes + 1659 s->tx_ucast_bytes + s->tx_offload_bytes); 1660 case IFCOUNTER_IMCASTS: 1661 return (s->rx_mcast_frames); 1662 case IFCOUNTER_OMCASTS: 1663 return (s->tx_mcast_frames); 1664 case IFCOUNTER_OQDROPS: { 1665 uint64_t drops; 1666 1667 drops = 0; 1668 if ((vi->flags & (VI_INIT_DONE | VI_NETMAP)) == VI_INIT_DONE) { 1669 int i; 1670 struct sge_txq *txq; 1671 1672 for_each_txq(vi, i, txq) 1673 drops += counter_u64_fetch(txq->r->drops); 1674 } 1675 1676 return (drops); 1677 1678 } 1679 1680 default: 1681 return (if_get_counter_default(ifp, c)); 1682 } 1683 } 1684 1685 uint64_t 1686 cxgbe_get_counter(struct ifnet *ifp, ift_counter c) 1687 { 1688 struct vi_info *vi = ifp->if_softc; 1689 struct port_info *pi = vi->pi; 1690 struct adapter *sc = pi->adapter; 1691 struct port_stats *s = &pi->stats; 1692 1693 if (pi->nvi > 1) 1694 return (vi_get_counter(ifp, c)); 1695 1696 cxgbe_refresh_stats(sc, pi); 1697 1698 switch (c) { 1699 case IFCOUNTER_IPACKETS: 1700 return (s->rx_frames - s->rx_pause); 1701 1702 case IFCOUNTER_IERRORS: 1703 return (s->rx_jabber + s->rx_runt + s->rx_too_long + 1704 s->rx_fcs_err + s->rx_len_err); 1705 1706 case IFCOUNTER_OPACKETS: 1707 return (s->tx_frames - s->tx_pause); 1708 1709 case IFCOUNTER_OERRORS: 1710 return (s->tx_error_frames); 1711 1712 case IFCOUNTER_IBYTES: 1713 return (s->rx_octets - s->rx_pause * 64); 1714 1715 case IFCOUNTER_OBYTES: 1716 return (s->tx_octets - s->tx_pause * 64); 1717 1718 case IFCOUNTER_IMCASTS: 1719 return (s->rx_mcast_frames - s->rx_pause); 1720 1721 case IFCOUNTER_OMCASTS: 1722 return (s->tx_mcast_frames - s->tx_pause); 1723 1724 case IFCOUNTER_IQDROPS: 1725 return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 1726 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 1727 s->rx_trunc3 + pi->tnl_cong_drops); 1728 1729 case IFCOUNTER_OQDROPS: { 1730 uint64_t drops; 1731 1732 drops = s->tx_drop; 1733 if (vi->flags & VI_INIT_DONE) { 1734 int i; 1735 struct sge_txq *txq; 1736 1737 for_each_txq(vi, i, txq) 1738 drops += counter_u64_fetch(txq->r->drops); 1739 } 1740 1741 return (drops); 1742 1743 } 1744 1745 default: 1746 return (if_get_counter_default(ifp, c)); 1747 } 1748 } 1749 1750 static int 1751 cxgbe_media_change(struct ifnet *ifp) 1752 { 1753 struct vi_info *vi = ifp->if_softc; 1754 1755 device_printf(vi->dev, "%s unimplemented.\n", __func__); 1756 1757 return (EOPNOTSUPP); 1758 } 1759 1760 static void 1761 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1762 { 1763 struct vi_info *vi = ifp->if_softc; 1764 struct port_info *pi = vi->pi; 1765 struct ifmedia_entry *cur; 1766 int speed = pi->link_cfg.speed; 1767 1768 cur = vi->media.ifm_cur; 1769 1770 ifmr->ifm_status = IFM_AVALID; 1771 if (!pi->link_cfg.link_ok) 1772 return; 1773 1774 ifmr->ifm_status |= IFM_ACTIVE; 1775 1776 /* active and current will differ iff current media is autoselect. */ 1777 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1778 return; 1779 1780 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1781 if (speed == SPEED_10000) 1782 ifmr->ifm_active |= IFM_10G_T; 1783 else if (speed == SPEED_1000) 1784 ifmr->ifm_active |= IFM_1000_T; 1785 else if (speed == SPEED_100) 1786 ifmr->ifm_active |= IFM_100_TX; 1787 else if (speed == SPEED_10) 1788 ifmr->ifm_active |= IFM_10_T; 1789 else 1790 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1791 speed)); 1792 } 1793 1794 static int 1795 vcxgbe_probe(device_t dev) 1796 { 1797 char buf[128]; 1798 struct vi_info *vi = device_get_softc(dev); 1799 1800 snprintf(buf, sizeof(buf), "port %d vi %td", vi->pi->port_id, 1801 vi - vi->pi->vi); 1802 device_set_desc_copy(dev, buf); 1803 1804 return (BUS_PROBE_DEFAULT); 1805 } 1806 1807 static int 1808 vcxgbe_attach(device_t dev) 1809 { 1810 struct vi_info *vi; 1811 struct port_info *pi; 1812 struct adapter *sc; 1813 int func, index, rc; 1814 u32 param, val; 1815 1816 vi = device_get_softc(dev); 1817 pi = vi->pi; 1818 sc = pi->adapter; 1819 1820 index = vi - pi->vi; 1821 KASSERT(index < nitems(vi_mac_funcs), 1822 ("%s: VI %s doesn't have a MAC func", __func__, 1823 device_get_nameunit(dev))); 1824 func = vi_mac_funcs[index]; 1825 rc = t4_alloc_vi_func(sc, sc->mbox, pi->tx_chan, sc->pf, 0, 1, 1826 vi->hw_addr, &vi->rss_size, func, 0); 1827 if (rc < 0) { 1828 device_printf(dev, "Failed to allocate virtual interface " 1829 "for port %d: %d\n", pi->port_id, -rc); 1830 return (-rc); 1831 } 1832 vi->viid = rc; 1833 1834 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 1835 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_RSSINFO) | 1836 V_FW_PARAMS_PARAM_YZ(vi->viid); 1837 rc = t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 1838 if (rc) 1839 vi->rss_base = 0xffff; 1840 else { 1841 /* MPASS((val >> 16) == rss_size); */ 1842 vi->rss_base = val & 0xffff; 1843 } 1844 1845 rc = cxgbe_vi_attach(dev, vi); 1846 if (rc) { 1847 t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid); 1848 return (rc); 1849 } 1850 return (0); 1851 } 1852 1853 static int 1854 vcxgbe_detach(device_t dev) 1855 { 1856 struct vi_info *vi; 1857 struct adapter *sc; 1858 1859 vi = device_get_softc(dev); 1860 sc = vi->pi->adapter; 1861 1862 doom_vi(sc, vi); 1863 1864 cxgbe_vi_detach(vi); 1865 t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid); 1866 1867 end_synchronized_op(sc, 0); 1868 1869 return (0); 1870 } 1871 1872 void 1873 t4_fatal_err(struct adapter *sc) 1874 { 1875 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1876 t4_intr_disable(sc); 1877 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1878 device_get_nameunit(sc->dev)); 1879 } 1880 1881 static int 1882 map_bars_0_and_4(struct adapter *sc) 1883 { 1884 sc->regs_rid = PCIR_BAR(0); 1885 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1886 &sc->regs_rid, RF_ACTIVE); 1887 if (sc->regs_res == NULL) { 1888 device_printf(sc->dev, "cannot map registers.\n"); 1889 return (ENXIO); 1890 } 1891 sc->bt = rman_get_bustag(sc->regs_res); 1892 sc->bh = rman_get_bushandle(sc->regs_res); 1893 sc->mmio_len = rman_get_size(sc->regs_res); 1894 setbit(&sc->doorbells, DOORBELL_KDB); 1895 1896 sc->msix_rid = PCIR_BAR(4); 1897 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1898 &sc->msix_rid, RF_ACTIVE); 1899 if (sc->msix_res == NULL) { 1900 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1901 return (ENXIO); 1902 } 1903 1904 return (0); 1905 } 1906 1907 static int 1908 map_bar_2(struct adapter *sc) 1909 { 1910 1911 /* 1912 * T4: only iWARP driver uses the userspace doorbells. There is no need 1913 * to map it if RDMA is disabled. 1914 */ 1915 if (is_t4(sc) && sc->rdmacaps == 0) 1916 return (0); 1917 1918 sc->udbs_rid = PCIR_BAR(2); 1919 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1920 &sc->udbs_rid, RF_ACTIVE); 1921 if (sc->udbs_res == NULL) { 1922 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1923 return (ENXIO); 1924 } 1925 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1926 1927 if (is_t5(sc)) { 1928 setbit(&sc->doorbells, DOORBELL_UDB); 1929 #if defined(__i386__) || defined(__amd64__) 1930 if (t5_write_combine) { 1931 int rc; 1932 1933 /* 1934 * Enable write combining on BAR2. This is the 1935 * userspace doorbell BAR and is split into 128B 1936 * (UDBS_SEG_SIZE) doorbell regions, each associated 1937 * with an egress queue. The first 64B has the doorbell 1938 * and the second 64B can be used to submit a tx work 1939 * request with an implicit doorbell. 1940 */ 1941 1942 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1943 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1944 if (rc == 0) { 1945 clrbit(&sc->doorbells, DOORBELL_UDB); 1946 setbit(&sc->doorbells, DOORBELL_WCWR); 1947 setbit(&sc->doorbells, DOORBELL_UDBWC); 1948 } else { 1949 device_printf(sc->dev, 1950 "couldn't enable write combining: %d\n", 1951 rc); 1952 } 1953 1954 t4_write_reg(sc, A_SGE_STAT_CFG, 1955 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1956 } 1957 #endif 1958 } 1959 1960 return (0); 1961 } 1962 1963 static const struct memwin t4_memwin[] = { 1964 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1965 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1966 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1967 }; 1968 1969 static const struct memwin t5_memwin[] = { 1970 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1971 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1972 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1973 }; 1974 1975 static void 1976 setup_memwin(struct adapter *sc) 1977 { 1978 const struct memwin *mw; 1979 int i, n; 1980 uint32_t bar0; 1981 1982 if (is_t4(sc)) { 1983 /* 1984 * Read low 32b of bar0 indirectly via the hardware backdoor 1985 * mechanism. Works from within PCI passthrough environments 1986 * too, where rman_get_start() can return a different value. We 1987 * need to program the T4 memory window decoders with the actual 1988 * addresses that will be coming across the PCIe link. 1989 */ 1990 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1991 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1992 1993 mw = &t4_memwin[0]; 1994 n = nitems(t4_memwin); 1995 } else { 1996 /* T5 uses the relative offset inside the PCIe BAR */ 1997 bar0 = 0; 1998 1999 mw = &t5_memwin[0]; 2000 n = nitems(t5_memwin); 2001 } 2002 2003 for (i = 0; i < n; i++, mw++) { 2004 t4_write_reg(sc, 2005 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 2006 (mw->base + bar0) | V_BIR(0) | 2007 V_WINDOW(ilog2(mw->aperture) - 10)); 2008 } 2009 2010 /* flush */ 2011 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 2012 } 2013 2014 /* 2015 * Verify that the memory range specified by the addr/len pair is valid and lies 2016 * entirely within a single region (EDCx or MCx). 2017 */ 2018 static int 2019 validate_mem_range(struct adapter *sc, uint32_t addr, int len) 2020 { 2021 uint32_t em, addr_len, maddr, mlen; 2022 2023 /* Memory can only be accessed in naturally aligned 4 byte units */ 2024 if (addr & 3 || len & 3 || len == 0) 2025 return (EINVAL); 2026 2027 /* Enabled memories */ 2028 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 2029 if (em & F_EDRAM0_ENABLE) { 2030 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 2031 maddr = G_EDRAM0_BASE(addr_len) << 20; 2032 mlen = G_EDRAM0_SIZE(addr_len) << 20; 2033 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 2034 addr + len <= maddr + mlen) 2035 return (0); 2036 } 2037 if (em & F_EDRAM1_ENABLE) { 2038 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 2039 maddr = G_EDRAM1_BASE(addr_len) << 20; 2040 mlen = G_EDRAM1_SIZE(addr_len) << 20; 2041 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 2042 addr + len <= maddr + mlen) 2043 return (0); 2044 } 2045 if (em & F_EXT_MEM_ENABLE) { 2046 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 2047 maddr = G_EXT_MEM_BASE(addr_len) << 20; 2048 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 2049 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 2050 addr + len <= maddr + mlen) 2051 return (0); 2052 } 2053 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 2054 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 2055 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 2056 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 2057 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 2058 addr + len <= maddr + mlen) 2059 return (0); 2060 } 2061 2062 return (EFAULT); 2063 } 2064 2065 static int 2066 fwmtype_to_hwmtype(int mtype) 2067 { 2068 2069 switch (mtype) { 2070 case FW_MEMTYPE_EDC0: 2071 return (MEM_EDC0); 2072 case FW_MEMTYPE_EDC1: 2073 return (MEM_EDC1); 2074 case FW_MEMTYPE_EXTMEM: 2075 return (MEM_MC0); 2076 case FW_MEMTYPE_EXTMEM1: 2077 return (MEM_MC1); 2078 default: 2079 panic("%s: cannot translate fw mtype %d.", __func__, mtype); 2080 } 2081 } 2082 2083 /* 2084 * Verify that the memory range specified by the memtype/offset/len pair is 2085 * valid and lies entirely within the memtype specified. The global address of 2086 * the start of the range is returned in addr. 2087 */ 2088 static int 2089 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 2090 uint32_t *addr) 2091 { 2092 uint32_t em, addr_len, maddr, mlen; 2093 2094 /* Memory can only be accessed in naturally aligned 4 byte units */ 2095 if (off & 3 || len & 3 || len == 0) 2096 return (EINVAL); 2097 2098 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 2099 switch (fwmtype_to_hwmtype(mtype)) { 2100 case MEM_EDC0: 2101 if (!(em & F_EDRAM0_ENABLE)) 2102 return (EINVAL); 2103 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 2104 maddr = G_EDRAM0_BASE(addr_len) << 20; 2105 mlen = G_EDRAM0_SIZE(addr_len) << 20; 2106 break; 2107 case MEM_EDC1: 2108 if (!(em & F_EDRAM1_ENABLE)) 2109 return (EINVAL); 2110 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 2111 maddr = G_EDRAM1_BASE(addr_len) << 20; 2112 mlen = G_EDRAM1_SIZE(addr_len) << 20; 2113 break; 2114 case MEM_MC: 2115 if (!(em & F_EXT_MEM_ENABLE)) 2116 return (EINVAL); 2117 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 2118 maddr = G_EXT_MEM_BASE(addr_len) << 20; 2119 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 2120 break; 2121 case MEM_MC1: 2122 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 2123 return (EINVAL); 2124 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 2125 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 2126 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 2127 break; 2128 default: 2129 return (EINVAL); 2130 } 2131 2132 if (mlen > 0 && off < mlen && off + len <= mlen) { 2133 *addr = maddr + off; /* global address */ 2134 return (0); 2135 } 2136 2137 return (EFAULT); 2138 } 2139 2140 static void 2141 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 2142 { 2143 const struct memwin *mw; 2144 2145 if (is_t4(sc)) { 2146 KASSERT(win >= 0 && win < nitems(t4_memwin), 2147 ("%s: incorrect memwin# (%d)", __func__, win)); 2148 mw = &t4_memwin[win]; 2149 } else { 2150 KASSERT(win >= 0 && win < nitems(t5_memwin), 2151 ("%s: incorrect memwin# (%d)", __func__, win)); 2152 mw = &t5_memwin[win]; 2153 } 2154 2155 if (base != NULL) 2156 *base = mw->base; 2157 if (aperture != NULL) 2158 *aperture = mw->aperture; 2159 } 2160 2161 /* 2162 * Positions the memory window such that it can be used to access the specified 2163 * address in the chip's address space. The return value is the offset of addr 2164 * from the start of the window. 2165 */ 2166 static uint32_t 2167 position_memwin(struct adapter *sc, int n, uint32_t addr) 2168 { 2169 uint32_t start, pf; 2170 uint32_t reg; 2171 2172 KASSERT(n >= 0 && n <= 3, 2173 ("%s: invalid window %d.", __func__, n)); 2174 KASSERT((addr & 3) == 0, 2175 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 2176 2177 if (is_t4(sc)) { 2178 pf = 0; 2179 start = addr & ~0xf; /* start must be 16B aligned */ 2180 } else { 2181 pf = V_PFNUM(sc->pf); 2182 start = addr & ~0x7f; /* start must be 128B aligned */ 2183 } 2184 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 2185 2186 t4_write_reg(sc, reg, start | pf); 2187 t4_read_reg(sc, reg); 2188 2189 return (addr - start); 2190 } 2191 2192 static int 2193 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, int num_vis, 2194 struct intrs_and_queues *iaq) 2195 { 2196 int rc, itype, navail, nrxq10g, nrxq1g, n; 2197 int nofldrxq10g = 0, nofldrxq1g = 0; 2198 int nnmrxq10g = 0, nnmrxq1g = 0; 2199 2200 bzero(iaq, sizeof(*iaq)); 2201 2202 iaq->ntxq10g = t4_ntxq10g; 2203 iaq->ntxq1g = t4_ntxq1g; 2204 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 2205 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 2206 iaq->rsrv_noflowq = t4_rsrv_noflowq; 2207 #ifdef TCP_OFFLOAD 2208 if (is_offload(sc)) { 2209 iaq->nofldtxq10g = t4_nofldtxq10g; 2210 iaq->nofldtxq1g = t4_nofldtxq1g; 2211 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 2212 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 2213 } 2214 #endif 2215 #ifdef DEV_NETMAP 2216 iaq->nnmtxq10g = t4_nnmtxq10g; 2217 iaq->nnmtxq1g = t4_nnmtxq1g; 2218 iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g; 2219 iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g; 2220 #endif 2221 2222 for (itype = INTR_MSIX; itype; itype >>= 1) { 2223 2224 if ((itype & t4_intr_types) == 0) 2225 continue; /* not allowed */ 2226 2227 if (itype == INTR_MSIX) 2228 navail = pci_msix_count(sc->dev); 2229 else if (itype == INTR_MSI) 2230 navail = pci_msi_count(sc->dev); 2231 else 2232 navail = 1; 2233 restart: 2234 if (navail == 0) 2235 continue; 2236 2237 iaq->intr_type = itype; 2238 iaq->intr_flags_10g = 0; 2239 iaq->intr_flags_1g = 0; 2240 2241 /* 2242 * Best option: an interrupt vector for errors, one for the 2243 * firmware event queue, and one for every rxq (NIC, TOE, and 2244 * netmap). 2245 */ 2246 iaq->nirq = T4_EXTRA_INTR; 2247 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g); 2248 iaq->nirq += n10g * 2 * (num_vis - 1); 2249 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g); 2250 iaq->nirq += n1g * 2 * (num_vis - 1); 2251 if (iaq->nirq <= navail && 2252 (itype != INTR_MSI || powerof2(iaq->nirq))) { 2253 iaq->intr_flags_10g = INTR_ALL; 2254 iaq->intr_flags_1g = INTR_ALL; 2255 goto allocate; 2256 } 2257 2258 /* 2259 * Second best option: a vector for errors, one for the firmware 2260 * event queue, and vectors for either all the NIC rx queues or 2261 * all the TOE rx queues. The queues that don't get vectors 2262 * will forward their interrupts to those that do. 2263 * 2264 * Note: netmap rx queues cannot be created early and so they 2265 * can't be setup to receive forwarded interrupts for others. 2266 */ 2267 iaq->nirq = T4_EXTRA_INTR; 2268 if (nrxq10g >= nofldrxq10g) { 2269 iaq->intr_flags_10g = INTR_RXQ; 2270 iaq->nirq += n10g * nrxq10g; 2271 iaq->nirq += n10g * (num_vis - 1); 2272 #ifdef DEV_NETMAP 2273 iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g); 2274 #endif 2275 } else { 2276 iaq->intr_flags_10g = INTR_OFLD_RXQ; 2277 iaq->nirq += n10g * nofldrxq10g; 2278 #ifdef DEV_NETMAP 2279 iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g); 2280 #endif 2281 } 2282 if (nrxq1g >= nofldrxq1g) { 2283 iaq->intr_flags_1g = INTR_RXQ; 2284 iaq->nirq += n1g * nrxq1g; 2285 iaq->nirq += n1g * (num_vis - 1); 2286 #ifdef DEV_NETMAP 2287 iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g); 2288 #endif 2289 } else { 2290 iaq->intr_flags_1g = INTR_OFLD_RXQ; 2291 iaq->nirq += n1g * nofldrxq1g; 2292 #ifdef DEV_NETMAP 2293 iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g); 2294 #endif 2295 } 2296 if (iaq->nirq <= navail && 2297 (itype != INTR_MSI || powerof2(iaq->nirq))) 2298 goto allocate; 2299 2300 /* 2301 * Next best option: an interrupt vector for errors, one for the 2302 * firmware event queue, and at least one per VI. At this 2303 * point we know we'll have to downsize nrxq and/or nofldrxq 2304 * and/or nnmrxq to fit what's available to us. 2305 */ 2306 iaq->nirq = T4_EXTRA_INTR; 2307 iaq->nirq += (n10g + n1g) * num_vis; 2308 if (iaq->nirq <= navail) { 2309 int leftover = navail - iaq->nirq; 2310 2311 if (n10g > 0) { 2312 int target = max(nrxq10g, nofldrxq10g); 2313 2314 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ? 2315 INTR_RXQ : INTR_OFLD_RXQ; 2316 2317 n = 1; 2318 while (n < target && leftover >= n10g) { 2319 leftover -= n10g; 2320 iaq->nirq += n10g; 2321 n++; 2322 } 2323 iaq->nrxq10g = min(n, nrxq10g); 2324 #ifdef TCP_OFFLOAD 2325 iaq->nofldrxq10g = min(n, nofldrxq10g); 2326 #endif 2327 #ifdef DEV_NETMAP 2328 iaq->nnmrxq10g = min(n, nnmrxq10g); 2329 #endif 2330 } 2331 2332 if (n1g > 0) { 2333 int target = max(nrxq1g, nofldrxq1g); 2334 2335 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ? 2336 INTR_RXQ : INTR_OFLD_RXQ; 2337 2338 n = 1; 2339 while (n < target && leftover >= n1g) { 2340 leftover -= n1g; 2341 iaq->nirq += n1g; 2342 n++; 2343 } 2344 iaq->nrxq1g = min(n, nrxq1g); 2345 #ifdef TCP_OFFLOAD 2346 iaq->nofldrxq1g = min(n, nofldrxq1g); 2347 #endif 2348 #ifdef DEV_NETMAP 2349 iaq->nnmrxq1g = min(n, nnmrxq1g); 2350 #endif 2351 } 2352 2353 if (itype != INTR_MSI || powerof2(iaq->nirq)) 2354 goto allocate; 2355 } 2356 2357 /* 2358 * Least desirable option: one interrupt vector for everything. 2359 */ 2360 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 2361 iaq->intr_flags_10g = iaq->intr_flags_1g = 0; 2362 #ifdef TCP_OFFLOAD 2363 if (is_offload(sc)) 2364 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 2365 #endif 2366 #ifdef DEV_NETMAP 2367 iaq->nnmrxq10g = iaq->nnmrxq1g = 1; 2368 #endif 2369 2370 allocate: 2371 navail = iaq->nirq; 2372 rc = 0; 2373 if (itype == INTR_MSIX) 2374 rc = pci_alloc_msix(sc->dev, &navail); 2375 else if (itype == INTR_MSI) 2376 rc = pci_alloc_msi(sc->dev, &navail); 2377 2378 if (rc == 0) { 2379 if (navail == iaq->nirq) 2380 return (0); 2381 2382 /* 2383 * Didn't get the number requested. Use whatever number 2384 * the kernel is willing to allocate (it's in navail). 2385 */ 2386 device_printf(sc->dev, "fewer vectors than requested, " 2387 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 2388 itype, iaq->nirq, navail); 2389 pci_release_msi(sc->dev); 2390 goto restart; 2391 } 2392 2393 device_printf(sc->dev, 2394 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 2395 itype, rc, iaq->nirq, navail); 2396 } 2397 2398 device_printf(sc->dev, 2399 "failed to find a usable interrupt type. " 2400 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 2401 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 2402 2403 return (ENXIO); 2404 } 2405 2406 #define FW_VERSION(chip) ( \ 2407 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ 2408 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ 2409 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ 2410 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) 2411 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) 2412 2413 struct fw_info { 2414 uint8_t chip; 2415 char *kld_name; 2416 char *fw_mod_name; 2417 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 2418 } fw_info[] = { 2419 { 2420 .chip = CHELSIO_T4, 2421 .kld_name = "t4fw_cfg", 2422 .fw_mod_name = "t4fw", 2423 .fw_hdr = { 2424 .chip = FW_HDR_CHIP_T4, 2425 .fw_ver = htobe32_const(FW_VERSION(T4)), 2426 .intfver_nic = FW_INTFVER(T4, NIC), 2427 .intfver_vnic = FW_INTFVER(T4, VNIC), 2428 .intfver_ofld = FW_INTFVER(T4, OFLD), 2429 .intfver_ri = FW_INTFVER(T4, RI), 2430 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 2431 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 2432 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 2433 .intfver_fcoe = FW_INTFVER(T4, FCOE), 2434 }, 2435 }, { 2436 .chip = CHELSIO_T5, 2437 .kld_name = "t5fw_cfg", 2438 .fw_mod_name = "t5fw", 2439 .fw_hdr = { 2440 .chip = FW_HDR_CHIP_T5, 2441 .fw_ver = htobe32_const(FW_VERSION(T5)), 2442 .intfver_nic = FW_INTFVER(T5, NIC), 2443 .intfver_vnic = FW_INTFVER(T5, VNIC), 2444 .intfver_ofld = FW_INTFVER(T5, OFLD), 2445 .intfver_ri = FW_INTFVER(T5, RI), 2446 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 2447 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 2448 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 2449 .intfver_fcoe = FW_INTFVER(T5, FCOE), 2450 }, 2451 } 2452 }; 2453 2454 static struct fw_info * 2455 find_fw_info(int chip) 2456 { 2457 int i; 2458 2459 for (i = 0; i < nitems(fw_info); i++) { 2460 if (fw_info[i].chip == chip) 2461 return (&fw_info[i]); 2462 } 2463 return (NULL); 2464 } 2465 2466 /* 2467 * Is the given firmware API compatible with the one the driver was compiled 2468 * with? 2469 */ 2470 static int 2471 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 2472 { 2473 2474 /* short circuit if it's the exact same firmware version */ 2475 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 2476 return (1); 2477 2478 /* 2479 * XXX: Is this too conservative? Perhaps I should limit this to the 2480 * features that are supported in the driver. 2481 */ 2482 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 2483 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 2484 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 2485 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 2486 return (1); 2487 #undef SAME_INTF 2488 2489 return (0); 2490 } 2491 2492 /* 2493 * The firmware in the KLD is usable, but should it be installed? This routine 2494 * explains itself in detail if it indicates the KLD firmware should be 2495 * installed. 2496 */ 2497 static int 2498 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 2499 { 2500 const char *reason; 2501 2502 if (!card_fw_usable) { 2503 reason = "incompatible or unusable"; 2504 goto install; 2505 } 2506 2507 if (k > c) { 2508 reason = "older than the version bundled with this driver"; 2509 goto install; 2510 } 2511 2512 if (t4_fw_install == 2 && k != c) { 2513 reason = "different than the version bundled with this driver"; 2514 goto install; 2515 } 2516 2517 return (0); 2518 2519 install: 2520 if (t4_fw_install == 0) { 2521 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2522 "but the driver is prohibited from installing a different " 2523 "firmware on the card.\n", 2524 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2525 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 2526 2527 return (0); 2528 } 2529 2530 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2531 "installing firmware %u.%u.%u.%u on card.\n", 2532 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2533 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 2534 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2535 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2536 2537 return (1); 2538 } 2539 /* 2540 * Establish contact with the firmware and determine if we are the master driver 2541 * or not, and whether we are responsible for chip initialization. 2542 */ 2543 static int 2544 prep_firmware(struct adapter *sc) 2545 { 2546 const struct firmware *fw = NULL, *default_cfg; 2547 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 2548 enum dev_state state; 2549 struct fw_info *fw_info; 2550 struct fw_hdr *card_fw; /* fw on the card */ 2551 const struct fw_hdr *kld_fw; /* fw in the KLD */ 2552 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 2553 against */ 2554 2555 /* Contact firmware. */ 2556 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 2557 if (rc < 0 || state == DEV_STATE_ERR) { 2558 rc = -rc; 2559 device_printf(sc->dev, 2560 "failed to connect to the firmware: %d, %d.\n", rc, state); 2561 return (rc); 2562 } 2563 pf = rc; 2564 if (pf == sc->mbox) 2565 sc->flags |= MASTER_PF; 2566 else if (state == DEV_STATE_UNINIT) { 2567 /* 2568 * We didn't get to be the master so we definitely won't be 2569 * configuring the chip. It's a bug if someone else hasn't 2570 * configured it already. 2571 */ 2572 device_printf(sc->dev, "couldn't be master(%d), " 2573 "device not already initialized either(%d).\n", rc, state); 2574 return (EDOOFUS); 2575 } 2576 2577 /* This is the firmware whose headers the driver was compiled against */ 2578 fw_info = find_fw_info(chip_id(sc)); 2579 if (fw_info == NULL) { 2580 device_printf(sc->dev, 2581 "unable to look up firmware information for chip %d.\n", 2582 chip_id(sc)); 2583 return (EINVAL); 2584 } 2585 drv_fw = &fw_info->fw_hdr; 2586 2587 /* 2588 * The firmware KLD contains many modules. The KLD name is also the 2589 * name of the module that contains the default config file. 2590 */ 2591 default_cfg = firmware_get(fw_info->kld_name); 2592 2593 /* Read the header of the firmware on the card */ 2594 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 2595 rc = -t4_read_flash(sc, FLASH_FW_START, 2596 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 2597 if (rc == 0) 2598 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 2599 else { 2600 device_printf(sc->dev, 2601 "Unable to read card's firmware header: %d\n", rc); 2602 card_fw_usable = 0; 2603 } 2604 2605 /* This is the firmware in the KLD */ 2606 fw = firmware_get(fw_info->fw_mod_name); 2607 if (fw != NULL) { 2608 kld_fw = (const void *)fw->data; 2609 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 2610 } else { 2611 kld_fw = NULL; 2612 kld_fw_usable = 0; 2613 } 2614 2615 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2616 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) { 2617 /* 2618 * Common case: the firmware on the card is an exact match and 2619 * the KLD is an exact match too, or the KLD is 2620 * absent/incompatible. Note that t4_fw_install = 2 is ignored 2621 * here -- use cxgbetool loadfw if you want to reinstall the 2622 * same firmware as the one on the card. 2623 */ 2624 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2625 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2626 be32toh(card_fw->fw_ver))) { 2627 2628 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 2629 if (rc != 0) { 2630 device_printf(sc->dev, 2631 "failed to install firmware: %d\n", rc); 2632 goto done; 2633 } 2634 2635 /* Installed successfully, update the cached header too. */ 2636 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2637 card_fw_usable = 1; 2638 need_fw_reset = 0; /* already reset as part of load_fw */ 2639 } 2640 2641 if (!card_fw_usable) { 2642 uint32_t d, c, k; 2643 2644 d = ntohl(drv_fw->fw_ver); 2645 c = ntohl(card_fw->fw_ver); 2646 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2647 2648 device_printf(sc->dev, "Cannot find a usable firmware: " 2649 "fw_install %d, chip state %d, " 2650 "driver compiled with %d.%d.%d.%d, " 2651 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2652 t4_fw_install, state, 2653 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2654 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2655 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2656 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2657 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2658 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2659 rc = EINVAL; 2660 goto done; 2661 } 2662 2663 /* We're using whatever's on the card and it's known to be good. */ 2664 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2665 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2666 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2667 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2668 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2669 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2670 t4_get_tp_version(sc, &sc->params.tp_vers); 2671 2672 /* Reset device */ 2673 if (need_fw_reset && 2674 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2675 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2676 if (rc != ETIMEDOUT && rc != EIO) 2677 t4_fw_bye(sc, sc->mbox); 2678 goto done; 2679 } 2680 sc->flags |= FW_OK; 2681 2682 rc = get_params__pre_init(sc); 2683 if (rc != 0) 2684 goto done; /* error message displayed already */ 2685 2686 /* Partition adapter resources as specified in the config file. */ 2687 if (state == DEV_STATE_UNINIT) { 2688 2689 KASSERT(sc->flags & MASTER_PF, 2690 ("%s: trying to change chip settings when not master.", 2691 __func__)); 2692 2693 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2694 if (rc != 0) 2695 goto done; /* error message displayed already */ 2696 2697 t4_tweak_chip_settings(sc); 2698 2699 /* get basic stuff going */ 2700 rc = -t4_fw_initialize(sc, sc->mbox); 2701 if (rc != 0) { 2702 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2703 goto done; 2704 } 2705 } else { 2706 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2707 sc->cfcsum = 0; 2708 } 2709 2710 done: 2711 free(card_fw, M_CXGBE); 2712 if (fw != NULL) 2713 firmware_put(fw, FIRMWARE_UNLOAD); 2714 if (default_cfg != NULL) 2715 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2716 2717 return (rc); 2718 } 2719 2720 #define FW_PARAM_DEV(param) \ 2721 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2722 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2723 #define FW_PARAM_PFVF(param) \ 2724 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2725 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2726 2727 /* 2728 * Partition chip resources for use between various PFs, VFs, etc. 2729 */ 2730 static int 2731 partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2732 const char *name_prefix) 2733 { 2734 const struct firmware *cfg = NULL; 2735 int rc = 0; 2736 struct fw_caps_config_cmd caps; 2737 uint32_t mtype, moff, finicsum, cfcsum; 2738 2739 /* 2740 * Figure out what configuration file to use. Pick the default config 2741 * file for the card if the user hasn't specified one explicitly. 2742 */ 2743 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2744 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2745 /* Card specific overrides go here. */ 2746 if (pci_get_device(sc->dev) == 0x440a) 2747 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2748 if (is_fpga(sc)) 2749 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2750 } 2751 2752 /* 2753 * We need to load another module if the profile is anything except 2754 * "default" or "flash". 2755 */ 2756 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2757 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2758 char s[32]; 2759 2760 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2761 cfg = firmware_get(s); 2762 if (cfg == NULL) { 2763 if (default_cfg != NULL) { 2764 device_printf(sc->dev, 2765 "unable to load module \"%s\" for " 2766 "configuration profile \"%s\", will use " 2767 "the default config file instead.\n", 2768 s, sc->cfg_file); 2769 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2770 "%s", DEFAULT_CF); 2771 } else { 2772 device_printf(sc->dev, 2773 "unable to load module \"%s\" for " 2774 "configuration profile \"%s\", will use " 2775 "the config file on the card's flash " 2776 "instead.\n", s, sc->cfg_file); 2777 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2778 "%s", FLASH_CF); 2779 } 2780 } 2781 } 2782 2783 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2784 default_cfg == NULL) { 2785 device_printf(sc->dev, 2786 "default config file not available, will use the config " 2787 "file on the card's flash instead.\n"); 2788 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2789 } 2790 2791 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2792 u_int cflen, i, n; 2793 const uint32_t *cfdata; 2794 uint32_t param, val, addr, off, mw_base, mw_aperture; 2795 2796 KASSERT(cfg != NULL || default_cfg != NULL, 2797 ("%s: no config to upload", __func__)); 2798 2799 /* 2800 * Ask the firmware where it wants us to upload the config file. 2801 */ 2802 param = FW_PARAM_DEV(CF); 2803 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2804 if (rc != 0) { 2805 /* No support for config file? Shouldn't happen. */ 2806 device_printf(sc->dev, 2807 "failed to query config file location: %d.\n", rc); 2808 goto done; 2809 } 2810 mtype = G_FW_PARAMS_PARAM_Y(val); 2811 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2812 2813 /* 2814 * XXX: sheer laziness. We deliberately added 4 bytes of 2815 * useless stuffing/comments at the end of the config file so 2816 * it's ok to simply throw away the last remaining bytes when 2817 * the config file is not an exact multiple of 4. This also 2818 * helps with the validate_mt_off_len check. 2819 */ 2820 if (cfg != NULL) { 2821 cflen = cfg->datasize & ~3; 2822 cfdata = cfg->data; 2823 } else { 2824 cflen = default_cfg->datasize & ~3; 2825 cfdata = default_cfg->data; 2826 } 2827 2828 if (cflen > FLASH_CFG_MAX_SIZE) { 2829 device_printf(sc->dev, 2830 "config file too long (%d, max allowed is %d). " 2831 "Will try to use the config on the card, if any.\n", 2832 cflen, FLASH_CFG_MAX_SIZE); 2833 goto use_config_on_flash; 2834 } 2835 2836 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2837 if (rc != 0) { 2838 device_printf(sc->dev, 2839 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2840 "Will try to use the config on the card, if any.\n", 2841 __func__, mtype, moff, cflen, rc); 2842 goto use_config_on_flash; 2843 } 2844 2845 memwin_info(sc, 2, &mw_base, &mw_aperture); 2846 while (cflen) { 2847 off = position_memwin(sc, 2, addr); 2848 n = min(cflen, mw_aperture - off); 2849 for (i = 0; i < n; i += 4) 2850 t4_write_reg(sc, mw_base + off + i, *cfdata++); 2851 cflen -= n; 2852 addr += n; 2853 } 2854 } else { 2855 use_config_on_flash: 2856 mtype = FW_MEMTYPE_FLASH; 2857 moff = t4_flash_cfg_addr(sc); 2858 } 2859 2860 bzero(&caps, sizeof(caps)); 2861 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2862 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2863 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2864 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2865 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2866 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2867 if (rc != 0) { 2868 device_printf(sc->dev, 2869 "failed to pre-process config file: %d " 2870 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2871 goto done; 2872 } 2873 2874 finicsum = be32toh(caps.finicsum); 2875 cfcsum = be32toh(caps.cfcsum); 2876 if (finicsum != cfcsum) { 2877 device_printf(sc->dev, 2878 "WARNING: config file checksum mismatch: %08x %08x\n", 2879 finicsum, cfcsum); 2880 } 2881 sc->cfcsum = cfcsum; 2882 2883 #define LIMIT_CAPS(x) do { \ 2884 caps.x &= htobe16(t4_##x##_allowed); \ 2885 } while (0) 2886 2887 /* 2888 * Let the firmware know what features will (not) be used so it can tune 2889 * things accordingly. 2890 */ 2891 LIMIT_CAPS(linkcaps); 2892 LIMIT_CAPS(niccaps); 2893 LIMIT_CAPS(toecaps); 2894 LIMIT_CAPS(rdmacaps); 2895 LIMIT_CAPS(iscsicaps); 2896 LIMIT_CAPS(fcoecaps); 2897 #undef LIMIT_CAPS 2898 2899 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2900 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2901 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2902 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2903 if (rc != 0) { 2904 device_printf(sc->dev, 2905 "failed to process config file: %d.\n", rc); 2906 } 2907 done: 2908 if (cfg != NULL) 2909 firmware_put(cfg, FIRMWARE_UNLOAD); 2910 return (rc); 2911 } 2912 2913 /* 2914 * Retrieve parameters that are needed (or nice to have) very early. 2915 */ 2916 static int 2917 get_params__pre_init(struct adapter *sc) 2918 { 2919 int rc; 2920 uint32_t param[2], val[2]; 2921 struct fw_devlog_cmd cmd; 2922 struct devlog_params *dlog = &sc->params.devlog; 2923 2924 param[0] = FW_PARAM_DEV(PORTVEC); 2925 param[1] = FW_PARAM_DEV(CCLK); 2926 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2927 if (rc != 0) { 2928 device_printf(sc->dev, 2929 "failed to query parameters (pre_init): %d.\n", rc); 2930 return (rc); 2931 } 2932 2933 sc->params.portvec = val[0]; 2934 sc->params.nports = bitcount32(val[0]); 2935 sc->params.vpd.cclk = val[1]; 2936 2937 /* Read device log parameters. */ 2938 bzero(&cmd, sizeof(cmd)); 2939 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2940 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2941 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2942 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2943 if (rc != 0) { 2944 device_printf(sc->dev, 2945 "failed to get devlog parameters: %d.\n", rc); 2946 bzero(dlog, sizeof (*dlog)); 2947 rc = 0; /* devlog isn't critical for device operation */ 2948 } else { 2949 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2950 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2951 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2952 dlog->size = be32toh(cmd.memsize_devlog); 2953 } 2954 2955 return (rc); 2956 } 2957 2958 /* 2959 * Retrieve various parameters that are of interest to the driver. The device 2960 * has been initialized by the firmware at this point. 2961 */ 2962 static int 2963 get_params__post_init(struct adapter *sc) 2964 { 2965 int rc; 2966 uint32_t param[7], val[7]; 2967 struct fw_caps_config_cmd caps; 2968 2969 param[0] = FW_PARAM_PFVF(IQFLINT_START); 2970 param[1] = FW_PARAM_PFVF(EQ_START); 2971 param[2] = FW_PARAM_PFVF(FILTER_START); 2972 param[3] = FW_PARAM_PFVF(FILTER_END); 2973 param[4] = FW_PARAM_PFVF(L2T_START); 2974 param[5] = FW_PARAM_PFVF(L2T_END); 2975 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2976 if (rc != 0) { 2977 device_printf(sc->dev, 2978 "failed to query parameters (post_init): %d.\n", rc); 2979 return (rc); 2980 } 2981 2982 sc->sge.iq_start = val[0]; 2983 sc->sge.eq_start = val[1]; 2984 sc->tids.ftid_base = val[2]; 2985 sc->tids.nftids = val[3] - val[2] + 1; 2986 sc->params.ftid_min = val[2]; 2987 sc->params.ftid_max = val[3]; 2988 sc->vres.l2t.start = val[4]; 2989 sc->vres.l2t.size = val[5] - val[4] + 1; 2990 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2991 ("%s: L2 table size (%u) larger than expected (%u)", 2992 __func__, sc->vres.l2t.size, L2T_SIZE)); 2993 2994 /* get capabilites */ 2995 bzero(&caps, sizeof(caps)); 2996 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2997 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2998 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2999 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 3000 if (rc != 0) { 3001 device_printf(sc->dev, 3002 "failed to get card capabilities: %d.\n", rc); 3003 return (rc); 3004 } 3005 3006 #define READ_CAPS(x) do { \ 3007 sc->x = htobe16(caps.x); \ 3008 } while (0) 3009 READ_CAPS(linkcaps); 3010 READ_CAPS(niccaps); 3011 READ_CAPS(toecaps); 3012 READ_CAPS(rdmacaps); 3013 READ_CAPS(iscsicaps); 3014 READ_CAPS(fcoecaps); 3015 3016 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { 3017 param[0] = FW_PARAM_PFVF(ETHOFLD_START); 3018 param[1] = FW_PARAM_PFVF(ETHOFLD_END); 3019 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 3020 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); 3021 if (rc != 0) { 3022 device_printf(sc->dev, 3023 "failed to query NIC parameters: %d.\n", rc); 3024 return (rc); 3025 } 3026 sc->tids.etid_base = val[0]; 3027 sc->params.etid_min = val[0]; 3028 sc->tids.netids = val[1] - val[0] + 1; 3029 sc->params.netids = sc->tids.netids; 3030 sc->params.eo_wr_cred = val[2]; 3031 sc->params.ethoffload = 1; 3032 } 3033 3034 if (sc->toecaps) { 3035 /* query offload-related parameters */ 3036 param[0] = FW_PARAM_DEV(NTID); 3037 param[1] = FW_PARAM_PFVF(SERVER_START); 3038 param[2] = FW_PARAM_PFVF(SERVER_END); 3039 param[3] = FW_PARAM_PFVF(TDDP_START); 3040 param[4] = FW_PARAM_PFVF(TDDP_END); 3041 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 3042 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 3043 if (rc != 0) { 3044 device_printf(sc->dev, 3045 "failed to query TOE parameters: %d.\n", rc); 3046 return (rc); 3047 } 3048 sc->tids.ntids = val[0]; 3049 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 3050 sc->tids.stid_base = val[1]; 3051 sc->tids.nstids = val[2] - val[1] + 1; 3052 sc->vres.ddp.start = val[3]; 3053 sc->vres.ddp.size = val[4] - val[3] + 1; 3054 sc->params.ofldq_wr_cred = val[5]; 3055 sc->params.offload = 1; 3056 } 3057 if (sc->rdmacaps) { 3058 param[0] = FW_PARAM_PFVF(STAG_START); 3059 param[1] = FW_PARAM_PFVF(STAG_END); 3060 param[2] = FW_PARAM_PFVF(RQ_START); 3061 param[3] = FW_PARAM_PFVF(RQ_END); 3062 param[4] = FW_PARAM_PFVF(PBL_START); 3063 param[5] = FW_PARAM_PFVF(PBL_END); 3064 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 3065 if (rc != 0) { 3066 device_printf(sc->dev, 3067 "failed to query RDMA parameters(1): %d.\n", rc); 3068 return (rc); 3069 } 3070 sc->vres.stag.start = val[0]; 3071 sc->vres.stag.size = val[1] - val[0] + 1; 3072 sc->vres.rq.start = val[2]; 3073 sc->vres.rq.size = val[3] - val[2] + 1; 3074 sc->vres.pbl.start = val[4]; 3075 sc->vres.pbl.size = val[5] - val[4] + 1; 3076 3077 param[0] = FW_PARAM_PFVF(SQRQ_START); 3078 param[1] = FW_PARAM_PFVF(SQRQ_END); 3079 param[2] = FW_PARAM_PFVF(CQ_START); 3080 param[3] = FW_PARAM_PFVF(CQ_END); 3081 param[4] = FW_PARAM_PFVF(OCQ_START); 3082 param[5] = FW_PARAM_PFVF(OCQ_END); 3083 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 3084 if (rc != 0) { 3085 device_printf(sc->dev, 3086 "failed to query RDMA parameters(2): %d.\n", rc); 3087 return (rc); 3088 } 3089 sc->vres.qp.start = val[0]; 3090 sc->vres.qp.size = val[1] - val[0] + 1; 3091 sc->vres.cq.start = val[2]; 3092 sc->vres.cq.size = val[3] - val[2] + 1; 3093 sc->vres.ocq.start = val[4]; 3094 sc->vres.ocq.size = val[5] - val[4] + 1; 3095 } 3096 if (sc->iscsicaps) { 3097 param[0] = FW_PARAM_PFVF(ISCSI_START); 3098 param[1] = FW_PARAM_PFVF(ISCSI_END); 3099 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 3100 if (rc != 0) { 3101 device_printf(sc->dev, 3102 "failed to query iSCSI parameters: %d.\n", rc); 3103 return (rc); 3104 } 3105 sc->vres.iscsi.start = val[0]; 3106 sc->vres.iscsi.size = val[1] - val[0] + 1; 3107 } 3108 3109 /* 3110 * We've got the params we wanted to query via the firmware. Now grab 3111 * some others directly from the chip. 3112 */ 3113 rc = t4_read_chip_settings(sc); 3114 3115 return (rc); 3116 } 3117 3118 static int 3119 set_params__post_init(struct adapter *sc) 3120 { 3121 uint32_t param, val; 3122 3123 /* ask for encapsulated CPLs */ 3124 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 3125 val = 1; 3126 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 3127 3128 return (0); 3129 } 3130 3131 #undef FW_PARAM_PFVF 3132 #undef FW_PARAM_DEV 3133 3134 static void 3135 t4_set_desc(struct adapter *sc) 3136 { 3137 char buf[128]; 3138 struct adapter_params *p = &sc->params; 3139 3140 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, " 3141 "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "", 3142 chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec); 3143 3144 device_set_desc_copy(sc->dev, buf); 3145 } 3146 3147 static void 3148 build_medialist(struct port_info *pi, struct ifmedia *media) 3149 { 3150 int m; 3151 3152 PORT_LOCK(pi); 3153 3154 ifmedia_removeall(media); 3155 3156 m = IFM_ETHER | IFM_FDX; 3157 3158 switch(pi->port_type) { 3159 case FW_PORT_TYPE_BT_XFI: 3160 case FW_PORT_TYPE_BT_XAUI: 3161 ifmedia_add(media, m | IFM_10G_T, 0, NULL); 3162 /* fall through */ 3163 3164 case FW_PORT_TYPE_BT_SGMII: 3165 ifmedia_add(media, m | IFM_1000_T, 0, NULL); 3166 ifmedia_add(media, m | IFM_100_TX, 0, NULL); 3167 ifmedia_add(media, IFM_ETHER | IFM_AUTO, 0, NULL); 3168 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 3169 break; 3170 3171 case FW_PORT_TYPE_CX4: 3172 ifmedia_add(media, m | IFM_10G_CX4, 0, NULL); 3173 ifmedia_set(media, m | IFM_10G_CX4); 3174 break; 3175 3176 case FW_PORT_TYPE_QSFP_10G: 3177 case FW_PORT_TYPE_SFP: 3178 case FW_PORT_TYPE_FIBER_XFI: 3179 case FW_PORT_TYPE_FIBER_XAUI: 3180 switch (pi->mod_type) { 3181 3182 case FW_PORT_MOD_TYPE_LR: 3183 ifmedia_add(media, m | IFM_10G_LR, 0, NULL); 3184 ifmedia_set(media, m | IFM_10G_LR); 3185 break; 3186 3187 case FW_PORT_MOD_TYPE_SR: 3188 ifmedia_add(media, m | IFM_10G_SR, 0, NULL); 3189 ifmedia_set(media, m | IFM_10G_SR); 3190 break; 3191 3192 case FW_PORT_MOD_TYPE_LRM: 3193 ifmedia_add(media, m | IFM_10G_LRM, 0, NULL); 3194 ifmedia_set(media, m | IFM_10G_LRM); 3195 break; 3196 3197 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 3198 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 3199 ifmedia_add(media, m | IFM_10G_TWINAX, 0, NULL); 3200 ifmedia_set(media, m | IFM_10G_TWINAX); 3201 break; 3202 3203 case FW_PORT_MOD_TYPE_NONE: 3204 m &= ~IFM_FDX; 3205 ifmedia_add(media, m | IFM_NONE, 0, NULL); 3206 ifmedia_set(media, m | IFM_NONE); 3207 break; 3208 3209 case FW_PORT_MOD_TYPE_NA: 3210 case FW_PORT_MOD_TYPE_ER: 3211 default: 3212 device_printf(pi->dev, 3213 "unknown port_type (%d), mod_type (%d)\n", 3214 pi->port_type, pi->mod_type); 3215 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 3216 ifmedia_set(media, m | IFM_UNKNOWN); 3217 break; 3218 } 3219 break; 3220 3221 case FW_PORT_TYPE_QSFP: 3222 switch (pi->mod_type) { 3223 3224 case FW_PORT_MOD_TYPE_LR: 3225 ifmedia_add(media, m | IFM_40G_LR4, 0, NULL); 3226 ifmedia_set(media, m | IFM_40G_LR4); 3227 break; 3228 3229 case FW_PORT_MOD_TYPE_SR: 3230 ifmedia_add(media, m | IFM_40G_SR4, 0, NULL); 3231 ifmedia_set(media, m | IFM_40G_SR4); 3232 break; 3233 3234 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 3235 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 3236 ifmedia_add(media, m | IFM_40G_CR4, 0, NULL); 3237 ifmedia_set(media, m | IFM_40G_CR4); 3238 break; 3239 3240 case FW_PORT_MOD_TYPE_NONE: 3241 m &= ~IFM_FDX; 3242 ifmedia_add(media, m | IFM_NONE, 0, NULL); 3243 ifmedia_set(media, m | IFM_NONE); 3244 break; 3245 3246 default: 3247 device_printf(pi->dev, 3248 "unknown port_type (%d), mod_type (%d)\n", 3249 pi->port_type, pi->mod_type); 3250 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 3251 ifmedia_set(media, m | IFM_UNKNOWN); 3252 break; 3253 } 3254 break; 3255 3256 default: 3257 device_printf(pi->dev, 3258 "unknown port_type (%d), mod_type (%d)\n", pi->port_type, 3259 pi->mod_type); 3260 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 3261 ifmedia_set(media, m | IFM_UNKNOWN); 3262 break; 3263 } 3264 3265 PORT_UNLOCK(pi); 3266 } 3267 3268 #define FW_MAC_EXACT_CHUNK 7 3269 3270 /* 3271 * Program the port's XGMAC based on parameters in ifnet. The caller also 3272 * indicates which parameters should be programmed (the rest are left alone). 3273 */ 3274 int 3275 update_mac_settings(struct ifnet *ifp, int flags) 3276 { 3277 int rc = 0; 3278 struct vi_info *vi = ifp->if_softc; 3279 struct port_info *pi = vi->pi; 3280 struct adapter *sc = pi->adapter; 3281 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 3282 3283 ASSERT_SYNCHRONIZED_OP(sc); 3284 KASSERT(flags, ("%s: not told what to update.", __func__)); 3285 3286 if (flags & XGMAC_MTU) 3287 mtu = ifp->if_mtu; 3288 3289 if (flags & XGMAC_PROMISC) 3290 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 3291 3292 if (flags & XGMAC_ALLMULTI) 3293 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 3294 3295 if (flags & XGMAC_VLANEX) 3296 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 3297 3298 if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) { 3299 rc = -t4_set_rxmode(sc, sc->mbox, vi->viid, mtu, promisc, 3300 allmulti, 1, vlanex, false); 3301 if (rc) { 3302 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, 3303 rc); 3304 return (rc); 3305 } 3306 } 3307 3308 if (flags & XGMAC_UCADDR) { 3309 uint8_t ucaddr[ETHER_ADDR_LEN]; 3310 3311 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 3312 rc = t4_change_mac(sc, sc->mbox, vi->viid, vi->xact_addr_filt, 3313 ucaddr, true, true); 3314 if (rc < 0) { 3315 rc = -rc; 3316 if_printf(ifp, "change_mac failed: %d\n", rc); 3317 return (rc); 3318 } else { 3319 vi->xact_addr_filt = rc; 3320 rc = 0; 3321 } 3322 } 3323 3324 if (flags & XGMAC_MCADDRS) { 3325 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 3326 int del = 1; 3327 uint64_t hash = 0; 3328 struct ifmultiaddr *ifma; 3329 int i = 0, j; 3330 3331 if_maddr_rlock(ifp); 3332 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3333 if (ifma->ifma_addr->sa_family != AF_LINK) 3334 continue; 3335 mcaddr[i] = 3336 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 3337 MPASS(ETHER_IS_MULTICAST(mcaddr[i])); 3338 i++; 3339 3340 if (i == FW_MAC_EXACT_CHUNK) { 3341 rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid, 3342 del, i, mcaddr, NULL, &hash, 0); 3343 if (rc < 0) { 3344 rc = -rc; 3345 for (j = 0; j < i; j++) { 3346 if_printf(ifp, 3347 "failed to add mc address" 3348 " %02x:%02x:%02x:" 3349 "%02x:%02x:%02x rc=%d\n", 3350 mcaddr[j][0], mcaddr[j][1], 3351 mcaddr[j][2], mcaddr[j][3], 3352 mcaddr[j][4], mcaddr[j][5], 3353 rc); 3354 } 3355 goto mcfail; 3356 } 3357 del = 0; 3358 i = 0; 3359 } 3360 } 3361 if (i > 0) { 3362 rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid, del, i, 3363 mcaddr, NULL, &hash, 0); 3364 if (rc < 0) { 3365 rc = -rc; 3366 for (j = 0; j < i; j++) { 3367 if_printf(ifp, 3368 "failed to add mc address" 3369 " %02x:%02x:%02x:" 3370 "%02x:%02x:%02x rc=%d\n", 3371 mcaddr[j][0], mcaddr[j][1], 3372 mcaddr[j][2], mcaddr[j][3], 3373 mcaddr[j][4], mcaddr[j][5], 3374 rc); 3375 } 3376 goto mcfail; 3377 } 3378 } 3379 3380 rc = -t4_set_addr_hash(sc, sc->mbox, vi->viid, 0, hash, 0); 3381 if (rc != 0) 3382 if_printf(ifp, "failed to set mc address hash: %d", rc); 3383 mcfail: 3384 if_maddr_runlock(ifp); 3385 } 3386 3387 return (rc); 3388 } 3389 3390 /* 3391 * {begin|end}_synchronized_op must be called from the same thread. 3392 */ 3393 int 3394 begin_synchronized_op(struct adapter *sc, struct vi_info *vi, int flags, 3395 char *wmesg) 3396 { 3397 int rc, pri; 3398 3399 #ifdef WITNESS 3400 /* the caller thinks it's ok to sleep, but is it really? */ 3401 if (flags & SLEEP_OK) 3402 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 3403 "begin_synchronized_op"); 3404 #endif 3405 3406 if (INTR_OK) 3407 pri = PCATCH; 3408 else 3409 pri = 0; 3410 3411 ADAPTER_LOCK(sc); 3412 for (;;) { 3413 3414 if (vi && IS_DOOMED(vi)) { 3415 rc = ENXIO; 3416 goto done; 3417 } 3418 3419 if (!IS_BUSY(sc)) { 3420 rc = 0; 3421 break; 3422 } 3423 3424 if (!(flags & SLEEP_OK)) { 3425 rc = EBUSY; 3426 goto done; 3427 } 3428 3429 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 3430 rc = EINTR; 3431 goto done; 3432 } 3433 } 3434 3435 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 3436 SET_BUSY(sc); 3437 #ifdef INVARIANTS 3438 sc->last_op = wmesg; 3439 sc->last_op_thr = curthread; 3440 sc->last_op_flags = flags; 3441 #endif 3442 3443 done: 3444 if (!(flags & HOLD_LOCK) || rc) 3445 ADAPTER_UNLOCK(sc); 3446 3447 return (rc); 3448 } 3449 3450 /* 3451 * Tell if_ioctl and if_init that the VI is going away. This is 3452 * special variant of begin_synchronized_op and must be paired with a 3453 * call to end_synchronized_op. 3454 */ 3455 void 3456 doom_vi(struct adapter *sc, struct vi_info *vi) 3457 { 3458 3459 ADAPTER_LOCK(sc); 3460 SET_DOOMED(vi); 3461 wakeup(&sc->flags); 3462 while (IS_BUSY(sc)) 3463 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 3464 SET_BUSY(sc); 3465 #ifdef INVARIANTS 3466 sc->last_op = "t4detach"; 3467 sc->last_op_thr = curthread; 3468 sc->last_op_flags = 0; 3469 #endif 3470 ADAPTER_UNLOCK(sc); 3471 } 3472 3473 /* 3474 * {begin|end}_synchronized_op must be called from the same thread. 3475 */ 3476 void 3477 end_synchronized_op(struct adapter *sc, int flags) 3478 { 3479 3480 if (flags & LOCK_HELD) 3481 ADAPTER_LOCK_ASSERT_OWNED(sc); 3482 else 3483 ADAPTER_LOCK(sc); 3484 3485 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 3486 CLR_BUSY(sc); 3487 wakeup(&sc->flags); 3488 ADAPTER_UNLOCK(sc); 3489 } 3490 3491 static int 3492 cxgbe_init_synchronized(struct vi_info *vi) 3493 { 3494 struct port_info *pi = vi->pi; 3495 struct adapter *sc = pi->adapter; 3496 struct ifnet *ifp = vi->ifp; 3497 int rc = 0, i; 3498 struct sge_txq *txq; 3499 3500 ASSERT_SYNCHRONIZED_OP(sc); 3501 3502 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 3503 return (0); /* already running */ 3504 3505 if (!(sc->flags & FULL_INIT_DONE) && 3506 ((rc = adapter_full_init(sc)) != 0)) 3507 return (rc); /* error message displayed already */ 3508 3509 if (!(vi->flags & VI_INIT_DONE) && 3510 ((rc = vi_full_init(vi)) != 0)) 3511 return (rc); /* error message displayed already */ 3512 3513 rc = update_mac_settings(ifp, XGMAC_ALL); 3514 if (rc) 3515 goto done; /* error message displayed already */ 3516 3517 rc = -t4_enable_vi(sc, sc->mbox, vi->viid, true, true); 3518 if (rc != 0) { 3519 if_printf(ifp, "enable_vi failed: %d\n", rc); 3520 goto done; 3521 } 3522 3523 /* 3524 * Can't fail from this point onwards. Review cxgbe_uninit_synchronized 3525 * if this changes. 3526 */ 3527 3528 for_each_txq(vi, i, txq) { 3529 TXQ_LOCK(txq); 3530 txq->eq.flags |= EQ_ENABLED; 3531 TXQ_UNLOCK(txq); 3532 } 3533 3534 /* 3535 * The first iq of the first port to come up is used for tracing. 3536 */ 3537 if (sc->traceq < 0 && IS_MAIN_VI(vi)) { 3538 sc->traceq = sc->sge.rxq[vi->first_rxq].iq.abs_id; 3539 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : 3540 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | 3541 V_QUEUENUMBER(sc->traceq)); 3542 pi->flags |= HAS_TRACEQ; 3543 } 3544 3545 /* all ok */ 3546 PORT_LOCK(pi); 3547 ifp->if_drv_flags |= IFF_DRV_RUNNING; 3548 pi->up_vis++; 3549 3550 if (pi->nvi > 1) 3551 callout_reset(&vi->tick, hz, vi_tick, vi); 3552 else 3553 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 3554 PORT_UNLOCK(pi); 3555 done: 3556 if (rc != 0) 3557 cxgbe_uninit_synchronized(vi); 3558 3559 return (rc); 3560 } 3561 3562 /* 3563 * Idempotent. 3564 */ 3565 static int 3566 cxgbe_uninit_synchronized(struct vi_info *vi) 3567 { 3568 struct port_info *pi = vi->pi; 3569 struct adapter *sc = pi->adapter; 3570 struct ifnet *ifp = vi->ifp; 3571 int rc, i; 3572 struct sge_txq *txq; 3573 3574 ASSERT_SYNCHRONIZED_OP(sc); 3575 3576 if (!(vi->flags & VI_INIT_DONE)) { 3577 KASSERT(!(ifp->if_drv_flags & IFF_DRV_RUNNING), 3578 ("uninited VI is running")); 3579 return (0); 3580 } 3581 3582 /* 3583 * Disable the VI so that all its data in either direction is discarded 3584 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 3585 * tick) intact as the TP can deliver negative advice or data that it's 3586 * holding in its RAM (for an offloaded connection) even after the VI is 3587 * disabled. 3588 */ 3589 rc = -t4_enable_vi(sc, sc->mbox, vi->viid, false, false); 3590 if (rc) { 3591 if_printf(ifp, "disable_vi failed: %d\n", rc); 3592 return (rc); 3593 } 3594 3595 for_each_txq(vi, i, txq) { 3596 TXQ_LOCK(txq); 3597 txq->eq.flags &= ~EQ_ENABLED; 3598 TXQ_UNLOCK(txq); 3599 } 3600 3601 PORT_LOCK(pi); 3602 if (pi->nvi == 1) 3603 callout_stop(&pi->tick); 3604 else 3605 callout_stop(&vi->tick); 3606 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 3607 PORT_UNLOCK(pi); 3608 return (0); 3609 } 3610 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3611 pi->up_vis--; 3612 if (pi->up_vis > 0) { 3613 PORT_UNLOCK(pi); 3614 return (0); 3615 } 3616 PORT_UNLOCK(pi); 3617 3618 pi->link_cfg.link_ok = 0; 3619 pi->link_cfg.speed = 0; 3620 pi->linkdnrc = -1; 3621 t4_os_link_changed(sc, pi->port_id, 0, -1); 3622 3623 return (0); 3624 } 3625 3626 /* 3627 * It is ok for this function to fail midway and return right away. t4_detach 3628 * will walk the entire sc->irq list and clean up whatever is valid. 3629 */ 3630 static int 3631 setup_intr_handlers(struct adapter *sc) 3632 { 3633 int rc, rid, p, q, v; 3634 char s[8]; 3635 struct irq *irq; 3636 struct port_info *pi; 3637 struct vi_info *vi; 3638 struct sge_rxq *rxq; 3639 #ifdef TCP_OFFLOAD 3640 struct sge_ofld_rxq *ofld_rxq; 3641 #endif 3642 #ifdef DEV_NETMAP 3643 struct sge_nm_rxq *nm_rxq; 3644 #endif 3645 #ifdef RSS 3646 int nbuckets = rss_getnumbuckets(); 3647 #endif 3648 3649 /* 3650 * Setup interrupts. 3651 */ 3652 irq = &sc->irq[0]; 3653 rid = sc->intr_type == INTR_INTX ? 0 : 1; 3654 if (sc->intr_count == 1) 3655 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all")); 3656 3657 /* Multiple interrupts. */ 3658 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 3659 ("%s: too few intr.", __func__)); 3660 3661 /* The first one is always error intr */ 3662 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 3663 if (rc != 0) 3664 return (rc); 3665 irq++; 3666 rid++; 3667 3668 /* The second one is always the firmware event queue */ 3669 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt"); 3670 if (rc != 0) 3671 return (rc); 3672 irq++; 3673 rid++; 3674 3675 for_each_port(sc, p) { 3676 pi = sc->port[p]; 3677 for_each_vi(pi, v, vi) { 3678 vi->first_intr = rid - 1; 3679 #ifdef DEV_NETMAP 3680 if (vi->flags & VI_NETMAP) { 3681 for_each_nm_rxq(vi, q, nm_rxq) { 3682 snprintf(s, sizeof(s), "%d-%d", p, q); 3683 rc = t4_alloc_irq(sc, irq, rid, 3684 t4_nm_intr, nm_rxq, s); 3685 if (rc != 0) 3686 return (rc); 3687 irq++; 3688 rid++; 3689 vi->nintr++; 3690 } 3691 continue; 3692 } 3693 #endif 3694 if (vi->flags & INTR_RXQ) { 3695 for_each_rxq(vi, q, rxq) { 3696 if (v == 0) 3697 snprintf(s, sizeof(s), "%d.%d", 3698 p, q); 3699 else 3700 snprintf(s, sizeof(s), 3701 "%d(%d).%d", p, v, q); 3702 rc = t4_alloc_irq(sc, irq, rid, 3703 t4_intr, rxq, s); 3704 if (rc != 0) 3705 return (rc); 3706 #ifdef RSS 3707 bus_bind_intr(sc->dev, irq->res, 3708 rss_getcpu(q % nbuckets)); 3709 #endif 3710 irq++; 3711 rid++; 3712 vi->nintr++; 3713 } 3714 } 3715 #ifdef TCP_OFFLOAD 3716 if (vi->flags & INTR_OFLD_RXQ) { 3717 for_each_ofld_rxq(vi, q, ofld_rxq) { 3718 snprintf(s, sizeof(s), "%d,%d", p, q); 3719 rc = t4_alloc_irq(sc, irq, rid, 3720 t4_intr, ofld_rxq, s); 3721 if (rc != 0) 3722 return (rc); 3723 irq++; 3724 rid++; 3725 vi->nintr++; 3726 } 3727 } 3728 #endif 3729 } 3730 } 3731 MPASS(irq == &sc->irq[sc->intr_count]); 3732 3733 return (0); 3734 } 3735 3736 int 3737 adapter_full_init(struct adapter *sc) 3738 { 3739 int rc, i; 3740 3741 ASSERT_SYNCHRONIZED_OP(sc); 3742 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3743 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 3744 ("%s: FULL_INIT_DONE already", __func__)); 3745 3746 /* 3747 * queues that belong to the adapter (not any particular port). 3748 */ 3749 rc = t4_setup_adapter_queues(sc); 3750 if (rc != 0) 3751 goto done; 3752 3753 for (i = 0; i < nitems(sc->tq); i++) { 3754 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 3755 taskqueue_thread_enqueue, &sc->tq[i]); 3756 if (sc->tq[i] == NULL) { 3757 device_printf(sc->dev, 3758 "failed to allocate task queue %d\n", i); 3759 rc = ENOMEM; 3760 goto done; 3761 } 3762 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3763 device_get_nameunit(sc->dev), i); 3764 } 3765 3766 t4_intr_enable(sc); 3767 sc->flags |= FULL_INIT_DONE; 3768 done: 3769 if (rc != 0) 3770 adapter_full_uninit(sc); 3771 3772 return (rc); 3773 } 3774 3775 int 3776 adapter_full_uninit(struct adapter *sc) 3777 { 3778 int i; 3779 3780 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3781 3782 t4_teardown_adapter_queues(sc); 3783 3784 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3785 taskqueue_free(sc->tq[i]); 3786 sc->tq[i] = NULL; 3787 } 3788 3789 sc->flags &= ~FULL_INIT_DONE; 3790 3791 return (0); 3792 } 3793 3794 #ifdef RSS 3795 #define SUPPORTED_RSS_HASHTYPES (RSS_HASHTYPE_RSS_IPV4 | \ 3796 RSS_HASHTYPE_RSS_TCP_IPV4 | RSS_HASHTYPE_RSS_IPV6 | \ 3797 RSS_HASHTYPE_RSS_TCP_IPV6 | RSS_HASHTYPE_RSS_UDP_IPV4 | \ 3798 RSS_HASHTYPE_RSS_UDP_IPV6) 3799 3800 /* Translates kernel hash types to hardware. */ 3801 static int 3802 hashconfig_to_hashen(int hashconfig) 3803 { 3804 int hashen = 0; 3805 3806 if (hashconfig & RSS_HASHTYPE_RSS_IPV4) 3807 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN; 3808 if (hashconfig & RSS_HASHTYPE_RSS_IPV6) 3809 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN; 3810 if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV4) { 3811 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN | 3812 F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN; 3813 } 3814 if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV6) { 3815 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN | 3816 F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN; 3817 } 3818 if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV4) 3819 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN; 3820 if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV6) 3821 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN; 3822 3823 return (hashen); 3824 } 3825 3826 /* Translates hardware hash types to kernel. */ 3827 static int 3828 hashen_to_hashconfig(int hashen) 3829 { 3830 int hashconfig = 0; 3831 3832 if (hashen & F_FW_RSS_VI_CONFIG_CMD_UDPEN) { 3833 /* 3834 * If UDP hashing was enabled it must have been enabled for 3835 * either IPv4 or IPv6 (inclusive or). Enabling UDP without 3836 * enabling any 4-tuple hash is nonsense configuration. 3837 */ 3838 MPASS(hashen & (F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN | 3839 F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)); 3840 3841 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) 3842 hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV4; 3843 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) 3844 hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV6; 3845 } 3846 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) 3847 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV4; 3848 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) 3849 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV6; 3850 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN) 3851 hashconfig |= RSS_HASHTYPE_RSS_IPV4; 3852 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN) 3853 hashconfig |= RSS_HASHTYPE_RSS_IPV6; 3854 3855 return (hashconfig); 3856 } 3857 #endif 3858 3859 int 3860 vi_full_init(struct vi_info *vi) 3861 { 3862 struct adapter *sc = vi->pi->adapter; 3863 struct ifnet *ifp = vi->ifp; 3864 uint16_t *rss; 3865 struct sge_rxq *rxq; 3866 int rc, i, j, hashen; 3867 #ifdef RSS 3868 int nbuckets = rss_getnumbuckets(); 3869 int hashconfig = rss_gethashconfig(); 3870 int extra; 3871 uint32_t raw_rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; 3872 uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; 3873 #endif 3874 3875 ASSERT_SYNCHRONIZED_OP(sc); 3876 KASSERT((vi->flags & VI_INIT_DONE) == 0, 3877 ("%s: VI_INIT_DONE already", __func__)); 3878 3879 sysctl_ctx_init(&vi->ctx); 3880 vi->flags |= VI_SYSCTL_CTX; 3881 3882 /* 3883 * Allocate tx/rx/fl queues for this VI. 3884 */ 3885 rc = t4_setup_vi_queues(vi); 3886 if (rc != 0) 3887 goto done; /* error message displayed already */ 3888 3889 #ifdef DEV_NETMAP 3890 /* Netmap VIs configure RSS when netmap is enabled. */ 3891 if (vi->flags & VI_NETMAP) { 3892 vi->flags |= VI_INIT_DONE; 3893 return (0); 3894 } 3895 #endif 3896 3897 /* 3898 * Setup RSS for this VI. Save a copy of the RSS table for later use. 3899 */ 3900 if (vi->nrxq > vi->rss_size) { 3901 if_printf(ifp, "nrxq (%d) > hw RSS table size (%d); " 3902 "some queues will never receive traffic.\n", vi->nrxq, 3903 vi->rss_size); 3904 } else if (vi->rss_size % vi->nrxq) { 3905 if_printf(ifp, "nrxq (%d), hw RSS table size (%d); " 3906 "expect uneven traffic distribution.\n", vi->nrxq, 3907 vi->rss_size); 3908 } 3909 #ifdef RSS 3910 MPASS(RSS_KEYSIZE == 40); 3911 if (vi->nrxq != nbuckets) { 3912 if_printf(ifp, "nrxq (%d) != kernel RSS buckets (%d);" 3913 "performance will be impacted.\n", vi->nrxq, nbuckets); 3914 } 3915 3916 rss_getkey((void *)&raw_rss_key[0]); 3917 for (i = 0; i < nitems(rss_key); i++) { 3918 rss_key[i] = htobe32(raw_rss_key[nitems(rss_key) - 1 - i]); 3919 } 3920 t4_write_rss_key(sc, (void *)&rss_key[0], -1); 3921 #endif 3922 rss = malloc(vi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); 3923 for (i = 0; i < vi->rss_size;) { 3924 #ifdef RSS 3925 j = rss_get_indirection_to_bucket(i); 3926 j %= vi->nrxq; 3927 rxq = &sc->sge.rxq[vi->first_rxq + j]; 3928 rss[i++] = rxq->iq.abs_id; 3929 #else 3930 for_each_rxq(vi, j, rxq) { 3931 rss[i++] = rxq->iq.abs_id; 3932 if (i == vi->rss_size) 3933 break; 3934 } 3935 #endif 3936 } 3937 3938 rc = -t4_config_rss_range(sc, sc->mbox, vi->viid, 0, vi->rss_size, rss, 3939 vi->rss_size); 3940 if (rc != 0) { 3941 if_printf(ifp, "rss_config failed: %d\n", rc); 3942 goto done; 3943 } 3944 3945 #ifdef RSS 3946 hashen = hashconfig_to_hashen(hashconfig); 3947 3948 /* 3949 * We may have had to enable some hashes even though the global config 3950 * wants them disabled. This is a potential problem that must be 3951 * reported to the user. 3952 */ 3953 extra = hashen_to_hashconfig(hashen) ^ hashconfig; 3954 3955 /* 3956 * If we consider only the supported hash types, then the enabled hashes 3957 * are a superset of the requested hashes. In other words, there cannot 3958 * be any supported hash that was requested but not enabled, but there 3959 * can be hashes that were not requested but had to be enabled. 3960 */ 3961 extra &= SUPPORTED_RSS_HASHTYPES; 3962 MPASS((extra & hashconfig) == 0); 3963 3964 if (extra) { 3965 if_printf(ifp, 3966 "global RSS config (0x%x) cannot be accomodated.\n", 3967 hashconfig); 3968 } 3969 if (extra & RSS_HASHTYPE_RSS_IPV4) 3970 if_printf(ifp, "IPv4 2-tuple hashing forced on.\n"); 3971 if (extra & RSS_HASHTYPE_RSS_TCP_IPV4) 3972 if_printf(ifp, "TCP/IPv4 4-tuple hashing forced on.\n"); 3973 if (extra & RSS_HASHTYPE_RSS_IPV6) 3974 if_printf(ifp, "IPv6 2-tuple hashing forced on.\n"); 3975 if (extra & RSS_HASHTYPE_RSS_TCP_IPV6) 3976 if_printf(ifp, "TCP/IPv6 4-tuple hashing forced on.\n"); 3977 if (extra & RSS_HASHTYPE_RSS_UDP_IPV4) 3978 if_printf(ifp, "UDP/IPv4 4-tuple hashing forced on.\n"); 3979 if (extra & RSS_HASHTYPE_RSS_UDP_IPV6) 3980 if_printf(ifp, "UDP/IPv6 4-tuple hashing forced on.\n"); 3981 #else 3982 hashen = F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN | 3983 F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN | 3984 F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN | 3985 F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN | F_FW_RSS_VI_CONFIG_CMD_UDPEN; 3986 #endif 3987 rc = -t4_config_vi_rss(sc, sc->mbox, vi->viid, hashen, rss[0]); 3988 if (rc != 0) { 3989 if_printf(ifp, "rss hash/defaultq config failed: %d\n", rc); 3990 goto done; 3991 } 3992 3993 vi->rss = rss; 3994 vi->flags |= VI_INIT_DONE; 3995 done: 3996 if (rc != 0) 3997 vi_full_uninit(vi); 3998 3999 return (rc); 4000 } 4001 4002 /* 4003 * Idempotent. 4004 */ 4005 int 4006 vi_full_uninit(struct vi_info *vi) 4007 { 4008 struct port_info *pi = vi->pi; 4009 struct adapter *sc = pi->adapter; 4010 int i; 4011 struct sge_rxq *rxq; 4012 struct sge_txq *txq; 4013 #ifdef TCP_OFFLOAD 4014 struct sge_ofld_rxq *ofld_rxq; 4015 struct sge_wrq *ofld_txq; 4016 #endif 4017 4018 if (vi->flags & VI_INIT_DONE) { 4019 4020 /* Need to quiesce queues. */ 4021 #ifdef DEV_NETMAP 4022 if (vi->flags & VI_NETMAP) 4023 goto skip; 4024 #endif 4025 4026 /* XXX: Only for the first VI? */ 4027 if (IS_MAIN_VI(vi)) 4028 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]); 4029 4030 for_each_txq(vi, i, txq) { 4031 quiesce_txq(sc, txq); 4032 } 4033 4034 #ifdef TCP_OFFLOAD 4035 for_each_ofld_txq(vi, i, ofld_txq) { 4036 quiesce_wrq(sc, ofld_txq); 4037 } 4038 #endif 4039 4040 for_each_rxq(vi, i, rxq) { 4041 quiesce_iq(sc, &rxq->iq); 4042 quiesce_fl(sc, &rxq->fl); 4043 } 4044 4045 #ifdef TCP_OFFLOAD 4046 for_each_ofld_rxq(vi, i, ofld_rxq) { 4047 quiesce_iq(sc, &ofld_rxq->iq); 4048 quiesce_fl(sc, &ofld_rxq->fl); 4049 } 4050 #endif 4051 free(vi->rss, M_CXGBE); 4052 } 4053 #ifdef DEV_NETMAP 4054 skip: 4055 #endif 4056 4057 t4_teardown_vi_queues(vi); 4058 vi->flags &= ~VI_INIT_DONE; 4059 4060 return (0); 4061 } 4062 4063 static void 4064 quiesce_txq(struct adapter *sc, struct sge_txq *txq) 4065 { 4066 struct sge_eq *eq = &txq->eq; 4067 struct sge_qstat *spg = (void *)&eq->desc[eq->sidx]; 4068 4069 (void) sc; /* unused */ 4070 4071 #ifdef INVARIANTS 4072 TXQ_LOCK(txq); 4073 MPASS((eq->flags & EQ_ENABLED) == 0); 4074 TXQ_UNLOCK(txq); 4075 #endif 4076 4077 /* Wait for the mp_ring to empty. */ 4078 while (!mp_ring_is_idle(txq->r)) { 4079 mp_ring_check_drainage(txq->r, 0); 4080 pause("rquiesce", 1); 4081 } 4082 4083 /* Then wait for the hardware to finish. */ 4084 while (spg->cidx != htobe16(eq->pidx)) 4085 pause("equiesce", 1); 4086 4087 /* Finally, wait for the driver to reclaim all descriptors. */ 4088 while (eq->cidx != eq->pidx) 4089 pause("dquiesce", 1); 4090 } 4091 4092 static void 4093 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq) 4094 { 4095 4096 /* XXXTX */ 4097 } 4098 4099 static void 4100 quiesce_iq(struct adapter *sc, struct sge_iq *iq) 4101 { 4102 (void) sc; /* unused */ 4103 4104 /* Synchronize with the interrupt handler */ 4105 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 4106 pause("iqfree", 1); 4107 } 4108 4109 static void 4110 quiesce_fl(struct adapter *sc, struct sge_fl *fl) 4111 { 4112 mtx_lock(&sc->sfl_lock); 4113 FL_LOCK(fl); 4114 fl->flags |= FL_DOOMED; 4115 FL_UNLOCK(fl); 4116 callout_stop(&sc->sfl_callout); 4117 mtx_unlock(&sc->sfl_lock); 4118 4119 KASSERT((fl->flags & FL_STARVING) == 0, 4120 ("%s: still starving", __func__)); 4121 } 4122 4123 static int 4124 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 4125 driver_intr_t *handler, void *arg, char *name) 4126 { 4127 int rc; 4128 4129 irq->rid = rid; 4130 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 4131 RF_SHAREABLE | RF_ACTIVE); 4132 if (irq->res == NULL) { 4133 device_printf(sc->dev, 4134 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 4135 return (ENOMEM); 4136 } 4137 4138 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 4139 NULL, handler, arg, &irq->tag); 4140 if (rc != 0) { 4141 device_printf(sc->dev, 4142 "failed to setup interrupt for rid %d, name %s: %d\n", 4143 rid, name, rc); 4144 } else if (name) 4145 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 4146 4147 return (rc); 4148 } 4149 4150 static int 4151 t4_free_irq(struct adapter *sc, struct irq *irq) 4152 { 4153 if (irq->tag) 4154 bus_teardown_intr(sc->dev, irq->res, irq->tag); 4155 if (irq->res) 4156 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 4157 4158 bzero(irq, sizeof(*irq)); 4159 4160 return (0); 4161 } 4162 4163 static void 4164 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 4165 unsigned int end) 4166 { 4167 uint32_t *p = (uint32_t *)(buf + start); 4168 4169 for ( ; start <= end; start += sizeof(uint32_t)) 4170 *p++ = t4_read_reg(sc, start); 4171 } 4172 4173 static void 4174 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 4175 { 4176 int i, n; 4177 const unsigned int *reg_ranges; 4178 static const unsigned int t4_reg_ranges[] = { 4179 0x1008, 0x1108, 4180 0x1180, 0x11b4, 4181 0x11fc, 0x123c, 4182 0x1300, 0x173c, 4183 0x1800, 0x18fc, 4184 0x3000, 0x30d8, 4185 0x30e0, 0x5924, 4186 0x5960, 0x59d4, 4187 0x5a00, 0x5af8, 4188 0x6000, 0x6098, 4189 0x6100, 0x6150, 4190 0x6200, 0x6208, 4191 0x6240, 0x6248, 4192 0x6280, 0x6338, 4193 0x6370, 0x638c, 4194 0x6400, 0x643c, 4195 0x6500, 0x6524, 4196 0x6a00, 0x6a38, 4197 0x6a60, 0x6a78, 4198 0x6b00, 0x6b84, 4199 0x6bf0, 0x6c84, 4200 0x6cf0, 0x6d84, 4201 0x6df0, 0x6e84, 4202 0x6ef0, 0x6f84, 4203 0x6ff0, 0x7084, 4204 0x70f0, 0x7184, 4205 0x71f0, 0x7284, 4206 0x72f0, 0x7384, 4207 0x73f0, 0x7450, 4208 0x7500, 0x7530, 4209 0x7600, 0x761c, 4210 0x7680, 0x76cc, 4211 0x7700, 0x7798, 4212 0x77c0, 0x77fc, 4213 0x7900, 0x79fc, 4214 0x7b00, 0x7c38, 4215 0x7d00, 0x7efc, 4216 0x8dc0, 0x8e1c, 4217 0x8e30, 0x8e78, 4218 0x8ea0, 0x8f6c, 4219 0x8fc0, 0x9074, 4220 0x90fc, 0x90fc, 4221 0x9400, 0x9458, 4222 0x9600, 0x96bc, 4223 0x9800, 0x9808, 4224 0x9820, 0x983c, 4225 0x9850, 0x9864, 4226 0x9c00, 0x9c6c, 4227 0x9c80, 0x9cec, 4228 0x9d00, 0x9d6c, 4229 0x9d80, 0x9dec, 4230 0x9e00, 0x9e6c, 4231 0x9e80, 0x9eec, 4232 0x9f00, 0x9f6c, 4233 0x9f80, 0x9fec, 4234 0xd004, 0xd03c, 4235 0xdfc0, 0xdfe0, 4236 0xe000, 0xea7c, 4237 0xf000, 0x11110, 4238 0x11118, 0x11190, 4239 0x19040, 0x1906c, 4240 0x19078, 0x19080, 4241 0x1908c, 0x19124, 4242 0x19150, 0x191b0, 4243 0x191d0, 0x191e8, 4244 0x19238, 0x1924c, 4245 0x193f8, 0x19474, 4246 0x19490, 0x194f8, 4247 0x19800, 0x19f30, 4248 0x1a000, 0x1a06c, 4249 0x1a0b0, 0x1a120, 4250 0x1a128, 0x1a138, 4251 0x1a190, 0x1a1c4, 4252 0x1a1fc, 0x1a1fc, 4253 0x1e040, 0x1e04c, 4254 0x1e284, 0x1e28c, 4255 0x1e2c0, 0x1e2c0, 4256 0x1e2e0, 0x1e2e0, 4257 0x1e300, 0x1e384, 4258 0x1e3c0, 0x1e3c8, 4259 0x1e440, 0x1e44c, 4260 0x1e684, 0x1e68c, 4261 0x1e6c0, 0x1e6c0, 4262 0x1e6e0, 0x1e6e0, 4263 0x1e700, 0x1e784, 4264 0x1e7c0, 0x1e7c8, 4265 0x1e840, 0x1e84c, 4266 0x1ea84, 0x1ea8c, 4267 0x1eac0, 0x1eac0, 4268 0x1eae0, 0x1eae0, 4269 0x1eb00, 0x1eb84, 4270 0x1ebc0, 0x1ebc8, 4271 0x1ec40, 0x1ec4c, 4272 0x1ee84, 0x1ee8c, 4273 0x1eec0, 0x1eec0, 4274 0x1eee0, 0x1eee0, 4275 0x1ef00, 0x1ef84, 4276 0x1efc0, 0x1efc8, 4277 0x1f040, 0x1f04c, 4278 0x1f284, 0x1f28c, 4279 0x1f2c0, 0x1f2c0, 4280 0x1f2e0, 0x1f2e0, 4281 0x1f300, 0x1f384, 4282 0x1f3c0, 0x1f3c8, 4283 0x1f440, 0x1f44c, 4284 0x1f684, 0x1f68c, 4285 0x1f6c0, 0x1f6c0, 4286 0x1f6e0, 0x1f6e0, 4287 0x1f700, 0x1f784, 4288 0x1f7c0, 0x1f7c8, 4289 0x1f840, 0x1f84c, 4290 0x1fa84, 0x1fa8c, 4291 0x1fac0, 0x1fac0, 4292 0x1fae0, 0x1fae0, 4293 0x1fb00, 0x1fb84, 4294 0x1fbc0, 0x1fbc8, 4295 0x1fc40, 0x1fc4c, 4296 0x1fe84, 0x1fe8c, 4297 0x1fec0, 0x1fec0, 4298 0x1fee0, 0x1fee0, 4299 0x1ff00, 0x1ff84, 4300 0x1ffc0, 0x1ffc8, 4301 0x20000, 0x2002c, 4302 0x20100, 0x2013c, 4303 0x20190, 0x201c8, 4304 0x20200, 0x20318, 4305 0x20400, 0x20528, 4306 0x20540, 0x20614, 4307 0x21000, 0x21040, 4308 0x2104c, 0x21060, 4309 0x210c0, 0x210ec, 4310 0x21200, 0x21268, 4311 0x21270, 0x21284, 4312 0x212fc, 0x21388, 4313 0x21400, 0x21404, 4314 0x21500, 0x21518, 4315 0x2152c, 0x2153c, 4316 0x21550, 0x21554, 4317 0x21600, 0x21600, 4318 0x21608, 0x21628, 4319 0x21630, 0x2163c, 4320 0x21700, 0x2171c, 4321 0x21780, 0x2178c, 4322 0x21800, 0x21c38, 4323 0x21c80, 0x21d7c, 4324 0x21e00, 0x21e04, 4325 0x22000, 0x2202c, 4326 0x22100, 0x2213c, 4327 0x22190, 0x221c8, 4328 0x22200, 0x22318, 4329 0x22400, 0x22528, 4330 0x22540, 0x22614, 4331 0x23000, 0x23040, 4332 0x2304c, 0x23060, 4333 0x230c0, 0x230ec, 4334 0x23200, 0x23268, 4335 0x23270, 0x23284, 4336 0x232fc, 0x23388, 4337 0x23400, 0x23404, 4338 0x23500, 0x23518, 4339 0x2352c, 0x2353c, 4340 0x23550, 0x23554, 4341 0x23600, 0x23600, 4342 0x23608, 0x23628, 4343 0x23630, 0x2363c, 4344 0x23700, 0x2371c, 4345 0x23780, 0x2378c, 4346 0x23800, 0x23c38, 4347 0x23c80, 0x23d7c, 4348 0x23e00, 0x23e04, 4349 0x24000, 0x2402c, 4350 0x24100, 0x2413c, 4351 0x24190, 0x241c8, 4352 0x24200, 0x24318, 4353 0x24400, 0x24528, 4354 0x24540, 0x24614, 4355 0x25000, 0x25040, 4356 0x2504c, 0x25060, 4357 0x250c0, 0x250ec, 4358 0x25200, 0x25268, 4359 0x25270, 0x25284, 4360 0x252fc, 0x25388, 4361 0x25400, 0x25404, 4362 0x25500, 0x25518, 4363 0x2552c, 0x2553c, 4364 0x25550, 0x25554, 4365 0x25600, 0x25600, 4366 0x25608, 0x25628, 4367 0x25630, 0x2563c, 4368 0x25700, 0x2571c, 4369 0x25780, 0x2578c, 4370 0x25800, 0x25c38, 4371 0x25c80, 0x25d7c, 4372 0x25e00, 0x25e04, 4373 0x26000, 0x2602c, 4374 0x26100, 0x2613c, 4375 0x26190, 0x261c8, 4376 0x26200, 0x26318, 4377 0x26400, 0x26528, 4378 0x26540, 0x26614, 4379 0x27000, 0x27040, 4380 0x2704c, 0x27060, 4381 0x270c0, 0x270ec, 4382 0x27200, 0x27268, 4383 0x27270, 0x27284, 4384 0x272fc, 0x27388, 4385 0x27400, 0x27404, 4386 0x27500, 0x27518, 4387 0x2752c, 0x2753c, 4388 0x27550, 0x27554, 4389 0x27600, 0x27600, 4390 0x27608, 0x27628, 4391 0x27630, 0x2763c, 4392 0x27700, 0x2771c, 4393 0x27780, 0x2778c, 4394 0x27800, 0x27c38, 4395 0x27c80, 0x27d7c, 4396 0x27e00, 0x27e04 4397 }; 4398 static const unsigned int t5_reg_ranges[] = { 4399 0x1008, 0x1148, 4400 0x1180, 0x11b4, 4401 0x11fc, 0x123c, 4402 0x1280, 0x173c, 4403 0x1800, 0x18fc, 4404 0x3000, 0x3028, 4405 0x3060, 0x30d8, 4406 0x30e0, 0x30fc, 4407 0x3140, 0x357c, 4408 0x35a8, 0x35cc, 4409 0x35ec, 0x35ec, 4410 0x3600, 0x5624, 4411 0x56cc, 0x575c, 4412 0x580c, 0x5814, 4413 0x5890, 0x58bc, 4414 0x5940, 0x59dc, 4415 0x59fc, 0x5a18, 4416 0x5a60, 0x5a9c, 4417 0x5b94, 0x5bfc, 4418 0x6000, 0x6040, 4419 0x6058, 0x614c, 4420 0x7700, 0x7798, 4421 0x77c0, 0x78fc, 4422 0x7b00, 0x7c54, 4423 0x7d00, 0x7efc, 4424 0x8dc0, 0x8de0, 4425 0x8df8, 0x8e84, 4426 0x8ea0, 0x8f84, 4427 0x8fc0, 0x90f8, 4428 0x9400, 0x9470, 4429 0x9600, 0x96f4, 4430 0x9800, 0x9808, 4431 0x9820, 0x983c, 4432 0x9850, 0x9864, 4433 0x9c00, 0x9c6c, 4434 0x9c80, 0x9cec, 4435 0x9d00, 0x9d6c, 4436 0x9d80, 0x9dec, 4437 0x9e00, 0x9e6c, 4438 0x9e80, 0x9eec, 4439 0x9f00, 0x9f6c, 4440 0x9f80, 0xa020, 4441 0xd004, 0xd03c, 4442 0xdfc0, 0xdfe0, 4443 0xe000, 0x11088, 4444 0x1109c, 0x11110, 4445 0x11118, 0x1117c, 4446 0x11190, 0x11204, 4447 0x19040, 0x1906c, 4448 0x19078, 0x19080, 4449 0x1908c, 0x19124, 4450 0x19150, 0x191b0, 4451 0x191d0, 0x191e8, 4452 0x19238, 0x19290, 4453 0x193f8, 0x19474, 4454 0x19490, 0x194cc, 4455 0x194f0, 0x194f8, 4456 0x19c00, 0x19c60, 4457 0x19c94, 0x19e10, 4458 0x19e50, 0x19f34, 4459 0x19f40, 0x19f50, 4460 0x19f90, 0x19fe4, 4461 0x1a000, 0x1a06c, 4462 0x1a0b0, 0x1a120, 4463 0x1a128, 0x1a138, 4464 0x1a190, 0x1a1c4, 4465 0x1a1fc, 0x1a1fc, 4466 0x1e008, 0x1e00c, 4467 0x1e040, 0x1e04c, 4468 0x1e284, 0x1e290, 4469 0x1e2c0, 0x1e2c0, 4470 0x1e2e0, 0x1e2e0, 4471 0x1e300, 0x1e384, 4472 0x1e3c0, 0x1e3c8, 4473 0x1e408, 0x1e40c, 4474 0x1e440, 0x1e44c, 4475 0x1e684, 0x1e690, 4476 0x1e6c0, 0x1e6c0, 4477 0x1e6e0, 0x1e6e0, 4478 0x1e700, 0x1e784, 4479 0x1e7c0, 0x1e7c8, 4480 0x1e808, 0x1e80c, 4481 0x1e840, 0x1e84c, 4482 0x1ea84, 0x1ea90, 4483 0x1eac0, 0x1eac0, 4484 0x1eae0, 0x1eae0, 4485 0x1eb00, 0x1eb84, 4486 0x1ebc0, 0x1ebc8, 4487 0x1ec08, 0x1ec0c, 4488 0x1ec40, 0x1ec4c, 4489 0x1ee84, 0x1ee90, 4490 0x1eec0, 0x1eec0, 4491 0x1eee0, 0x1eee0, 4492 0x1ef00, 0x1ef84, 4493 0x1efc0, 0x1efc8, 4494 0x1f008, 0x1f00c, 4495 0x1f040, 0x1f04c, 4496 0x1f284, 0x1f290, 4497 0x1f2c0, 0x1f2c0, 4498 0x1f2e0, 0x1f2e0, 4499 0x1f300, 0x1f384, 4500 0x1f3c0, 0x1f3c8, 4501 0x1f408, 0x1f40c, 4502 0x1f440, 0x1f44c, 4503 0x1f684, 0x1f690, 4504 0x1f6c0, 0x1f6c0, 4505 0x1f6e0, 0x1f6e0, 4506 0x1f700, 0x1f784, 4507 0x1f7c0, 0x1f7c8, 4508 0x1f808, 0x1f80c, 4509 0x1f840, 0x1f84c, 4510 0x1fa84, 0x1fa90, 4511 0x1fac0, 0x1fac0, 4512 0x1fae0, 0x1fae0, 4513 0x1fb00, 0x1fb84, 4514 0x1fbc0, 0x1fbc8, 4515 0x1fc08, 0x1fc0c, 4516 0x1fc40, 0x1fc4c, 4517 0x1fe84, 0x1fe90, 4518 0x1fec0, 0x1fec0, 4519 0x1fee0, 0x1fee0, 4520 0x1ff00, 0x1ff84, 4521 0x1ffc0, 0x1ffc8, 4522 0x30000, 0x30030, 4523 0x30100, 0x30144, 4524 0x30190, 0x301d0, 4525 0x30200, 0x30318, 4526 0x30400, 0x3052c, 4527 0x30540, 0x3061c, 4528 0x30800, 0x30834, 4529 0x308c0, 0x30908, 4530 0x30910, 0x309ac, 4531 0x30a00, 0x30a2c, 4532 0x30a44, 0x30a50, 4533 0x30a74, 0x30c24, 4534 0x30d00, 0x30d00, 4535 0x30d08, 0x30d14, 4536 0x30d1c, 0x30d20, 4537 0x30d3c, 0x30d50, 4538 0x31200, 0x3120c, 4539 0x31220, 0x31220, 4540 0x31240, 0x31240, 4541 0x31600, 0x3160c, 4542 0x31a00, 0x31a1c, 4543 0x31e00, 0x31e20, 4544 0x31e38, 0x31e3c, 4545 0x31e80, 0x31e80, 4546 0x31e88, 0x31ea8, 4547 0x31eb0, 0x31eb4, 4548 0x31ec8, 0x31ed4, 4549 0x31fb8, 0x32004, 4550 0x32200, 0x32200, 4551 0x32208, 0x32240, 4552 0x32248, 0x32280, 4553 0x32288, 0x322c0, 4554 0x322c8, 0x322fc, 4555 0x32600, 0x32630, 4556 0x32a00, 0x32abc, 4557 0x32b00, 0x32b70, 4558 0x33000, 0x33048, 4559 0x33060, 0x3309c, 4560 0x330f0, 0x33148, 4561 0x33160, 0x3319c, 4562 0x331f0, 0x332e4, 4563 0x332f8, 0x333e4, 4564 0x333f8, 0x33448, 4565 0x33460, 0x3349c, 4566 0x334f0, 0x33548, 4567 0x33560, 0x3359c, 4568 0x335f0, 0x336e4, 4569 0x336f8, 0x337e4, 4570 0x337f8, 0x337fc, 4571 0x33814, 0x33814, 4572 0x3382c, 0x3382c, 4573 0x33880, 0x3388c, 4574 0x338e8, 0x338ec, 4575 0x33900, 0x33948, 4576 0x33960, 0x3399c, 4577 0x339f0, 0x33ae4, 4578 0x33af8, 0x33b10, 4579 0x33b28, 0x33b28, 4580 0x33b3c, 0x33b50, 4581 0x33bf0, 0x33c10, 4582 0x33c28, 0x33c28, 4583 0x33c3c, 0x33c50, 4584 0x33cf0, 0x33cfc, 4585 0x34000, 0x34030, 4586 0x34100, 0x34144, 4587 0x34190, 0x341d0, 4588 0x34200, 0x34318, 4589 0x34400, 0x3452c, 4590 0x34540, 0x3461c, 4591 0x34800, 0x34834, 4592 0x348c0, 0x34908, 4593 0x34910, 0x349ac, 4594 0x34a00, 0x34a2c, 4595 0x34a44, 0x34a50, 4596 0x34a74, 0x34c24, 4597 0x34d00, 0x34d00, 4598 0x34d08, 0x34d14, 4599 0x34d1c, 0x34d20, 4600 0x34d3c, 0x34d50, 4601 0x35200, 0x3520c, 4602 0x35220, 0x35220, 4603 0x35240, 0x35240, 4604 0x35600, 0x3560c, 4605 0x35a00, 0x35a1c, 4606 0x35e00, 0x35e20, 4607 0x35e38, 0x35e3c, 4608 0x35e80, 0x35e80, 4609 0x35e88, 0x35ea8, 4610 0x35eb0, 0x35eb4, 4611 0x35ec8, 0x35ed4, 4612 0x35fb8, 0x36004, 4613 0x36200, 0x36200, 4614 0x36208, 0x36240, 4615 0x36248, 0x36280, 4616 0x36288, 0x362c0, 4617 0x362c8, 0x362fc, 4618 0x36600, 0x36630, 4619 0x36a00, 0x36abc, 4620 0x36b00, 0x36b70, 4621 0x37000, 0x37048, 4622 0x37060, 0x3709c, 4623 0x370f0, 0x37148, 4624 0x37160, 0x3719c, 4625 0x371f0, 0x372e4, 4626 0x372f8, 0x373e4, 4627 0x373f8, 0x37448, 4628 0x37460, 0x3749c, 4629 0x374f0, 0x37548, 4630 0x37560, 0x3759c, 4631 0x375f0, 0x376e4, 4632 0x376f8, 0x377e4, 4633 0x377f8, 0x377fc, 4634 0x37814, 0x37814, 4635 0x3782c, 0x3782c, 4636 0x37880, 0x3788c, 4637 0x378e8, 0x378ec, 4638 0x37900, 0x37948, 4639 0x37960, 0x3799c, 4640 0x379f0, 0x37ae4, 4641 0x37af8, 0x37b10, 4642 0x37b28, 0x37b28, 4643 0x37b3c, 0x37b50, 4644 0x37bf0, 0x37c10, 4645 0x37c28, 0x37c28, 4646 0x37c3c, 0x37c50, 4647 0x37cf0, 0x37cfc, 4648 0x38000, 0x38030, 4649 0x38100, 0x38144, 4650 0x38190, 0x381d0, 4651 0x38200, 0x38318, 4652 0x38400, 0x3852c, 4653 0x38540, 0x3861c, 4654 0x38800, 0x38834, 4655 0x388c0, 0x38908, 4656 0x38910, 0x389ac, 4657 0x38a00, 0x38a2c, 4658 0x38a44, 0x38a50, 4659 0x38a74, 0x38c24, 4660 0x38d00, 0x38d00, 4661 0x38d08, 0x38d14, 4662 0x38d1c, 0x38d20, 4663 0x38d3c, 0x38d50, 4664 0x39200, 0x3920c, 4665 0x39220, 0x39220, 4666 0x39240, 0x39240, 4667 0x39600, 0x3960c, 4668 0x39a00, 0x39a1c, 4669 0x39e00, 0x39e20, 4670 0x39e38, 0x39e3c, 4671 0x39e80, 0x39e80, 4672 0x39e88, 0x39ea8, 4673 0x39eb0, 0x39eb4, 4674 0x39ec8, 0x39ed4, 4675 0x39fb8, 0x3a004, 4676 0x3a200, 0x3a200, 4677 0x3a208, 0x3a240, 4678 0x3a248, 0x3a280, 4679 0x3a288, 0x3a2c0, 4680 0x3a2c8, 0x3a2fc, 4681 0x3a600, 0x3a630, 4682 0x3aa00, 0x3aabc, 4683 0x3ab00, 0x3ab70, 4684 0x3b000, 0x3b048, 4685 0x3b060, 0x3b09c, 4686 0x3b0f0, 0x3b148, 4687 0x3b160, 0x3b19c, 4688 0x3b1f0, 0x3b2e4, 4689 0x3b2f8, 0x3b3e4, 4690 0x3b3f8, 0x3b448, 4691 0x3b460, 0x3b49c, 4692 0x3b4f0, 0x3b548, 4693 0x3b560, 0x3b59c, 4694 0x3b5f0, 0x3b6e4, 4695 0x3b6f8, 0x3b7e4, 4696 0x3b7f8, 0x3b7fc, 4697 0x3b814, 0x3b814, 4698 0x3b82c, 0x3b82c, 4699 0x3b880, 0x3b88c, 4700 0x3b8e8, 0x3b8ec, 4701 0x3b900, 0x3b948, 4702 0x3b960, 0x3b99c, 4703 0x3b9f0, 0x3bae4, 4704 0x3baf8, 0x3bb10, 4705 0x3bb28, 0x3bb28, 4706 0x3bb3c, 0x3bb50, 4707 0x3bbf0, 0x3bc10, 4708 0x3bc28, 0x3bc28, 4709 0x3bc3c, 0x3bc50, 4710 0x3bcf0, 0x3bcfc, 4711 0x3c000, 0x3c030, 4712 0x3c100, 0x3c144, 4713 0x3c190, 0x3c1d0, 4714 0x3c200, 0x3c318, 4715 0x3c400, 0x3c52c, 4716 0x3c540, 0x3c61c, 4717 0x3c800, 0x3c834, 4718 0x3c8c0, 0x3c908, 4719 0x3c910, 0x3c9ac, 4720 0x3ca00, 0x3ca2c, 4721 0x3ca44, 0x3ca50, 4722 0x3ca74, 0x3cc24, 4723 0x3cd00, 0x3cd00, 4724 0x3cd08, 0x3cd14, 4725 0x3cd1c, 0x3cd20, 4726 0x3cd3c, 0x3cd50, 4727 0x3d200, 0x3d20c, 4728 0x3d220, 0x3d220, 4729 0x3d240, 0x3d240, 4730 0x3d600, 0x3d60c, 4731 0x3da00, 0x3da1c, 4732 0x3de00, 0x3de20, 4733 0x3de38, 0x3de3c, 4734 0x3de80, 0x3de80, 4735 0x3de88, 0x3dea8, 4736 0x3deb0, 0x3deb4, 4737 0x3dec8, 0x3ded4, 4738 0x3dfb8, 0x3e004, 4739 0x3e200, 0x3e200, 4740 0x3e208, 0x3e240, 4741 0x3e248, 0x3e280, 4742 0x3e288, 0x3e2c0, 4743 0x3e2c8, 0x3e2fc, 4744 0x3e600, 0x3e630, 4745 0x3ea00, 0x3eabc, 4746 0x3eb00, 0x3eb70, 4747 0x3f000, 0x3f048, 4748 0x3f060, 0x3f09c, 4749 0x3f0f0, 0x3f148, 4750 0x3f160, 0x3f19c, 4751 0x3f1f0, 0x3f2e4, 4752 0x3f2f8, 0x3f3e4, 4753 0x3f3f8, 0x3f448, 4754 0x3f460, 0x3f49c, 4755 0x3f4f0, 0x3f548, 4756 0x3f560, 0x3f59c, 4757 0x3f5f0, 0x3f6e4, 4758 0x3f6f8, 0x3f7e4, 4759 0x3f7f8, 0x3f7fc, 4760 0x3f814, 0x3f814, 4761 0x3f82c, 0x3f82c, 4762 0x3f880, 0x3f88c, 4763 0x3f8e8, 0x3f8ec, 4764 0x3f900, 0x3f948, 4765 0x3f960, 0x3f99c, 4766 0x3f9f0, 0x3fae4, 4767 0x3faf8, 0x3fb10, 4768 0x3fb28, 0x3fb28, 4769 0x3fb3c, 0x3fb50, 4770 0x3fbf0, 0x3fc10, 4771 0x3fc28, 0x3fc28, 4772 0x3fc3c, 0x3fc50, 4773 0x3fcf0, 0x3fcfc, 4774 0x40000, 0x4000c, 4775 0x40040, 0x40068, 4776 0x4007c, 0x40144, 4777 0x40180, 0x4018c, 4778 0x40200, 0x40298, 4779 0x402ac, 0x4033c, 4780 0x403f8, 0x403fc, 4781 0x41304, 0x413c4, 4782 0x41400, 0x4141c, 4783 0x41480, 0x414d0, 4784 0x44000, 0x44078, 4785 0x440c0, 0x44278, 4786 0x442c0, 0x44478, 4787 0x444c0, 0x44678, 4788 0x446c0, 0x44878, 4789 0x448c0, 0x449fc, 4790 0x45000, 0x45068, 4791 0x45080, 0x45084, 4792 0x450a0, 0x450b0, 4793 0x45200, 0x45268, 4794 0x45280, 0x45284, 4795 0x452a0, 0x452b0, 4796 0x460c0, 0x460e4, 4797 0x47000, 0x4708c, 4798 0x47200, 0x47250, 4799 0x47400, 0x47420, 4800 0x47600, 0x47618, 4801 0x47800, 0x47814, 4802 0x48000, 0x4800c, 4803 0x48040, 0x48068, 4804 0x4807c, 0x48144, 4805 0x48180, 0x4818c, 4806 0x48200, 0x48298, 4807 0x482ac, 0x4833c, 4808 0x483f8, 0x483fc, 4809 0x49304, 0x493c4, 4810 0x49400, 0x4941c, 4811 0x49480, 0x494d0, 4812 0x4c000, 0x4c078, 4813 0x4c0c0, 0x4c278, 4814 0x4c2c0, 0x4c478, 4815 0x4c4c0, 0x4c678, 4816 0x4c6c0, 0x4c878, 4817 0x4c8c0, 0x4c9fc, 4818 0x4d000, 0x4d068, 4819 0x4d080, 0x4d084, 4820 0x4d0a0, 0x4d0b0, 4821 0x4d200, 0x4d268, 4822 0x4d280, 0x4d284, 4823 0x4d2a0, 0x4d2b0, 4824 0x4e0c0, 0x4e0e4, 4825 0x4f000, 0x4f08c, 4826 0x4f200, 0x4f250, 4827 0x4f400, 0x4f420, 4828 0x4f600, 0x4f618, 4829 0x4f800, 0x4f814, 4830 0x50000, 0x500cc, 4831 0x50400, 0x50400, 4832 0x50800, 0x508cc, 4833 0x50c00, 0x50c00, 4834 0x51000, 0x5101c, 4835 0x51300, 0x51308, 4836 }; 4837 4838 if (is_t4(sc)) { 4839 reg_ranges = &t4_reg_ranges[0]; 4840 n = nitems(t4_reg_ranges); 4841 } else { 4842 reg_ranges = &t5_reg_ranges[0]; 4843 n = nitems(t5_reg_ranges); 4844 } 4845 4846 regs->version = chip_id(sc) | chip_rev(sc) << 10; 4847 for (i = 0; i < n; i += 2) 4848 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 4849 } 4850 4851 #define A_PL_INDIR_CMD 0x1f8 4852 4853 #define S_PL_AUTOINC 31 4854 #define M_PL_AUTOINC 0x1U 4855 #define V_PL_AUTOINC(x) ((x) << S_PL_AUTOINC) 4856 #define G_PL_AUTOINC(x) (((x) >> S_PL_AUTOINC) & M_PL_AUTOINC) 4857 4858 #define S_PL_VFID 20 4859 #define M_PL_VFID 0xffU 4860 #define V_PL_VFID(x) ((x) << S_PL_VFID) 4861 #define G_PL_VFID(x) (((x) >> S_PL_VFID) & M_PL_VFID) 4862 4863 #define S_PL_ADDR 0 4864 #define M_PL_ADDR 0xfffffU 4865 #define V_PL_ADDR(x) ((x) << S_PL_ADDR) 4866 #define G_PL_ADDR(x) (((x) >> S_PL_ADDR) & M_PL_ADDR) 4867 4868 #define A_PL_INDIR_DATA 0x1fc 4869 4870 static uint64_t 4871 read_vf_stat(struct adapter *sc, unsigned int viid, int reg) 4872 { 4873 u32 stats[2]; 4874 4875 mtx_assert(&sc->regwin_lock, MA_OWNED); 4876 t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) | 4877 V_PL_VFID(G_FW_VIID_VIN(viid)) | V_PL_ADDR(VF_MPS_REG(reg))); 4878 stats[0] = t4_read_reg(sc, A_PL_INDIR_DATA); 4879 stats[1] = t4_read_reg(sc, A_PL_INDIR_DATA); 4880 return (((uint64_t)stats[1]) << 32 | stats[0]); 4881 } 4882 4883 static void 4884 t4_get_vi_stats(struct adapter *sc, unsigned int viid, 4885 struct fw_vi_stats_vf *stats) 4886 { 4887 4888 #define GET_STAT(name) \ 4889 read_vf_stat(sc, viid, A_MPS_VF_STAT_##name##_L) 4890 4891 stats->tx_bcast_bytes = GET_STAT(TX_VF_BCAST_BYTES); 4892 stats->tx_bcast_frames = GET_STAT(TX_VF_BCAST_FRAMES); 4893 stats->tx_mcast_bytes = GET_STAT(TX_VF_MCAST_BYTES); 4894 stats->tx_mcast_frames = GET_STAT(TX_VF_MCAST_FRAMES); 4895 stats->tx_ucast_bytes = GET_STAT(TX_VF_UCAST_BYTES); 4896 stats->tx_ucast_frames = GET_STAT(TX_VF_UCAST_FRAMES); 4897 stats->tx_drop_frames = GET_STAT(TX_VF_DROP_FRAMES); 4898 stats->tx_offload_bytes = GET_STAT(TX_VF_OFFLOAD_BYTES); 4899 stats->tx_offload_frames = GET_STAT(TX_VF_OFFLOAD_FRAMES); 4900 stats->rx_bcast_bytes = GET_STAT(RX_VF_BCAST_BYTES); 4901 stats->rx_bcast_frames = GET_STAT(RX_VF_BCAST_FRAMES); 4902 stats->rx_mcast_bytes = GET_STAT(RX_VF_MCAST_BYTES); 4903 stats->rx_mcast_frames = GET_STAT(RX_VF_MCAST_FRAMES); 4904 stats->rx_ucast_bytes = GET_STAT(RX_VF_UCAST_BYTES); 4905 stats->rx_ucast_frames = GET_STAT(RX_VF_UCAST_FRAMES); 4906 stats->rx_err_frames = GET_STAT(RX_VF_ERR_FRAMES); 4907 4908 #undef GET_STAT 4909 } 4910 4911 static void 4912 t4_clr_vi_stats(struct adapter *sc, unsigned int viid) 4913 { 4914 int reg; 4915 4916 t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) | 4917 V_PL_VFID(G_FW_VIID_VIN(viid)) | 4918 V_PL_ADDR(VF_MPS_REG(A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L))); 4919 for (reg = A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L; 4920 reg <= A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H; reg += 4) 4921 t4_write_reg(sc, A_PL_INDIR_DATA, 0); 4922 } 4923 4924 static void 4925 vi_refresh_stats(struct adapter *sc, struct vi_info *vi) 4926 { 4927 struct timeval tv; 4928 const struct timeval interval = {0, 250000}; /* 250ms */ 4929 4930 if (!(vi->flags & VI_INIT_DONE)) 4931 return; 4932 4933 getmicrotime(&tv); 4934 timevalsub(&tv, &interval); 4935 if (timevalcmp(&tv, &vi->last_refreshed, <)) 4936 return; 4937 4938 mtx_lock(&sc->regwin_lock); 4939 t4_get_vi_stats(sc, vi->viid, &vi->stats); 4940 getmicrotime(&vi->last_refreshed); 4941 mtx_unlock(&sc->regwin_lock); 4942 } 4943 4944 static void 4945 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi) 4946 { 4947 int i; 4948 u_int v, tnl_cong_drops; 4949 struct timeval tv; 4950 const struct timeval interval = {0, 250000}; /* 250ms */ 4951 4952 getmicrotime(&tv); 4953 timevalsub(&tv, &interval); 4954 if (timevalcmp(&tv, &pi->last_refreshed, <)) 4955 return; 4956 4957 tnl_cong_drops = 0; 4958 t4_get_port_stats(sc, pi->tx_chan, &pi->stats); 4959 for (i = 0; i < NCHAN; i++) { 4960 if (pi->rx_chan_map & (1 << i)) { 4961 mtx_lock(&sc->regwin_lock); 4962 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 4963 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 4964 mtx_unlock(&sc->regwin_lock); 4965 tnl_cong_drops += v; 4966 } 4967 } 4968 pi->tnl_cong_drops = tnl_cong_drops; 4969 getmicrotime(&pi->last_refreshed); 4970 } 4971 4972 static void 4973 cxgbe_tick(void *arg) 4974 { 4975 struct port_info *pi = arg; 4976 struct adapter *sc = pi->adapter; 4977 4978 PORT_LOCK_ASSERT_OWNED(pi); 4979 cxgbe_refresh_stats(sc, pi); 4980 4981 callout_schedule(&pi->tick, hz); 4982 } 4983 4984 void 4985 vi_tick(void *arg) 4986 { 4987 struct vi_info *vi = arg; 4988 struct adapter *sc = vi->pi->adapter; 4989 4990 vi_refresh_stats(sc, vi); 4991 4992 callout_schedule(&vi->tick, hz); 4993 } 4994 4995 static void 4996 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 4997 { 4998 struct ifnet *vlan; 4999 5000 if (arg != ifp || ifp->if_type != IFT_ETHER) 5001 return; 5002 5003 vlan = VLAN_DEVAT(ifp, vid); 5004 VLAN_SETCOOKIE(vlan, ifp); 5005 } 5006 5007 static int 5008 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 5009 { 5010 5011 #ifdef INVARIANTS 5012 panic("%s: opcode 0x%02x on iq %p with payload %p", 5013 __func__, rss->opcode, iq, m); 5014 #else 5015 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 5016 __func__, rss->opcode, iq, m); 5017 m_freem(m); 5018 #endif 5019 return (EDOOFUS); 5020 } 5021 5022 int 5023 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 5024 { 5025 uintptr_t *loc, new; 5026 5027 if (opcode >= nitems(sc->cpl_handler)) 5028 return (EINVAL); 5029 5030 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 5031 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 5032 atomic_store_rel_ptr(loc, new); 5033 5034 return (0); 5035 } 5036 5037 static int 5038 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 5039 { 5040 5041 #ifdef INVARIANTS 5042 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 5043 #else 5044 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 5045 __func__, iq, ctrl); 5046 #endif 5047 return (EDOOFUS); 5048 } 5049 5050 int 5051 t4_register_an_handler(struct adapter *sc, an_handler_t h) 5052 { 5053 uintptr_t *loc, new; 5054 5055 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 5056 loc = (uintptr_t *) &sc->an_handler; 5057 atomic_store_rel_ptr(loc, new); 5058 5059 return (0); 5060 } 5061 5062 static int 5063 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 5064 { 5065 const struct cpl_fw6_msg *cpl = 5066 __containerof(rpl, struct cpl_fw6_msg, data[0]); 5067 5068 #ifdef INVARIANTS 5069 panic("%s: fw_msg type %d", __func__, cpl->type); 5070 #else 5071 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 5072 #endif 5073 return (EDOOFUS); 5074 } 5075 5076 int 5077 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 5078 { 5079 uintptr_t *loc, new; 5080 5081 if (type >= nitems(sc->fw_msg_handler)) 5082 return (EINVAL); 5083 5084 /* 5085 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 5086 * handler dispatch table. Reject any attempt to install a handler for 5087 * this subtype. 5088 */ 5089 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 5090 return (EINVAL); 5091 5092 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 5093 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 5094 atomic_store_rel_ptr(loc, new); 5095 5096 return (0); 5097 } 5098 5099 static void 5100 t4_sysctls(struct adapter *sc) 5101 { 5102 struct sysctl_ctx_list *ctx; 5103 struct sysctl_oid *oid; 5104 struct sysctl_oid_list *children, *c0; 5105 static char *caps[] = { 5106 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 5107 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL" /* caps[1] niccaps */ 5108 "\6HASHFILTER\7ETHOFLD", 5109 "\20\1TOE", /* caps[2] toecaps */ 5110 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 5111 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 5112 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 5113 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 5114 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 5115 "\4PO_INITIAOR\5PO_TARGET" 5116 }; 5117 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 5118 5119 ctx = device_get_sysctl_ctx(sc->dev); 5120 5121 /* 5122 * dev.t4nex.X. 5123 */ 5124 oid = device_get_sysctl_tree(sc->dev); 5125 c0 = children = SYSCTL_CHILDREN(oid); 5126 5127 sc->sc_do_rxcopy = 1; 5128 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, 5129 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); 5130 5131 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 5132 sc->params.nports, "# of ports"); 5133 5134 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 5135 NULL, chip_rev(sc), "chip hardware revision"); 5136 5137 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 5138 CTLFLAG_RD, sc->fw_version, 0, "firmware version"); 5139 5140 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 5141 CTLFLAG_RD, sc->cfg_file, 0, "configuration file"); 5142 5143 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 5144 sc->cfcsum, "config file checksum"); 5145 5146 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 5147 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 5148 sysctl_bitfield, "A", "available doorbells"); 5149 5150 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 5151 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 5152 sysctl_bitfield, "A", "available link capabilities"); 5153 5154 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 5155 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 5156 sysctl_bitfield, "A", "available NIC capabilities"); 5157 5158 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 5159 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 5160 sysctl_bitfield, "A", "available TCP offload capabilities"); 5161 5162 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 5163 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 5164 sysctl_bitfield, "A", "available RDMA capabilities"); 5165 5166 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 5167 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 5168 sysctl_bitfield, "A", "available iSCSI capabilities"); 5169 5170 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 5171 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 5172 sysctl_bitfield, "A", "available FCoE capabilities"); 5173 5174 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 5175 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 5176 5177 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 5178 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 5179 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 5180 "interrupt holdoff timer values (us)"); 5181 5182 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 5183 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 5184 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 5185 "interrupt holdoff packet counter values"); 5186 5187 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, 5188 NULL, sc->tids.nftids, "number of filters"); 5189 5190 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | 5191 CTLFLAG_RD, sc, 0, sysctl_temperature, "I", 5192 "chip temperature (in Celsius)"); 5193 5194 t4_sge_sysctls(sc, ctx, children); 5195 5196 sc->lro_timeout = 100; 5197 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, 5198 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); 5199 5200 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "debug_flags", CTLFLAG_RW, 5201 &sc->debug_flags, 0, "flags to enable runtime debugging"); 5202 5203 #ifdef SBUF_DRAIN 5204 /* 5205 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 5206 */ 5207 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 5208 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 5209 "logs and miscellaneous information"); 5210 children = SYSCTL_CHILDREN(oid); 5211 5212 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 5213 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5214 sysctl_cctrl, "A", "congestion control"); 5215 5216 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 5217 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5218 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 5219 5220 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 5221 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 5222 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 5223 5224 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 5225 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 5226 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 5227 5228 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 5229 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 5230 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 5231 5232 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 5233 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 5234 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 5235 5236 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 5237 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 5238 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 5239 5240 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 5241 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5242 sysctl_cim_la, "A", "CIM logic analyzer"); 5243 5244 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", 5245 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5246 sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); 5247 5248 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 5249 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 5250 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 5251 5252 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 5253 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 5254 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 5255 5256 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 5257 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 5258 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 5259 5260 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 5261 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 5262 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 5263 5264 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 5265 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 5266 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 5267 5268 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 5269 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 5270 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 5271 5272 if (is_t5(sc)) { 5273 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 5274 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 5275 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 5276 5277 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 5278 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 5279 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 5280 } 5281 5282 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", 5283 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5284 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); 5285 5286 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 5287 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5288 sysctl_cim_qcfg, "A", "CIM queue configuration"); 5289 5290 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 5291 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5292 sysctl_cpl_stats, "A", "CPL statistics"); 5293 5294 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 5295 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5296 sysctl_ddp_stats, "A", "non-TCP DDP statistics"); 5297 5298 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 5299 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5300 sysctl_devlog, "A", "firmware's device log"); 5301 5302 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 5303 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5304 sysctl_fcoe_stats, "A", "FCoE statistics"); 5305 5306 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 5307 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5308 sysctl_hw_sched, "A", "hardware scheduler "); 5309 5310 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 5311 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5312 sysctl_l2t, "A", "hardware L2 table"); 5313 5314 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 5315 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5316 sysctl_lb_stats, "A", "loopback statistics"); 5317 5318 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 5319 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5320 sysctl_meminfo, "A", "memory regions"); 5321 5322 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", 5323 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5324 sysctl_mps_tcam, "A", "MPS TCAM entries"); 5325 5326 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 5327 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5328 sysctl_path_mtus, "A", "path MTUs"); 5329 5330 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 5331 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5332 sysctl_pm_stats, "A", "PM statistics"); 5333 5334 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 5335 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5336 sysctl_rdma_stats, "A", "RDMA statistics"); 5337 5338 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 5339 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5340 sysctl_tcp_stats, "A", "TCP statistics"); 5341 5342 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 5343 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5344 sysctl_tids, "A", "TID information"); 5345 5346 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 5347 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5348 sysctl_tp_err_stats, "A", "TP error statistics"); 5349 5350 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", 5351 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5352 sysctl_tp_la, "A", "TP logic analyzer"); 5353 5354 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 5355 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5356 sysctl_tx_rate, "A", "Tx rate"); 5357 5358 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", 5359 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5360 sysctl_ulprx_la, "A", "ULPRX logic analyzer"); 5361 5362 if (is_t5(sc)) { 5363 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 5364 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5365 sysctl_wcwr_stats, "A", "write combined work requests"); 5366 } 5367 #endif 5368 5369 #ifdef TCP_OFFLOAD 5370 if (is_offload(sc)) { 5371 /* 5372 * dev.t4nex.X.toe. 5373 */ 5374 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 5375 NULL, "TOE parameters"); 5376 children = SYSCTL_CHILDREN(oid); 5377 5378 sc->tt.sndbuf = 256 * 1024; 5379 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 5380 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 5381 5382 sc->tt.ddp = 0; 5383 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 5384 &sc->tt.ddp, 0, "DDP allowed"); 5385 5386 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 5387 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 5388 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 5389 5390 sc->tt.ddp_thres = 5391 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 5392 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 5393 &sc->tt.ddp_thres, 0, "DDP threshold"); 5394 5395 sc->tt.rx_coalesce = 1; 5396 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", 5397 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); 5398 5399 sc->tt.tx_align = 1; 5400 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align", 5401 CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload"); 5402 } 5403 #endif 5404 } 5405 5406 void 5407 vi_sysctls(struct vi_info *vi) 5408 { 5409 struct sysctl_ctx_list *ctx; 5410 struct sysctl_oid *oid; 5411 struct sysctl_oid_list *children; 5412 5413 ctx = device_get_sysctl_ctx(vi->dev); 5414 5415 /* 5416 * dev.[nv](cxgbe|cxl).X. 5417 */ 5418 oid = device_get_sysctl_tree(vi->dev); 5419 children = SYSCTL_CHILDREN(oid); 5420 5421 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "viid", CTLFLAG_RD, NULL, 5422 vi->viid, "VI identifer"); 5423 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 5424 &vi->nrxq, 0, "# of rx queues"); 5425 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 5426 &vi->ntxq, 0, "# of tx queues"); 5427 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 5428 &vi->first_rxq, 0, "index of first rx queue"); 5429 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 5430 &vi->first_txq, 0, "index of first tx queue"); 5431 5432 if (vi->flags & VI_NETMAP) 5433 return; 5434 5435 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT | 5436 CTLFLAG_RW, vi, 0, sysctl_noflowq, "IU", 5437 "Reserve queue 0 for non-flowid packets"); 5438 5439 #ifdef TCP_OFFLOAD 5440 if (vi->nofldrxq != 0) { 5441 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 5442 &vi->nofldrxq, 0, 5443 "# of rx queues for offloaded TCP connections"); 5444 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 5445 &vi->nofldtxq, 0, 5446 "# of tx queues for offloaded TCP connections"); 5447 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 5448 CTLFLAG_RD, &vi->first_ofld_rxq, 0, 5449 "index of first TOE rx queue"); 5450 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 5451 CTLFLAG_RD, &vi->first_ofld_txq, 0, 5452 "index of first TOE tx queue"); 5453 } 5454 #endif 5455 5456 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 5457 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_holdoff_tmr_idx, "I", 5458 "holdoff timer index"); 5459 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 5460 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_holdoff_pktc_idx, "I", 5461 "holdoff packet counter index"); 5462 5463 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 5464 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_qsize_rxq, "I", 5465 "rx queue size"); 5466 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 5467 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_qsize_txq, "I", 5468 "tx queue size"); 5469 } 5470 5471 static void 5472 cxgbe_sysctls(struct port_info *pi) 5473 { 5474 struct sysctl_ctx_list *ctx; 5475 struct sysctl_oid *oid; 5476 struct sysctl_oid_list *children; 5477 struct adapter *sc = pi->adapter; 5478 5479 ctx = device_get_sysctl_ctx(pi->dev); 5480 5481 /* 5482 * dev.cxgbe.X. 5483 */ 5484 oid = device_get_sysctl_tree(pi->dev); 5485 children = SYSCTL_CHILDREN(oid); 5486 5487 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | 5488 CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); 5489 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { 5490 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 5491 CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I", 5492 "PHY temperature (in Celsius)"); 5493 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", 5494 CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I", 5495 "PHY firmware version"); 5496 } 5497 5498 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings", 5499 CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings, 5500 "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)"); 5501 5502 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "max_speed", CTLFLAG_RD, NULL, 5503 port_top_speed(pi), "max speed (in Gbps)"); 5504 5505 /* 5506 * dev.cxgbe.X.stats. 5507 */ 5508 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 5509 NULL, "port statistics"); 5510 children = SYSCTL_CHILDREN(oid); 5511 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD, 5512 &pi->tx_parse_error, 0, 5513 "# of tx packets with invalid length or # of segments"); 5514 5515 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 5516 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 5517 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ 5518 sysctl_handle_t4_reg64, "QU", desc) 5519 5520 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 5521 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 5522 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 5523 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 5524 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 5525 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 5526 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 5527 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 5528 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 5529 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 5530 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 5531 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 5532 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 5533 "# of tx frames in this range", 5534 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 5535 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 5536 "# of tx frames in this range", 5537 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 5538 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 5539 "# of tx frames in this range", 5540 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 5541 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 5542 "# of tx frames in this range", 5543 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 5544 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 5545 "# of tx frames in this range", 5546 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 5547 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 5548 "# of tx frames in this range", 5549 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 5550 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 5551 "# of tx frames in this range", 5552 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 5553 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 5554 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 5555 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 5556 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 5557 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 5558 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 5559 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 5560 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 5561 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 5562 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 5563 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 5564 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 5565 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 5566 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 5567 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 5568 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 5569 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 5570 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 5571 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 5572 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 5573 5574 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 5575 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 5576 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 5577 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 5578 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 5579 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 5580 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 5581 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 5582 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 5583 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 5584 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 5585 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 5586 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 5587 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 5588 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 5589 "# of frames received with bad FCS", 5590 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 5591 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 5592 "# of frames received with length error", 5593 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 5594 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 5595 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 5596 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 5597 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 5598 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 5599 "# of rx frames in this range", 5600 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 5601 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 5602 "# of rx frames in this range", 5603 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 5604 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 5605 "# of rx frames in this range", 5606 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 5607 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 5608 "# of rx frames in this range", 5609 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 5610 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 5611 "# of rx frames in this range", 5612 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 5613 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 5614 "# of rx frames in this range", 5615 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 5616 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 5617 "# of rx frames in this range", 5618 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 5619 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 5620 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 5621 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 5622 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 5623 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 5624 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 5625 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 5626 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 5627 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 5628 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 5629 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 5630 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 5631 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 5632 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 5633 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 5634 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 5635 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 5636 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 5637 5638 #undef SYSCTL_ADD_T4_REG64 5639 5640 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 5641 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 5642 &pi->stats.name, desc) 5643 5644 /* We get these from port_stats and they may be stale by upto 1s */ 5645 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 5646 "# drops due to buffer-group 0 overflows"); 5647 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 5648 "# drops due to buffer-group 1 overflows"); 5649 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 5650 "# drops due to buffer-group 2 overflows"); 5651 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 5652 "# drops due to buffer-group 3 overflows"); 5653 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 5654 "# of buffer-group 0 truncated packets"); 5655 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 5656 "# of buffer-group 1 truncated packets"); 5657 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 5658 "# of buffer-group 2 truncated packets"); 5659 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 5660 "# of buffer-group 3 truncated packets"); 5661 5662 #undef SYSCTL_ADD_T4_PORTSTAT 5663 } 5664 5665 static int 5666 sysctl_int_array(SYSCTL_HANDLER_ARGS) 5667 { 5668 int rc, *i, space = 0; 5669 struct sbuf sb; 5670 5671 sbuf_new_for_sysctl(&sb, NULL, 64, req); 5672 for (i = arg1; arg2; arg2 -= sizeof(int), i++) { 5673 if (space) 5674 sbuf_printf(&sb, " "); 5675 sbuf_printf(&sb, "%d", *i); 5676 space = 1; 5677 } 5678 rc = sbuf_finish(&sb); 5679 sbuf_delete(&sb); 5680 return (rc); 5681 } 5682 5683 static int 5684 sysctl_bitfield(SYSCTL_HANDLER_ARGS) 5685 { 5686 int rc; 5687 struct sbuf *sb; 5688 5689 rc = sysctl_wire_old_buffer(req, 0); 5690 if (rc != 0) 5691 return(rc); 5692 5693 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5694 if (sb == NULL) 5695 return (ENOMEM); 5696 5697 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 5698 rc = sbuf_finish(sb); 5699 sbuf_delete(sb); 5700 5701 return (rc); 5702 } 5703 5704 static int 5705 sysctl_btphy(SYSCTL_HANDLER_ARGS) 5706 { 5707 struct port_info *pi = arg1; 5708 int op = arg2; 5709 struct adapter *sc = pi->adapter; 5710 u_int v; 5711 int rc; 5712 5713 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, "t4btt"); 5714 if (rc) 5715 return (rc); 5716 /* XXX: magic numbers */ 5717 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, 5718 &v); 5719 end_synchronized_op(sc, 0); 5720 if (rc) 5721 return (rc); 5722 if (op == 0) 5723 v /= 256; 5724 5725 rc = sysctl_handle_int(oidp, &v, 0, req); 5726 return (rc); 5727 } 5728 5729 static int 5730 sysctl_noflowq(SYSCTL_HANDLER_ARGS) 5731 { 5732 struct vi_info *vi = arg1; 5733 int rc, val; 5734 5735 val = vi->rsrv_noflowq; 5736 rc = sysctl_handle_int(oidp, &val, 0, req); 5737 if (rc != 0 || req->newptr == NULL) 5738 return (rc); 5739 5740 if ((val >= 1) && (vi->ntxq > 1)) 5741 vi->rsrv_noflowq = 1; 5742 else 5743 vi->rsrv_noflowq = 0; 5744 5745 return (rc); 5746 } 5747 5748 static int 5749 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 5750 { 5751 struct vi_info *vi = arg1; 5752 struct adapter *sc = vi->pi->adapter; 5753 int idx, rc, i; 5754 struct sge_rxq *rxq; 5755 #ifdef TCP_OFFLOAD 5756 struct sge_ofld_rxq *ofld_rxq; 5757 #endif 5758 uint8_t v; 5759 5760 idx = vi->tmr_idx; 5761 5762 rc = sysctl_handle_int(oidp, &idx, 0, req); 5763 if (rc != 0 || req->newptr == NULL) 5764 return (rc); 5765 5766 if (idx < 0 || idx >= SGE_NTIMERS) 5767 return (EINVAL); 5768 5769 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5770 "t4tmr"); 5771 if (rc) 5772 return (rc); 5773 5774 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(vi->pktc_idx != -1); 5775 for_each_rxq(vi, i, rxq) { 5776 #ifdef atomic_store_rel_8 5777 atomic_store_rel_8(&rxq->iq.intr_params, v); 5778 #else 5779 rxq->iq.intr_params = v; 5780 #endif 5781 } 5782 #ifdef TCP_OFFLOAD 5783 for_each_ofld_rxq(vi, i, ofld_rxq) { 5784 #ifdef atomic_store_rel_8 5785 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); 5786 #else 5787 ofld_rxq->iq.intr_params = v; 5788 #endif 5789 } 5790 #endif 5791 vi->tmr_idx = idx; 5792 5793 end_synchronized_op(sc, LOCK_HELD); 5794 return (0); 5795 } 5796 5797 static int 5798 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 5799 { 5800 struct vi_info *vi = arg1; 5801 struct adapter *sc = vi->pi->adapter; 5802 int idx, rc; 5803 5804 idx = vi->pktc_idx; 5805 5806 rc = sysctl_handle_int(oidp, &idx, 0, req); 5807 if (rc != 0 || req->newptr == NULL) 5808 return (rc); 5809 5810 if (idx < -1 || idx >= SGE_NCOUNTERS) 5811 return (EINVAL); 5812 5813 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5814 "t4pktc"); 5815 if (rc) 5816 return (rc); 5817 5818 if (vi->flags & VI_INIT_DONE) 5819 rc = EBUSY; /* cannot be changed once the queues are created */ 5820 else 5821 vi->pktc_idx = idx; 5822 5823 end_synchronized_op(sc, LOCK_HELD); 5824 return (rc); 5825 } 5826 5827 static int 5828 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 5829 { 5830 struct vi_info *vi = arg1; 5831 struct adapter *sc = vi->pi->adapter; 5832 int qsize, rc; 5833 5834 qsize = vi->qsize_rxq; 5835 5836 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5837 if (rc != 0 || req->newptr == NULL) 5838 return (rc); 5839 5840 if (qsize < 128 || (qsize & 7)) 5841 return (EINVAL); 5842 5843 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5844 "t4rxqs"); 5845 if (rc) 5846 return (rc); 5847 5848 if (vi->flags & VI_INIT_DONE) 5849 rc = EBUSY; /* cannot be changed once the queues are created */ 5850 else 5851 vi->qsize_rxq = qsize; 5852 5853 end_synchronized_op(sc, LOCK_HELD); 5854 return (rc); 5855 } 5856 5857 static int 5858 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 5859 { 5860 struct vi_info *vi = arg1; 5861 struct adapter *sc = vi->pi->adapter; 5862 int qsize, rc; 5863 5864 qsize = vi->qsize_txq; 5865 5866 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5867 if (rc != 0 || req->newptr == NULL) 5868 return (rc); 5869 5870 if (qsize < 128 || qsize > 65536) 5871 return (EINVAL); 5872 5873 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5874 "t4txqs"); 5875 if (rc) 5876 return (rc); 5877 5878 if (vi->flags & VI_INIT_DONE) 5879 rc = EBUSY; /* cannot be changed once the queues are created */ 5880 else 5881 vi->qsize_txq = qsize; 5882 5883 end_synchronized_op(sc, LOCK_HELD); 5884 return (rc); 5885 } 5886 5887 static int 5888 sysctl_pause_settings(SYSCTL_HANDLER_ARGS) 5889 { 5890 struct port_info *pi = arg1; 5891 struct adapter *sc = pi->adapter; 5892 struct link_config *lc = &pi->link_cfg; 5893 int rc; 5894 5895 if (req->newptr == NULL) { 5896 struct sbuf *sb; 5897 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX"; 5898 5899 rc = sysctl_wire_old_buffer(req, 0); 5900 if (rc != 0) 5901 return(rc); 5902 5903 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5904 if (sb == NULL) 5905 return (ENOMEM); 5906 5907 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits); 5908 rc = sbuf_finish(sb); 5909 sbuf_delete(sb); 5910 } else { 5911 char s[2]; 5912 int n; 5913 5914 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX)); 5915 s[1] = 0; 5916 5917 rc = sysctl_handle_string(oidp, s, sizeof(s), req); 5918 if (rc != 0) 5919 return(rc); 5920 5921 if (s[1] != 0) 5922 return (EINVAL); 5923 if (s[0] < '0' || s[0] > '9') 5924 return (EINVAL); /* not a number */ 5925 n = s[0] - '0'; 5926 if (n & ~(PAUSE_TX | PAUSE_RX)) 5927 return (EINVAL); /* some other bit is set too */ 5928 5929 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, 5930 "t4PAUSE"); 5931 if (rc) 5932 return (rc); 5933 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) { 5934 int link_ok = lc->link_ok; 5935 5936 lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX); 5937 lc->requested_fc |= n; 5938 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc); 5939 lc->link_ok = link_ok; /* restore */ 5940 } 5941 end_synchronized_op(sc, 0); 5942 } 5943 5944 return (rc); 5945 } 5946 5947 static int 5948 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 5949 { 5950 struct adapter *sc = arg1; 5951 int reg = arg2; 5952 uint64_t val; 5953 5954 val = t4_read_reg64(sc, reg); 5955 5956 return (sysctl_handle_64(oidp, &val, 0, req)); 5957 } 5958 5959 static int 5960 sysctl_temperature(SYSCTL_HANDLER_ARGS) 5961 { 5962 struct adapter *sc = arg1; 5963 int rc, t; 5964 uint32_t param, val; 5965 5966 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); 5967 if (rc) 5968 return (rc); 5969 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 5970 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 5971 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); 5972 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 5973 end_synchronized_op(sc, 0); 5974 if (rc) 5975 return (rc); 5976 5977 /* unknown is returned as 0 but we display -1 in that case */ 5978 t = val == 0 ? -1 : val; 5979 5980 rc = sysctl_handle_int(oidp, &t, 0, req); 5981 return (rc); 5982 } 5983 5984 #ifdef SBUF_DRAIN 5985 static int 5986 sysctl_cctrl(SYSCTL_HANDLER_ARGS) 5987 { 5988 struct adapter *sc = arg1; 5989 struct sbuf *sb; 5990 int rc, i; 5991 uint16_t incr[NMTUS][NCCTRL_WIN]; 5992 static const char *dec_fac[] = { 5993 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 5994 "0.9375" 5995 }; 5996 5997 rc = sysctl_wire_old_buffer(req, 0); 5998 if (rc != 0) 5999 return (rc); 6000 6001 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6002 if (sb == NULL) 6003 return (ENOMEM); 6004 6005 t4_read_cong_tbl(sc, incr); 6006 6007 for (i = 0; i < NCCTRL_WIN; ++i) { 6008 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 6009 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 6010 incr[5][i], incr[6][i], incr[7][i]); 6011 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 6012 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 6013 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 6014 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 6015 } 6016 6017 rc = sbuf_finish(sb); 6018 sbuf_delete(sb); 6019 6020 return (rc); 6021 } 6022 6023 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 6024 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 6025 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 6026 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 6027 }; 6028 6029 static int 6030 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 6031 { 6032 struct adapter *sc = arg1; 6033 struct sbuf *sb; 6034 int rc, i, n, qid = arg2; 6035 uint32_t *buf, *p; 6036 char *qtype; 6037 u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5; 6038 6039 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 6040 ("%s: bad qid %d\n", __func__, qid)); 6041 6042 if (qid < CIM_NUM_IBQ) { 6043 /* inbound queue */ 6044 qtype = "IBQ"; 6045 n = 4 * CIM_IBQ_SIZE; 6046 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 6047 rc = t4_read_cim_ibq(sc, qid, buf, n); 6048 } else { 6049 /* outbound queue */ 6050 qtype = "OBQ"; 6051 qid -= CIM_NUM_IBQ; 6052 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 6053 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 6054 rc = t4_read_cim_obq(sc, qid, buf, n); 6055 } 6056 6057 if (rc < 0) { 6058 rc = -rc; 6059 goto done; 6060 } 6061 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 6062 6063 rc = sysctl_wire_old_buffer(req, 0); 6064 if (rc != 0) 6065 goto done; 6066 6067 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 6068 if (sb == NULL) { 6069 rc = ENOMEM; 6070 goto done; 6071 } 6072 6073 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 6074 for (i = 0, p = buf; i < n; i += 16, p += 4) 6075 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 6076 p[2], p[3]); 6077 6078 rc = sbuf_finish(sb); 6079 sbuf_delete(sb); 6080 done: 6081 free(buf, M_CXGBE); 6082 return (rc); 6083 } 6084 6085 static int 6086 sysctl_cim_la(SYSCTL_HANDLER_ARGS) 6087 { 6088 struct adapter *sc = arg1; 6089 u_int cfg; 6090 struct sbuf *sb; 6091 uint32_t *buf, *p; 6092 int rc; 6093 6094 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 6095 if (rc != 0) 6096 return (rc); 6097 6098 rc = sysctl_wire_old_buffer(req, 0); 6099 if (rc != 0) 6100 return (rc); 6101 6102 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6103 if (sb == NULL) 6104 return (ENOMEM); 6105 6106 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 6107 M_ZERO | M_WAITOK); 6108 6109 rc = -t4_cim_read_la(sc, buf, NULL); 6110 if (rc != 0) 6111 goto done; 6112 6113 sbuf_printf(sb, "Status Data PC%s", 6114 cfg & F_UPDBGLACAPTPCONLY ? "" : 6115 " LS0Stat LS0Addr LS0Data"); 6116 6117 KASSERT((sc->params.cim_la_size & 7) == 0, 6118 ("%s: p will walk off the end of buf", __func__)); 6119 6120 for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) { 6121 if (cfg & F_UPDBGLACAPTPCONLY) { 6122 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 6123 p[6], p[7]); 6124 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 6125 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 6126 p[4] & 0xff, p[5] >> 8); 6127 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 6128 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 6129 p[1] & 0xf, p[2] >> 4); 6130 } else { 6131 sbuf_printf(sb, 6132 "\n %02x %x%07x %x%07x %08x %08x " 6133 "%08x%08x%08x%08x", 6134 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 6135 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 6136 p[6], p[7]); 6137 } 6138 } 6139 6140 rc = sbuf_finish(sb); 6141 sbuf_delete(sb); 6142 done: 6143 free(buf, M_CXGBE); 6144 return (rc); 6145 } 6146 6147 static int 6148 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) 6149 { 6150 struct adapter *sc = arg1; 6151 u_int i; 6152 struct sbuf *sb; 6153 uint32_t *buf, *p; 6154 int rc; 6155 6156 rc = sysctl_wire_old_buffer(req, 0); 6157 if (rc != 0) 6158 return (rc); 6159 6160 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6161 if (sb == NULL) 6162 return (ENOMEM); 6163 6164 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, 6165 M_ZERO | M_WAITOK); 6166 6167 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); 6168 p = buf; 6169 6170 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 6171 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], 6172 p[1], p[0]); 6173 } 6174 6175 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); 6176 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 6177 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", 6178 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, 6179 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, 6180 (p[1] >> 2) | ((p[2] & 3) << 30), 6181 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, 6182 p[0] & 1); 6183 } 6184 6185 rc = sbuf_finish(sb); 6186 sbuf_delete(sb); 6187 free(buf, M_CXGBE); 6188 return (rc); 6189 } 6190 6191 static int 6192 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) 6193 { 6194 struct adapter *sc = arg1; 6195 u_int i; 6196 struct sbuf *sb; 6197 uint32_t *buf, *p; 6198 int rc; 6199 6200 rc = sysctl_wire_old_buffer(req, 0); 6201 if (rc != 0) 6202 return (rc); 6203 6204 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6205 if (sb == NULL) 6206 return (ENOMEM); 6207 6208 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, 6209 M_ZERO | M_WAITOK); 6210 6211 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); 6212 p = buf; 6213 6214 sbuf_printf(sb, "Cntl ID DataBE Addr Data"); 6215 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 6216 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", 6217 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, 6218 p[4], p[3], p[2], p[1], p[0]); 6219 } 6220 6221 sbuf_printf(sb, "\n\nCntl ID Data"); 6222 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 6223 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", 6224 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); 6225 } 6226 6227 rc = sbuf_finish(sb); 6228 sbuf_delete(sb); 6229 free(buf, M_CXGBE); 6230 return (rc); 6231 } 6232 6233 static int 6234 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 6235 { 6236 struct adapter *sc = arg1; 6237 struct sbuf *sb; 6238 int rc, i; 6239 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 6240 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 6241 uint16_t thres[CIM_NUM_IBQ]; 6242 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 6243 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 6244 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 6245 6246 if (is_t4(sc)) { 6247 cim_num_obq = CIM_NUM_OBQ; 6248 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 6249 obq_rdaddr = A_UP_OBQ_0_REALADDR; 6250 } else { 6251 cim_num_obq = CIM_NUM_OBQ_T5; 6252 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 6253 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 6254 } 6255 nq = CIM_NUM_IBQ + cim_num_obq; 6256 6257 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 6258 if (rc == 0) 6259 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 6260 if (rc != 0) 6261 return (rc); 6262 6263 t4_read_cimq_cfg(sc, base, size, thres); 6264 6265 rc = sysctl_wire_old_buffer(req, 0); 6266 if (rc != 0) 6267 return (rc); 6268 6269 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 6270 if (sb == NULL) 6271 return (ENOMEM); 6272 6273 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 6274 6275 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 6276 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 6277 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 6278 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 6279 G_QUEREMFLITS(p[2]) * 16); 6280 for ( ; i < nq; i++, p += 4, wr += 2) 6281 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 6282 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 6283 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 6284 G_QUEREMFLITS(p[2]) * 16); 6285 6286 rc = sbuf_finish(sb); 6287 sbuf_delete(sb); 6288 6289 return (rc); 6290 } 6291 6292 static int 6293 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 6294 { 6295 struct adapter *sc = arg1; 6296 struct sbuf *sb; 6297 int rc; 6298 struct tp_cpl_stats stats; 6299 6300 rc = sysctl_wire_old_buffer(req, 0); 6301 if (rc != 0) 6302 return (rc); 6303 6304 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6305 if (sb == NULL) 6306 return (ENOMEM); 6307 6308 t4_tp_get_cpl_stats(sc, &stats); 6309 6310 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6311 "channel 3\n"); 6312 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 6313 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 6314 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 6315 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 6316 6317 rc = sbuf_finish(sb); 6318 sbuf_delete(sb); 6319 6320 return (rc); 6321 } 6322 6323 static int 6324 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 6325 { 6326 struct adapter *sc = arg1; 6327 struct sbuf *sb; 6328 int rc; 6329 struct tp_usm_stats stats; 6330 6331 rc = sysctl_wire_old_buffer(req, 0); 6332 if (rc != 0) 6333 return(rc); 6334 6335 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6336 if (sb == NULL) 6337 return (ENOMEM); 6338 6339 t4_get_usm_stats(sc, &stats); 6340 6341 sbuf_printf(sb, "Frames: %u\n", stats.frames); 6342 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 6343 sbuf_printf(sb, "Drops: %u", stats.drops); 6344 6345 rc = sbuf_finish(sb); 6346 sbuf_delete(sb); 6347 6348 return (rc); 6349 } 6350 6351 const char *devlog_level_strings[] = { 6352 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 6353 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 6354 [FW_DEVLOG_LEVEL_ERR] = "ERR", 6355 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 6356 [FW_DEVLOG_LEVEL_INFO] = "INFO", 6357 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 6358 }; 6359 6360 const char *devlog_facility_strings[] = { 6361 [FW_DEVLOG_FACILITY_CORE] = "CORE", 6362 [FW_DEVLOG_FACILITY_CF] = "CF", 6363 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 6364 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 6365 [FW_DEVLOG_FACILITY_RES] = "RES", 6366 [FW_DEVLOG_FACILITY_HW] = "HW", 6367 [FW_DEVLOG_FACILITY_FLR] = "FLR", 6368 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 6369 [FW_DEVLOG_FACILITY_PHY] = "PHY", 6370 [FW_DEVLOG_FACILITY_MAC] = "MAC", 6371 [FW_DEVLOG_FACILITY_PORT] = "PORT", 6372 [FW_DEVLOG_FACILITY_VI] = "VI", 6373 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 6374 [FW_DEVLOG_FACILITY_ACL] = "ACL", 6375 [FW_DEVLOG_FACILITY_TM] = "TM", 6376 [FW_DEVLOG_FACILITY_QFC] = "QFC", 6377 [FW_DEVLOG_FACILITY_DCB] = "DCB", 6378 [FW_DEVLOG_FACILITY_ETH] = "ETH", 6379 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 6380 [FW_DEVLOG_FACILITY_RI] = "RI", 6381 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 6382 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 6383 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 6384 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 6385 }; 6386 6387 static int 6388 sysctl_devlog(SYSCTL_HANDLER_ARGS) 6389 { 6390 struct adapter *sc = arg1; 6391 struct devlog_params *dparams = &sc->params.devlog; 6392 struct fw_devlog_e *buf, *e; 6393 int i, j, rc, nentries, first = 0, m; 6394 struct sbuf *sb; 6395 uint64_t ftstamp = UINT64_MAX; 6396 6397 if (dparams->start == 0) { 6398 dparams->memtype = FW_MEMTYPE_EDC0; 6399 dparams->start = 0x84000; 6400 dparams->size = 32768; 6401 } 6402 6403 nentries = dparams->size / sizeof(struct fw_devlog_e); 6404 6405 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 6406 if (buf == NULL) 6407 return (ENOMEM); 6408 6409 m = fwmtype_to_hwmtype(dparams->memtype); 6410 rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf); 6411 if (rc != 0) 6412 goto done; 6413 6414 for (i = 0; i < nentries; i++) { 6415 e = &buf[i]; 6416 6417 if (e->timestamp == 0) 6418 break; /* end */ 6419 6420 e->timestamp = be64toh(e->timestamp); 6421 e->seqno = be32toh(e->seqno); 6422 for (j = 0; j < 8; j++) 6423 e->params[j] = be32toh(e->params[j]); 6424 6425 if (e->timestamp < ftstamp) { 6426 ftstamp = e->timestamp; 6427 first = i; 6428 } 6429 } 6430 6431 if (buf[first].timestamp == 0) 6432 goto done; /* nothing in the log */ 6433 6434 rc = sysctl_wire_old_buffer(req, 0); 6435 if (rc != 0) 6436 goto done; 6437 6438 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6439 if (sb == NULL) { 6440 rc = ENOMEM; 6441 goto done; 6442 } 6443 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 6444 "Seq#", "Tstamp", "Level", "Facility", "Message"); 6445 6446 i = first; 6447 do { 6448 e = &buf[i]; 6449 if (e->timestamp == 0) 6450 break; /* end */ 6451 6452 sbuf_printf(sb, "%10d %15ju %8s %8s ", 6453 e->seqno, e->timestamp, 6454 (e->level < nitems(devlog_level_strings) ? 6455 devlog_level_strings[e->level] : "UNKNOWN"), 6456 (e->facility < nitems(devlog_facility_strings) ? 6457 devlog_facility_strings[e->facility] : "UNKNOWN")); 6458 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 6459 e->params[2], e->params[3], e->params[4], 6460 e->params[5], e->params[6], e->params[7]); 6461 6462 if (++i == nentries) 6463 i = 0; 6464 } while (i != first); 6465 6466 rc = sbuf_finish(sb); 6467 sbuf_delete(sb); 6468 done: 6469 free(buf, M_CXGBE); 6470 return (rc); 6471 } 6472 6473 static int 6474 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 6475 { 6476 struct adapter *sc = arg1; 6477 struct sbuf *sb; 6478 int rc; 6479 struct tp_fcoe_stats stats[4]; 6480 6481 rc = sysctl_wire_old_buffer(req, 0); 6482 if (rc != 0) 6483 return (rc); 6484 6485 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6486 if (sb == NULL) 6487 return (ENOMEM); 6488 6489 t4_get_fcoe_stats(sc, 0, &stats[0]); 6490 t4_get_fcoe_stats(sc, 1, &stats[1]); 6491 t4_get_fcoe_stats(sc, 2, &stats[2]); 6492 t4_get_fcoe_stats(sc, 3, &stats[3]); 6493 6494 sbuf_printf(sb, " channel 0 channel 1 " 6495 "channel 2 channel 3\n"); 6496 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 6497 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 6498 stats[3].octetsDDP); 6499 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 6500 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 6501 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 6502 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 6503 stats[3].framesDrop); 6504 6505 rc = sbuf_finish(sb); 6506 sbuf_delete(sb); 6507 6508 return (rc); 6509 } 6510 6511 static int 6512 sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 6513 { 6514 struct adapter *sc = arg1; 6515 struct sbuf *sb; 6516 int rc, i; 6517 unsigned int map, kbps, ipg, mode; 6518 unsigned int pace_tab[NTX_SCHED]; 6519 6520 rc = sysctl_wire_old_buffer(req, 0); 6521 if (rc != 0) 6522 return (rc); 6523 6524 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6525 if (sb == NULL) 6526 return (ENOMEM); 6527 6528 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 6529 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 6530 t4_read_pace_tbl(sc, pace_tab); 6531 6532 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 6533 "Class IPG (0.1 ns) Flow IPG (us)"); 6534 6535 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 6536 t4_get_tx_sched(sc, i, &kbps, &ipg); 6537 sbuf_printf(sb, "\n %u %-5s %u ", i, 6538 (mode & (1 << i)) ? "flow" : "class", map & 3); 6539 if (kbps) 6540 sbuf_printf(sb, "%9u ", kbps); 6541 else 6542 sbuf_printf(sb, " disabled "); 6543 6544 if (ipg) 6545 sbuf_printf(sb, "%13u ", ipg); 6546 else 6547 sbuf_printf(sb, " disabled "); 6548 6549 if (pace_tab[i]) 6550 sbuf_printf(sb, "%10u", pace_tab[i]); 6551 else 6552 sbuf_printf(sb, " disabled"); 6553 } 6554 6555 rc = sbuf_finish(sb); 6556 sbuf_delete(sb); 6557 6558 return (rc); 6559 } 6560 6561 static int 6562 sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 6563 { 6564 struct adapter *sc = arg1; 6565 struct sbuf *sb; 6566 int rc, i, j; 6567 uint64_t *p0, *p1; 6568 struct lb_port_stats s[2]; 6569 static const char *stat_name[] = { 6570 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 6571 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 6572 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 6573 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 6574 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 6575 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 6576 "BG2FramesTrunc:", "BG3FramesTrunc:" 6577 }; 6578 6579 rc = sysctl_wire_old_buffer(req, 0); 6580 if (rc != 0) 6581 return (rc); 6582 6583 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6584 if (sb == NULL) 6585 return (ENOMEM); 6586 6587 memset(s, 0, sizeof(s)); 6588 6589 for (i = 0; i < 4; i += 2) { 6590 t4_get_lb_stats(sc, i, &s[0]); 6591 t4_get_lb_stats(sc, i + 1, &s[1]); 6592 6593 p0 = &s[0].octets; 6594 p1 = &s[1].octets; 6595 sbuf_printf(sb, "%s Loopback %u" 6596 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 6597 6598 for (j = 0; j < nitems(stat_name); j++) 6599 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 6600 *p0++, *p1++); 6601 } 6602 6603 rc = sbuf_finish(sb); 6604 sbuf_delete(sb); 6605 6606 return (rc); 6607 } 6608 6609 static int 6610 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) 6611 { 6612 int rc = 0; 6613 struct port_info *pi = arg1; 6614 struct sbuf *sb; 6615 static const char *linkdnreasons[] = { 6616 "non-specific", "remote fault", "autoneg failed", "reserved3", 6617 "PHY overheated", "unknown", "rx los", "reserved7" 6618 }; 6619 6620 rc = sysctl_wire_old_buffer(req, 0); 6621 if (rc != 0) 6622 return(rc); 6623 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); 6624 if (sb == NULL) 6625 return (ENOMEM); 6626 6627 if (pi->linkdnrc < 0) 6628 sbuf_printf(sb, "n/a"); 6629 else if (pi->linkdnrc < nitems(linkdnreasons)) 6630 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]); 6631 else 6632 sbuf_printf(sb, "%d", pi->linkdnrc); 6633 6634 rc = sbuf_finish(sb); 6635 sbuf_delete(sb); 6636 6637 return (rc); 6638 } 6639 6640 struct mem_desc { 6641 unsigned int base; 6642 unsigned int limit; 6643 unsigned int idx; 6644 }; 6645 6646 static int 6647 mem_desc_cmp(const void *a, const void *b) 6648 { 6649 return ((const struct mem_desc *)a)->base - 6650 ((const struct mem_desc *)b)->base; 6651 } 6652 6653 static void 6654 mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 6655 unsigned int to) 6656 { 6657 unsigned int size; 6658 6659 size = to - from + 1; 6660 if (size == 0) 6661 return; 6662 6663 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 6664 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 6665 } 6666 6667 static int 6668 sysctl_meminfo(SYSCTL_HANDLER_ARGS) 6669 { 6670 struct adapter *sc = arg1; 6671 struct sbuf *sb; 6672 int rc, i, n; 6673 uint32_t lo, hi, used, alloc; 6674 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 6675 static const char *region[] = { 6676 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 6677 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 6678 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 6679 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 6680 "RQUDP region:", "PBL region:", "TXPBL region:", 6681 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 6682 "On-chip queues:" 6683 }; 6684 struct mem_desc avail[4]; 6685 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 6686 struct mem_desc *md = mem; 6687 6688 rc = sysctl_wire_old_buffer(req, 0); 6689 if (rc != 0) 6690 return (rc); 6691 6692 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6693 if (sb == NULL) 6694 return (ENOMEM); 6695 6696 for (i = 0; i < nitems(mem); i++) { 6697 mem[i].limit = 0; 6698 mem[i].idx = i; 6699 } 6700 6701 /* Find and sort the populated memory ranges */ 6702 i = 0; 6703 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 6704 if (lo & F_EDRAM0_ENABLE) { 6705 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 6706 avail[i].base = G_EDRAM0_BASE(hi) << 20; 6707 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 6708 avail[i].idx = 0; 6709 i++; 6710 } 6711 if (lo & F_EDRAM1_ENABLE) { 6712 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 6713 avail[i].base = G_EDRAM1_BASE(hi) << 20; 6714 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 6715 avail[i].idx = 1; 6716 i++; 6717 } 6718 if (lo & F_EXT_MEM_ENABLE) { 6719 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 6720 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 6721 avail[i].limit = avail[i].base + 6722 (G_EXT_MEM_SIZE(hi) << 20); 6723 avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */ 6724 i++; 6725 } 6726 if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) { 6727 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 6728 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 6729 avail[i].limit = avail[i].base + 6730 (G_EXT_MEM1_SIZE(hi) << 20); 6731 avail[i].idx = 4; 6732 i++; 6733 } 6734 if (!i) /* no memory available */ 6735 return 0; 6736 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 6737 6738 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 6739 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 6740 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 6741 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 6742 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 6743 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 6744 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 6745 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 6746 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 6747 6748 /* the next few have explicit upper bounds */ 6749 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 6750 md->limit = md->base - 1 + 6751 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 6752 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 6753 md++; 6754 6755 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 6756 md->limit = md->base - 1 + 6757 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 6758 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 6759 md++; 6760 6761 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6762 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 6763 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 6764 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 6765 } else { 6766 md->base = 0; 6767 md->idx = nitems(region); /* hide it */ 6768 } 6769 md++; 6770 6771 #define ulp_region(reg) \ 6772 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 6773 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 6774 6775 ulp_region(RX_ISCSI); 6776 ulp_region(RX_TDDP); 6777 ulp_region(TX_TPT); 6778 ulp_region(RX_STAG); 6779 ulp_region(RX_RQ); 6780 ulp_region(RX_RQUDP); 6781 ulp_region(RX_PBL); 6782 ulp_region(TX_PBL); 6783 #undef ulp_region 6784 6785 md->base = 0; 6786 md->idx = nitems(region); 6787 if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) { 6788 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); 6789 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc, 6790 A_SGE_DBVFIFO_SIZE))) << 2) - 1; 6791 } 6792 md++; 6793 6794 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 6795 md->limit = md->base + sc->tids.ntids - 1; 6796 md++; 6797 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 6798 md->limit = md->base + sc->tids.ntids - 1; 6799 md++; 6800 6801 md->base = sc->vres.ocq.start; 6802 if (sc->vres.ocq.size) 6803 md->limit = md->base + sc->vres.ocq.size - 1; 6804 else 6805 md->idx = nitems(region); /* hide it */ 6806 md++; 6807 6808 /* add any address-space holes, there can be up to 3 */ 6809 for (n = 0; n < i - 1; n++) 6810 if (avail[n].limit < avail[n + 1].base) 6811 (md++)->base = avail[n].limit; 6812 if (avail[n].limit) 6813 (md++)->base = avail[n].limit; 6814 6815 n = md - mem; 6816 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 6817 6818 for (lo = 0; lo < i; lo++) 6819 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 6820 avail[lo].limit - 1); 6821 6822 sbuf_printf(sb, "\n"); 6823 for (i = 0; i < n; i++) { 6824 if (mem[i].idx >= nitems(region)) 6825 continue; /* skip holes */ 6826 if (!mem[i].limit) 6827 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 6828 mem_region_show(sb, region[mem[i].idx], mem[i].base, 6829 mem[i].limit); 6830 } 6831 6832 sbuf_printf(sb, "\n"); 6833 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 6834 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 6835 mem_region_show(sb, "uP RAM:", lo, hi); 6836 6837 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 6838 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 6839 mem_region_show(sb, "uP Extmem2:", lo, hi); 6840 6841 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 6842 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 6843 G_PMRXMAXPAGE(lo), 6844 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 6845 (lo & F_PMRXNUMCHN) ? 2 : 1); 6846 6847 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 6848 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 6849 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 6850 G_PMTXMAXPAGE(lo), 6851 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 6852 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 6853 sbuf_printf(sb, "%u p-structs\n", 6854 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 6855 6856 for (i = 0; i < 4; i++) { 6857 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 6858 if (is_t4(sc)) { 6859 used = G_USED(lo); 6860 alloc = G_ALLOC(lo); 6861 } else { 6862 used = G_T5_USED(lo); 6863 alloc = G_T5_ALLOC(lo); 6864 } 6865 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 6866 i, used, alloc); 6867 } 6868 for (i = 0; i < 4; i++) { 6869 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 6870 if (is_t4(sc)) { 6871 used = G_USED(lo); 6872 alloc = G_ALLOC(lo); 6873 } else { 6874 used = G_T5_USED(lo); 6875 alloc = G_T5_ALLOC(lo); 6876 } 6877 sbuf_printf(sb, 6878 "\nLoopback %d using %u pages out of %u allocated", 6879 i, used, alloc); 6880 } 6881 6882 rc = sbuf_finish(sb); 6883 sbuf_delete(sb); 6884 6885 return (rc); 6886 } 6887 6888 static inline void 6889 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) 6890 { 6891 *mask = x | y; 6892 y = htobe64(y); 6893 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); 6894 } 6895 6896 static int 6897 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) 6898 { 6899 struct adapter *sc = arg1; 6900 struct sbuf *sb; 6901 int rc, i, n; 6902 6903 rc = sysctl_wire_old_buffer(req, 0); 6904 if (rc != 0) 6905 return (rc); 6906 6907 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6908 if (sb == NULL) 6909 return (ENOMEM); 6910 6911 sbuf_printf(sb, 6912 "Idx Ethernet address Mask Vld Ports PF" 6913 " VF Replication P0 P1 P2 P3 ML"); 6914 n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES : 6915 NUM_MPS_T5_CLS_SRAM_L_INSTANCES; 6916 for (i = 0; i < n; i++) { 6917 uint64_t tcamx, tcamy, mask; 6918 uint32_t cls_lo, cls_hi; 6919 uint8_t addr[ETHER_ADDR_LEN]; 6920 6921 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); 6922 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); 6923 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 6924 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 6925 6926 if (tcamx & tcamy) 6927 continue; 6928 6929 tcamxy2valmask(tcamx, tcamy, addr, &mask); 6930 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" 6931 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], 6932 addr[3], addr[4], addr[5], (uintmax_t)mask, 6933 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', 6934 G_PORTMAP(cls_hi), G_PF(cls_lo), 6935 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); 6936 6937 if (cls_lo & F_REPLICATE) { 6938 struct fw_ldst_cmd ldst_cmd; 6939 6940 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 6941 ldst_cmd.op_to_addrspace = 6942 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 6943 F_FW_CMD_REQUEST | F_FW_CMD_READ | 6944 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 6945 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 6946 ldst_cmd.u.mps.rplc.fid_idx = 6947 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 6948 V_FW_LDST_CMD_IDX(i)); 6949 6950 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 6951 "t4mps"); 6952 if (rc) 6953 break; 6954 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 6955 sizeof(ldst_cmd), &ldst_cmd); 6956 end_synchronized_op(sc, 0); 6957 6958 if (rc != 0) { 6959 sbuf_printf(sb, 6960 " ------------ error %3u ------------", rc); 6961 rc = 0; 6962 } else { 6963 sbuf_printf(sb, " %08x %08x %08x %08x", 6964 be32toh(ldst_cmd.u.mps.rplc.rplc127_96), 6965 be32toh(ldst_cmd.u.mps.rplc.rplc95_64), 6966 be32toh(ldst_cmd.u.mps.rplc.rplc63_32), 6967 be32toh(ldst_cmd.u.mps.rplc.rplc31_0)); 6968 } 6969 } else 6970 sbuf_printf(sb, "%36s", ""); 6971 6972 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), 6973 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), 6974 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); 6975 } 6976 6977 if (rc) 6978 (void) sbuf_finish(sb); 6979 else 6980 rc = sbuf_finish(sb); 6981 sbuf_delete(sb); 6982 6983 return (rc); 6984 } 6985 6986 static int 6987 sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 6988 { 6989 struct adapter *sc = arg1; 6990 struct sbuf *sb; 6991 int rc; 6992 uint16_t mtus[NMTUS]; 6993 6994 rc = sysctl_wire_old_buffer(req, 0); 6995 if (rc != 0) 6996 return (rc); 6997 6998 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6999 if (sb == NULL) 7000 return (ENOMEM); 7001 7002 t4_read_mtu_tbl(sc, mtus, NULL); 7003 7004 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 7005 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 7006 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 7007 mtus[14], mtus[15]); 7008 7009 rc = sbuf_finish(sb); 7010 sbuf_delete(sb); 7011 7012 return (rc); 7013 } 7014 7015 static int 7016 sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 7017 { 7018 struct adapter *sc = arg1; 7019 struct sbuf *sb; 7020 int rc, i; 7021 uint32_t cnt[PM_NSTATS]; 7022 uint64_t cyc[PM_NSTATS]; 7023 static const char *rx_stats[] = { 7024 "Read:", "Write bypass:", "Write mem:", "Flush:" 7025 }; 7026 static const char *tx_stats[] = { 7027 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:" 7028 }; 7029 7030 rc = sysctl_wire_old_buffer(req, 0); 7031 if (rc != 0) 7032 return (rc); 7033 7034 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7035 if (sb == NULL) 7036 return (ENOMEM); 7037 7038 t4_pmtx_get_stats(sc, cnt, cyc); 7039 sbuf_printf(sb, " Tx pcmds Tx bytes"); 7040 for (i = 0; i < ARRAY_SIZE(tx_stats); i++) 7041 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i], 7042 cyc[i]); 7043 7044 t4_pmrx_get_stats(sc, cnt, cyc); 7045 sbuf_printf(sb, "\n Rx pcmds Rx bytes"); 7046 for (i = 0; i < ARRAY_SIZE(rx_stats); i++) 7047 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i], 7048 cyc[i]); 7049 7050 rc = sbuf_finish(sb); 7051 sbuf_delete(sb); 7052 7053 return (rc); 7054 } 7055 7056 static int 7057 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 7058 { 7059 struct adapter *sc = arg1; 7060 struct sbuf *sb; 7061 int rc; 7062 struct tp_rdma_stats stats; 7063 7064 rc = sysctl_wire_old_buffer(req, 0); 7065 if (rc != 0) 7066 return (rc); 7067 7068 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7069 if (sb == NULL) 7070 return (ENOMEM); 7071 7072 t4_tp_get_rdma_stats(sc, &stats); 7073 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 7074 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 7075 7076 rc = sbuf_finish(sb); 7077 sbuf_delete(sb); 7078 7079 return (rc); 7080 } 7081 7082 static int 7083 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 7084 { 7085 struct adapter *sc = arg1; 7086 struct sbuf *sb; 7087 int rc; 7088 struct tp_tcp_stats v4, v6; 7089 7090 rc = sysctl_wire_old_buffer(req, 0); 7091 if (rc != 0) 7092 return (rc); 7093 7094 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7095 if (sb == NULL) 7096 return (ENOMEM); 7097 7098 t4_tp_get_tcp_stats(sc, &v4, &v6); 7099 sbuf_printf(sb, 7100 " IP IPv6\n"); 7101 sbuf_printf(sb, "OutRsts: %20u %20u\n", 7102 v4.tcpOutRsts, v6.tcpOutRsts); 7103 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 7104 v4.tcpInSegs, v6.tcpInSegs); 7105 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 7106 v4.tcpOutSegs, v6.tcpOutSegs); 7107 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 7108 v4.tcpRetransSegs, v6.tcpRetransSegs); 7109 7110 rc = sbuf_finish(sb); 7111 sbuf_delete(sb); 7112 7113 return (rc); 7114 } 7115 7116 static int 7117 sysctl_tids(SYSCTL_HANDLER_ARGS) 7118 { 7119 struct adapter *sc = arg1; 7120 struct sbuf *sb; 7121 int rc; 7122 struct tid_info *t = &sc->tids; 7123 7124 rc = sysctl_wire_old_buffer(req, 0); 7125 if (rc != 0) 7126 return (rc); 7127 7128 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7129 if (sb == NULL) 7130 return (ENOMEM); 7131 7132 if (t->natids) { 7133 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 7134 t->atids_in_use); 7135 } 7136 7137 if (t->ntids) { 7138 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 7139 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 7140 7141 if (b) { 7142 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 7143 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 7144 t->ntids - 1); 7145 } else { 7146 sbuf_printf(sb, "TID range: %u-%u", 7147 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 7148 t->ntids - 1); 7149 } 7150 } else 7151 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 7152 sbuf_printf(sb, ", in use: %u\n", 7153 atomic_load_acq_int(&t->tids_in_use)); 7154 } 7155 7156 if (t->nstids) { 7157 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 7158 t->stid_base + t->nstids - 1, t->stids_in_use); 7159 } 7160 7161 if (t->nftids) { 7162 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 7163 t->ftid_base + t->nftids - 1); 7164 } 7165 7166 if (t->netids) { 7167 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base, 7168 t->etid_base + t->netids - 1); 7169 } 7170 7171 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 7172 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 7173 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 7174 7175 rc = sbuf_finish(sb); 7176 sbuf_delete(sb); 7177 7178 return (rc); 7179 } 7180 7181 static int 7182 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 7183 { 7184 struct adapter *sc = arg1; 7185 struct sbuf *sb; 7186 int rc; 7187 struct tp_err_stats stats; 7188 7189 rc = sysctl_wire_old_buffer(req, 0); 7190 if (rc != 0) 7191 return (rc); 7192 7193 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7194 if (sb == NULL) 7195 return (ENOMEM); 7196 7197 t4_tp_get_err_stats(sc, &stats); 7198 7199 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 7200 "channel 3\n"); 7201 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 7202 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 7203 stats.macInErrs[3]); 7204 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 7205 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 7206 stats.hdrInErrs[3]); 7207 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 7208 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 7209 stats.tcpInErrs[3]); 7210 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 7211 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 7212 stats.tcp6InErrs[3]); 7213 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 7214 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 7215 stats.tnlCongDrops[3]); 7216 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 7217 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 7218 stats.tnlTxDrops[3]); 7219 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 7220 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 7221 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 7222 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 7223 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 7224 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 7225 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 7226 stats.ofldNoNeigh, stats.ofldCongDefer); 7227 7228 rc = sbuf_finish(sb); 7229 sbuf_delete(sb); 7230 7231 return (rc); 7232 } 7233 7234 struct field_desc { 7235 const char *name; 7236 u_int start; 7237 u_int width; 7238 }; 7239 7240 static void 7241 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) 7242 { 7243 char buf[32]; 7244 int line_size = 0; 7245 7246 while (f->name) { 7247 uint64_t mask = (1ULL << f->width) - 1; 7248 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, 7249 ((uintmax_t)v >> f->start) & mask); 7250 7251 if (line_size + len >= 79) { 7252 line_size = 8; 7253 sbuf_printf(sb, "\n "); 7254 } 7255 sbuf_printf(sb, "%s ", buf); 7256 line_size += len + 1; 7257 f++; 7258 } 7259 sbuf_printf(sb, "\n"); 7260 } 7261 7262 static struct field_desc tp_la0[] = { 7263 { "RcfOpCodeOut", 60, 4 }, 7264 { "State", 56, 4 }, 7265 { "WcfState", 52, 4 }, 7266 { "RcfOpcSrcOut", 50, 2 }, 7267 { "CRxError", 49, 1 }, 7268 { "ERxError", 48, 1 }, 7269 { "SanityFailed", 47, 1 }, 7270 { "SpuriousMsg", 46, 1 }, 7271 { "FlushInputMsg", 45, 1 }, 7272 { "FlushInputCpl", 44, 1 }, 7273 { "RssUpBit", 43, 1 }, 7274 { "RssFilterHit", 42, 1 }, 7275 { "Tid", 32, 10 }, 7276 { "InitTcb", 31, 1 }, 7277 { "LineNumber", 24, 7 }, 7278 { "Emsg", 23, 1 }, 7279 { "EdataOut", 22, 1 }, 7280 { "Cmsg", 21, 1 }, 7281 { "CdataOut", 20, 1 }, 7282 { "EreadPdu", 19, 1 }, 7283 { "CreadPdu", 18, 1 }, 7284 { "TunnelPkt", 17, 1 }, 7285 { "RcfPeerFin", 16, 1 }, 7286 { "RcfReasonOut", 12, 4 }, 7287 { "TxCchannel", 10, 2 }, 7288 { "RcfTxChannel", 8, 2 }, 7289 { "RxEchannel", 6, 2 }, 7290 { "RcfRxChannel", 5, 1 }, 7291 { "RcfDataOutSrdy", 4, 1 }, 7292 { "RxDvld", 3, 1 }, 7293 { "RxOoDvld", 2, 1 }, 7294 { "RxCongestion", 1, 1 }, 7295 { "TxCongestion", 0, 1 }, 7296 { NULL } 7297 }; 7298 7299 static struct field_desc tp_la1[] = { 7300 { "CplCmdIn", 56, 8 }, 7301 { "CplCmdOut", 48, 8 }, 7302 { "ESynOut", 47, 1 }, 7303 { "EAckOut", 46, 1 }, 7304 { "EFinOut", 45, 1 }, 7305 { "ERstOut", 44, 1 }, 7306 { "SynIn", 43, 1 }, 7307 { "AckIn", 42, 1 }, 7308 { "FinIn", 41, 1 }, 7309 { "RstIn", 40, 1 }, 7310 { "DataIn", 39, 1 }, 7311 { "DataInVld", 38, 1 }, 7312 { "PadIn", 37, 1 }, 7313 { "RxBufEmpty", 36, 1 }, 7314 { "RxDdp", 35, 1 }, 7315 { "RxFbCongestion", 34, 1 }, 7316 { "TxFbCongestion", 33, 1 }, 7317 { "TxPktSumSrdy", 32, 1 }, 7318 { "RcfUlpType", 28, 4 }, 7319 { "Eread", 27, 1 }, 7320 { "Ebypass", 26, 1 }, 7321 { "Esave", 25, 1 }, 7322 { "Static0", 24, 1 }, 7323 { "Cread", 23, 1 }, 7324 { "Cbypass", 22, 1 }, 7325 { "Csave", 21, 1 }, 7326 { "CPktOut", 20, 1 }, 7327 { "RxPagePoolFull", 18, 2 }, 7328 { "RxLpbkPkt", 17, 1 }, 7329 { "TxLpbkPkt", 16, 1 }, 7330 { "RxVfValid", 15, 1 }, 7331 { "SynLearned", 14, 1 }, 7332 { "SetDelEntry", 13, 1 }, 7333 { "SetInvEntry", 12, 1 }, 7334 { "CpcmdDvld", 11, 1 }, 7335 { "CpcmdSave", 10, 1 }, 7336 { "RxPstructsFull", 8, 2 }, 7337 { "EpcmdDvld", 7, 1 }, 7338 { "EpcmdFlush", 6, 1 }, 7339 { "EpcmdTrimPrefix", 5, 1 }, 7340 { "EpcmdTrimPostfix", 4, 1 }, 7341 { "ERssIp4Pkt", 3, 1 }, 7342 { "ERssIp6Pkt", 2, 1 }, 7343 { "ERssTcpUdpPkt", 1, 1 }, 7344 { "ERssFceFipPkt", 0, 1 }, 7345 { NULL } 7346 }; 7347 7348 static struct field_desc tp_la2[] = { 7349 { "CplCmdIn", 56, 8 }, 7350 { "MpsVfVld", 55, 1 }, 7351 { "MpsPf", 52, 3 }, 7352 { "MpsVf", 44, 8 }, 7353 { "SynIn", 43, 1 }, 7354 { "AckIn", 42, 1 }, 7355 { "FinIn", 41, 1 }, 7356 { "RstIn", 40, 1 }, 7357 { "DataIn", 39, 1 }, 7358 { "DataInVld", 38, 1 }, 7359 { "PadIn", 37, 1 }, 7360 { "RxBufEmpty", 36, 1 }, 7361 { "RxDdp", 35, 1 }, 7362 { "RxFbCongestion", 34, 1 }, 7363 { "TxFbCongestion", 33, 1 }, 7364 { "TxPktSumSrdy", 32, 1 }, 7365 { "RcfUlpType", 28, 4 }, 7366 { "Eread", 27, 1 }, 7367 { "Ebypass", 26, 1 }, 7368 { "Esave", 25, 1 }, 7369 { "Static0", 24, 1 }, 7370 { "Cread", 23, 1 }, 7371 { "Cbypass", 22, 1 }, 7372 { "Csave", 21, 1 }, 7373 { "CPktOut", 20, 1 }, 7374 { "RxPagePoolFull", 18, 2 }, 7375 { "RxLpbkPkt", 17, 1 }, 7376 { "TxLpbkPkt", 16, 1 }, 7377 { "RxVfValid", 15, 1 }, 7378 { "SynLearned", 14, 1 }, 7379 { "SetDelEntry", 13, 1 }, 7380 { "SetInvEntry", 12, 1 }, 7381 { "CpcmdDvld", 11, 1 }, 7382 { "CpcmdSave", 10, 1 }, 7383 { "RxPstructsFull", 8, 2 }, 7384 { "EpcmdDvld", 7, 1 }, 7385 { "EpcmdFlush", 6, 1 }, 7386 { "EpcmdTrimPrefix", 5, 1 }, 7387 { "EpcmdTrimPostfix", 4, 1 }, 7388 { "ERssIp4Pkt", 3, 1 }, 7389 { "ERssIp6Pkt", 2, 1 }, 7390 { "ERssTcpUdpPkt", 1, 1 }, 7391 { "ERssFceFipPkt", 0, 1 }, 7392 { NULL } 7393 }; 7394 7395 static void 7396 tp_la_show(struct sbuf *sb, uint64_t *p, int idx) 7397 { 7398 7399 field_desc_show(sb, *p, tp_la0); 7400 } 7401 7402 static void 7403 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) 7404 { 7405 7406 if (idx) 7407 sbuf_printf(sb, "\n"); 7408 field_desc_show(sb, p[0], tp_la0); 7409 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 7410 field_desc_show(sb, p[1], tp_la0); 7411 } 7412 7413 static void 7414 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) 7415 { 7416 7417 if (idx) 7418 sbuf_printf(sb, "\n"); 7419 field_desc_show(sb, p[0], tp_la0); 7420 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 7421 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); 7422 } 7423 7424 static int 7425 sysctl_tp_la(SYSCTL_HANDLER_ARGS) 7426 { 7427 struct adapter *sc = arg1; 7428 struct sbuf *sb; 7429 uint64_t *buf, *p; 7430 int rc; 7431 u_int i, inc; 7432 void (*show_func)(struct sbuf *, uint64_t *, int); 7433 7434 rc = sysctl_wire_old_buffer(req, 0); 7435 if (rc != 0) 7436 return (rc); 7437 7438 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7439 if (sb == NULL) 7440 return (ENOMEM); 7441 7442 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); 7443 7444 t4_tp_read_la(sc, buf, NULL); 7445 p = buf; 7446 7447 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { 7448 case 2: 7449 inc = 2; 7450 show_func = tp_la_show2; 7451 break; 7452 case 3: 7453 inc = 2; 7454 show_func = tp_la_show3; 7455 break; 7456 default: 7457 inc = 1; 7458 show_func = tp_la_show; 7459 } 7460 7461 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) 7462 (*show_func)(sb, p, i); 7463 7464 rc = sbuf_finish(sb); 7465 sbuf_delete(sb); 7466 free(buf, M_CXGBE); 7467 return (rc); 7468 } 7469 7470 static int 7471 sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 7472 { 7473 struct adapter *sc = arg1; 7474 struct sbuf *sb; 7475 int rc; 7476 u64 nrate[NCHAN], orate[NCHAN]; 7477 7478 rc = sysctl_wire_old_buffer(req, 0); 7479 if (rc != 0) 7480 return (rc); 7481 7482 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7483 if (sb == NULL) 7484 return (ENOMEM); 7485 7486 t4_get_chan_txrate(sc, nrate, orate); 7487 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 7488 "channel 3\n"); 7489 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 7490 nrate[0], nrate[1], nrate[2], nrate[3]); 7491 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 7492 orate[0], orate[1], orate[2], orate[3]); 7493 7494 rc = sbuf_finish(sb); 7495 sbuf_delete(sb); 7496 7497 return (rc); 7498 } 7499 7500 static int 7501 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) 7502 { 7503 struct adapter *sc = arg1; 7504 struct sbuf *sb; 7505 uint32_t *buf, *p; 7506 int rc, i; 7507 7508 rc = sysctl_wire_old_buffer(req, 0); 7509 if (rc != 0) 7510 return (rc); 7511 7512 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7513 if (sb == NULL) 7514 return (ENOMEM); 7515 7516 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, 7517 M_ZERO | M_WAITOK); 7518 7519 t4_ulprx_read_la(sc, buf); 7520 p = buf; 7521 7522 sbuf_printf(sb, " Pcmd Type Message" 7523 " Data"); 7524 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { 7525 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", 7526 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); 7527 } 7528 7529 rc = sbuf_finish(sb); 7530 sbuf_delete(sb); 7531 free(buf, M_CXGBE); 7532 return (rc); 7533 } 7534 7535 static int 7536 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 7537 { 7538 struct adapter *sc = arg1; 7539 struct sbuf *sb; 7540 int rc, v; 7541 7542 rc = sysctl_wire_old_buffer(req, 0); 7543 if (rc != 0) 7544 return (rc); 7545 7546 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7547 if (sb == NULL) 7548 return (ENOMEM); 7549 7550 v = t4_read_reg(sc, A_SGE_STAT_CFG); 7551 if (G_STATSOURCE_T5(v) == 7) { 7552 if (G_STATMODE(v) == 0) { 7553 sbuf_printf(sb, "total %d, incomplete %d", 7554 t4_read_reg(sc, A_SGE_STAT_TOTAL), 7555 t4_read_reg(sc, A_SGE_STAT_MATCH)); 7556 } else if (G_STATMODE(v) == 1) { 7557 sbuf_printf(sb, "total %d, data overflow %d", 7558 t4_read_reg(sc, A_SGE_STAT_TOTAL), 7559 t4_read_reg(sc, A_SGE_STAT_MATCH)); 7560 } 7561 } 7562 rc = sbuf_finish(sb); 7563 sbuf_delete(sb); 7564 7565 return (rc); 7566 } 7567 #endif 7568 7569 static uint32_t 7570 fconf_to_mode(uint32_t fconf) 7571 { 7572 uint32_t mode; 7573 7574 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 7575 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 7576 7577 if (fconf & F_FRAGMENTATION) 7578 mode |= T4_FILTER_IP_FRAGMENT; 7579 7580 if (fconf & F_MPSHITTYPE) 7581 mode |= T4_FILTER_MPS_HIT_TYPE; 7582 7583 if (fconf & F_MACMATCH) 7584 mode |= T4_FILTER_MAC_IDX; 7585 7586 if (fconf & F_ETHERTYPE) 7587 mode |= T4_FILTER_ETH_TYPE; 7588 7589 if (fconf & F_PROTOCOL) 7590 mode |= T4_FILTER_IP_PROTO; 7591 7592 if (fconf & F_TOS) 7593 mode |= T4_FILTER_IP_TOS; 7594 7595 if (fconf & F_VLAN) 7596 mode |= T4_FILTER_VLAN; 7597 7598 if (fconf & F_VNIC_ID) 7599 mode |= T4_FILTER_VNIC; 7600 7601 if (fconf & F_PORT) 7602 mode |= T4_FILTER_PORT; 7603 7604 if (fconf & F_FCOE) 7605 mode |= T4_FILTER_FCoE; 7606 7607 return (mode); 7608 } 7609 7610 static uint32_t 7611 mode_to_fconf(uint32_t mode) 7612 { 7613 uint32_t fconf = 0; 7614 7615 if (mode & T4_FILTER_IP_FRAGMENT) 7616 fconf |= F_FRAGMENTATION; 7617 7618 if (mode & T4_FILTER_MPS_HIT_TYPE) 7619 fconf |= F_MPSHITTYPE; 7620 7621 if (mode & T4_FILTER_MAC_IDX) 7622 fconf |= F_MACMATCH; 7623 7624 if (mode & T4_FILTER_ETH_TYPE) 7625 fconf |= F_ETHERTYPE; 7626 7627 if (mode & T4_FILTER_IP_PROTO) 7628 fconf |= F_PROTOCOL; 7629 7630 if (mode & T4_FILTER_IP_TOS) 7631 fconf |= F_TOS; 7632 7633 if (mode & T4_FILTER_VLAN) 7634 fconf |= F_VLAN; 7635 7636 if (mode & T4_FILTER_VNIC) 7637 fconf |= F_VNIC_ID; 7638 7639 if (mode & T4_FILTER_PORT) 7640 fconf |= F_PORT; 7641 7642 if (mode & T4_FILTER_FCoE) 7643 fconf |= F_FCOE; 7644 7645 return (fconf); 7646 } 7647 7648 static uint32_t 7649 fspec_to_fconf(struct t4_filter_specification *fs) 7650 { 7651 uint32_t fconf = 0; 7652 7653 if (fs->val.frag || fs->mask.frag) 7654 fconf |= F_FRAGMENTATION; 7655 7656 if (fs->val.matchtype || fs->mask.matchtype) 7657 fconf |= F_MPSHITTYPE; 7658 7659 if (fs->val.macidx || fs->mask.macidx) 7660 fconf |= F_MACMATCH; 7661 7662 if (fs->val.ethtype || fs->mask.ethtype) 7663 fconf |= F_ETHERTYPE; 7664 7665 if (fs->val.proto || fs->mask.proto) 7666 fconf |= F_PROTOCOL; 7667 7668 if (fs->val.tos || fs->mask.tos) 7669 fconf |= F_TOS; 7670 7671 if (fs->val.vlan_vld || fs->mask.vlan_vld) 7672 fconf |= F_VLAN; 7673 7674 if (fs->val.vnic_vld || fs->mask.vnic_vld) 7675 fconf |= F_VNIC_ID; 7676 7677 if (fs->val.iport || fs->mask.iport) 7678 fconf |= F_PORT; 7679 7680 if (fs->val.fcoe || fs->mask.fcoe) 7681 fconf |= F_FCOE; 7682 7683 return (fconf); 7684 } 7685 7686 static int 7687 get_filter_mode(struct adapter *sc, uint32_t *mode) 7688 { 7689 int rc; 7690 uint32_t fconf; 7691 7692 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7693 "t4getfm"); 7694 if (rc) 7695 return (rc); 7696 7697 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 7698 A_TP_VLAN_PRI_MAP); 7699 7700 if (sc->params.tp.vlan_pri_map != fconf) { 7701 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 7702 device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map, 7703 fconf); 7704 } 7705 7706 *mode = fconf_to_mode(fconf); 7707 7708 end_synchronized_op(sc, LOCK_HELD); 7709 return (0); 7710 } 7711 7712 static int 7713 set_filter_mode(struct adapter *sc, uint32_t mode) 7714 { 7715 uint32_t fconf; 7716 int rc; 7717 7718 fconf = mode_to_fconf(mode); 7719 7720 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7721 "t4setfm"); 7722 if (rc) 7723 return (rc); 7724 7725 if (sc->tids.ftids_in_use > 0) { 7726 rc = EBUSY; 7727 goto done; 7728 } 7729 7730 #ifdef TCP_OFFLOAD 7731 if (uld_active(sc, ULD_TOM)) { 7732 rc = EBUSY; 7733 goto done; 7734 } 7735 #endif 7736 7737 rc = -t4_set_filter_mode(sc, fconf); 7738 done: 7739 end_synchronized_op(sc, LOCK_HELD); 7740 return (rc); 7741 } 7742 7743 static inline uint64_t 7744 get_filter_hits(struct adapter *sc, uint32_t fid) 7745 { 7746 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 7747 uint64_t hits; 7748 7749 memwin_info(sc, 0, &mw_base, NULL); 7750 off = position_memwin(sc, 0, 7751 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 7752 if (is_t4(sc)) { 7753 hits = t4_read_reg64(sc, mw_base + off + 16); 7754 hits = be64toh(hits); 7755 } else { 7756 hits = t4_read_reg(sc, mw_base + off + 24); 7757 hits = be32toh(hits); 7758 } 7759 7760 return (hits); 7761 } 7762 7763 static int 7764 get_filter(struct adapter *sc, struct t4_filter *t) 7765 { 7766 int i, rc, nfilters = sc->tids.nftids; 7767 struct filter_entry *f; 7768 7769 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7770 "t4getf"); 7771 if (rc) 7772 return (rc); 7773 7774 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 7775 t->idx >= nfilters) { 7776 t->idx = 0xffffffff; 7777 goto done; 7778 } 7779 7780 f = &sc->tids.ftid_tab[t->idx]; 7781 for (i = t->idx; i < nfilters; i++, f++) { 7782 if (f->valid) { 7783 t->idx = i; 7784 t->l2tidx = f->l2t ? f->l2t->idx : 0; 7785 t->smtidx = f->smtidx; 7786 if (f->fs.hitcnts) 7787 t->hits = get_filter_hits(sc, t->idx); 7788 else 7789 t->hits = UINT64_MAX; 7790 t->fs = f->fs; 7791 7792 goto done; 7793 } 7794 } 7795 7796 t->idx = 0xffffffff; 7797 done: 7798 end_synchronized_op(sc, LOCK_HELD); 7799 return (0); 7800 } 7801 7802 static int 7803 set_filter(struct adapter *sc, struct t4_filter *t) 7804 { 7805 unsigned int nfilters, nports; 7806 struct filter_entry *f; 7807 int i, rc; 7808 7809 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 7810 if (rc) 7811 return (rc); 7812 7813 nfilters = sc->tids.nftids; 7814 nports = sc->params.nports; 7815 7816 if (nfilters == 0) { 7817 rc = ENOTSUP; 7818 goto done; 7819 } 7820 7821 if (!(sc->flags & FULL_INIT_DONE)) { 7822 rc = EAGAIN; 7823 goto done; 7824 } 7825 7826 if (t->idx >= nfilters) { 7827 rc = EINVAL; 7828 goto done; 7829 } 7830 7831 /* Validate against the global filter mode */ 7832 if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) != 7833 sc->params.tp.vlan_pri_map) { 7834 rc = E2BIG; 7835 goto done; 7836 } 7837 7838 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 7839 rc = EINVAL; 7840 goto done; 7841 } 7842 7843 if (t->fs.val.iport >= nports) { 7844 rc = EINVAL; 7845 goto done; 7846 } 7847 7848 /* Can't specify an iq if not steering to it */ 7849 if (!t->fs.dirsteer && t->fs.iq) { 7850 rc = EINVAL; 7851 goto done; 7852 } 7853 7854 /* IPv6 filter idx must be 4 aligned */ 7855 if (t->fs.type == 1 && 7856 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 7857 rc = EINVAL; 7858 goto done; 7859 } 7860 7861 if (sc->tids.ftid_tab == NULL) { 7862 KASSERT(sc->tids.ftids_in_use == 0, 7863 ("%s: no memory allocated but filters_in_use > 0", 7864 __func__)); 7865 7866 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 7867 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 7868 if (sc->tids.ftid_tab == NULL) { 7869 rc = ENOMEM; 7870 goto done; 7871 } 7872 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 7873 } 7874 7875 for (i = 0; i < 4; i++) { 7876 f = &sc->tids.ftid_tab[t->idx + i]; 7877 7878 if (f->pending || f->valid) { 7879 rc = EBUSY; 7880 goto done; 7881 } 7882 if (f->locked) { 7883 rc = EPERM; 7884 goto done; 7885 } 7886 7887 if (t->fs.type == 0) 7888 break; 7889 } 7890 7891 f = &sc->tids.ftid_tab[t->idx]; 7892 f->fs = t->fs; 7893 7894 rc = set_filter_wr(sc, t->idx); 7895 done: 7896 end_synchronized_op(sc, 0); 7897 7898 if (rc == 0) { 7899 mtx_lock(&sc->tids.ftid_lock); 7900 for (;;) { 7901 if (f->pending == 0) { 7902 rc = f->valid ? 0 : EIO; 7903 break; 7904 } 7905 7906 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7907 PCATCH, "t4setfw", 0)) { 7908 rc = EINPROGRESS; 7909 break; 7910 } 7911 } 7912 mtx_unlock(&sc->tids.ftid_lock); 7913 } 7914 return (rc); 7915 } 7916 7917 static int 7918 del_filter(struct adapter *sc, struct t4_filter *t) 7919 { 7920 unsigned int nfilters; 7921 struct filter_entry *f; 7922 int rc; 7923 7924 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 7925 if (rc) 7926 return (rc); 7927 7928 nfilters = sc->tids.nftids; 7929 7930 if (nfilters == 0) { 7931 rc = ENOTSUP; 7932 goto done; 7933 } 7934 7935 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 7936 t->idx >= nfilters) { 7937 rc = EINVAL; 7938 goto done; 7939 } 7940 7941 if (!(sc->flags & FULL_INIT_DONE)) { 7942 rc = EAGAIN; 7943 goto done; 7944 } 7945 7946 f = &sc->tids.ftid_tab[t->idx]; 7947 7948 if (f->pending) { 7949 rc = EBUSY; 7950 goto done; 7951 } 7952 if (f->locked) { 7953 rc = EPERM; 7954 goto done; 7955 } 7956 7957 if (f->valid) { 7958 t->fs = f->fs; /* extra info for the caller */ 7959 rc = del_filter_wr(sc, t->idx); 7960 } 7961 7962 done: 7963 end_synchronized_op(sc, 0); 7964 7965 if (rc == 0) { 7966 mtx_lock(&sc->tids.ftid_lock); 7967 for (;;) { 7968 if (f->pending == 0) { 7969 rc = f->valid ? EIO : 0; 7970 break; 7971 } 7972 7973 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7974 PCATCH, "t4delfw", 0)) { 7975 rc = EINPROGRESS; 7976 break; 7977 } 7978 } 7979 mtx_unlock(&sc->tids.ftid_lock); 7980 } 7981 7982 return (rc); 7983 } 7984 7985 static void 7986 clear_filter(struct filter_entry *f) 7987 { 7988 if (f->l2t) 7989 t4_l2t_release(f->l2t); 7990 7991 bzero(f, sizeof (*f)); 7992 } 7993 7994 static int 7995 set_filter_wr(struct adapter *sc, int fidx) 7996 { 7997 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7998 struct fw_filter_wr *fwr; 7999 unsigned int ftid; 8000 struct wrq_cookie cookie; 8001 8002 ASSERT_SYNCHRONIZED_OP(sc); 8003 8004 if (f->fs.newdmac || f->fs.newvlan) { 8005 /* This filter needs an L2T entry; allocate one. */ 8006 f->l2t = t4_l2t_alloc_switching(sc->l2t); 8007 if (f->l2t == NULL) 8008 return (EAGAIN); 8009 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 8010 f->fs.dmac)) { 8011 t4_l2t_release(f->l2t); 8012 f->l2t = NULL; 8013 return (ENOMEM); 8014 } 8015 } 8016 8017 ftid = sc->tids.ftid_base + fidx; 8018 8019 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 8020 if (fwr == NULL) 8021 return (ENOMEM); 8022 bzero(fwr, sizeof(*fwr)); 8023 8024 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 8025 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 8026 fwr->tid_to_iq = 8027 htobe32(V_FW_FILTER_WR_TID(ftid) | 8028 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 8029 V_FW_FILTER_WR_NOREPLY(0) | 8030 V_FW_FILTER_WR_IQ(f->fs.iq)); 8031 fwr->del_filter_to_l2tix = 8032 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 8033 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 8034 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 8035 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 8036 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 8037 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 8038 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 8039 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 8040 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 8041 f->fs.newvlan == VLAN_REWRITE) | 8042 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 8043 f->fs.newvlan == VLAN_REWRITE) | 8044 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 8045 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 8046 V_FW_FILTER_WR_PRIO(f->fs.prio) | 8047 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 8048 fwr->ethtype = htobe16(f->fs.val.ethtype); 8049 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 8050 fwr->frag_to_ovlan_vldm = 8051 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 8052 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 8053 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 8054 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 8055 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 8056 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 8057 fwr->smac_sel = 0; 8058 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 8059 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 8060 fwr->maci_to_matchtypem = 8061 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 8062 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 8063 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 8064 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 8065 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 8066 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 8067 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 8068 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 8069 fwr->ptcl = f->fs.val.proto; 8070 fwr->ptclm = f->fs.mask.proto; 8071 fwr->ttyp = f->fs.val.tos; 8072 fwr->ttypm = f->fs.mask.tos; 8073 fwr->ivlan = htobe16(f->fs.val.vlan); 8074 fwr->ivlanm = htobe16(f->fs.mask.vlan); 8075 fwr->ovlan = htobe16(f->fs.val.vnic); 8076 fwr->ovlanm = htobe16(f->fs.mask.vnic); 8077 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 8078 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 8079 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 8080 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 8081 fwr->lp = htobe16(f->fs.val.dport); 8082 fwr->lpm = htobe16(f->fs.mask.dport); 8083 fwr->fp = htobe16(f->fs.val.sport); 8084 fwr->fpm = htobe16(f->fs.mask.sport); 8085 if (f->fs.newsmac) 8086 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 8087 8088 f->pending = 1; 8089 sc->tids.ftids_in_use++; 8090 8091 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 8092 return (0); 8093 } 8094 8095 static int 8096 del_filter_wr(struct adapter *sc, int fidx) 8097 { 8098 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 8099 struct fw_filter_wr *fwr; 8100 unsigned int ftid; 8101 struct wrq_cookie cookie; 8102 8103 ftid = sc->tids.ftid_base + fidx; 8104 8105 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 8106 if (fwr == NULL) 8107 return (ENOMEM); 8108 bzero(fwr, sizeof (*fwr)); 8109 8110 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 8111 8112 f->pending = 1; 8113 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 8114 return (0); 8115 } 8116 8117 int 8118 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 8119 { 8120 struct adapter *sc = iq->adapter; 8121 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 8122 unsigned int idx = GET_TID(rpl); 8123 unsigned int rc; 8124 struct filter_entry *f; 8125 8126 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 8127 rss->opcode)); 8128 8129 if (is_ftid(sc, idx)) { 8130 8131 idx -= sc->tids.ftid_base; 8132 f = &sc->tids.ftid_tab[idx]; 8133 rc = G_COOKIE(rpl->cookie); 8134 8135 mtx_lock(&sc->tids.ftid_lock); 8136 if (rc == FW_FILTER_WR_FLT_ADDED) { 8137 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 8138 __func__, idx)); 8139 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 8140 f->pending = 0; /* asynchronous setup completed */ 8141 f->valid = 1; 8142 } else { 8143 if (rc != FW_FILTER_WR_FLT_DELETED) { 8144 /* Add or delete failed, display an error */ 8145 log(LOG_ERR, 8146 "filter %u setup failed with error %u\n", 8147 idx, rc); 8148 } 8149 8150 clear_filter(f); 8151 sc->tids.ftids_in_use--; 8152 } 8153 wakeup(&sc->tids.ftid_tab); 8154 mtx_unlock(&sc->tids.ftid_lock); 8155 } 8156 8157 return (0); 8158 } 8159 8160 static int 8161 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 8162 { 8163 int rc; 8164 8165 if (cntxt->cid > M_CTXTQID) 8166 return (EINVAL); 8167 8168 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 8169 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 8170 return (EINVAL); 8171 8172 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 8173 if (rc) 8174 return (rc); 8175 8176 if (sc->flags & FW_OK) { 8177 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 8178 &cntxt->data[0]); 8179 if (rc == 0) 8180 goto done; 8181 } 8182 8183 /* 8184 * Read via firmware failed or wasn't even attempted. Read directly via 8185 * the backdoor. 8186 */ 8187 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 8188 done: 8189 end_synchronized_op(sc, 0); 8190 return (rc); 8191 } 8192 8193 static int 8194 load_fw(struct adapter *sc, struct t4_data *fw) 8195 { 8196 int rc; 8197 uint8_t *fw_data; 8198 8199 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 8200 if (rc) 8201 return (rc); 8202 8203 if (sc->flags & FULL_INIT_DONE) { 8204 rc = EBUSY; 8205 goto done; 8206 } 8207 8208 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 8209 if (fw_data == NULL) { 8210 rc = ENOMEM; 8211 goto done; 8212 } 8213 8214 rc = copyin(fw->data, fw_data, fw->len); 8215 if (rc == 0) 8216 rc = -t4_load_fw(sc, fw_data, fw->len); 8217 8218 free(fw_data, M_CXGBE); 8219 done: 8220 end_synchronized_op(sc, 0); 8221 return (rc); 8222 } 8223 8224 static int 8225 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 8226 { 8227 uint32_t addr, off, remaining, i, n; 8228 uint32_t *buf, *b; 8229 uint32_t mw_base, mw_aperture; 8230 int rc; 8231 uint8_t *dst; 8232 8233 rc = validate_mem_range(sc, mr->addr, mr->len); 8234 if (rc != 0) 8235 return (rc); 8236 8237 memwin_info(sc, win, &mw_base, &mw_aperture); 8238 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 8239 addr = mr->addr; 8240 remaining = mr->len; 8241 dst = (void *)mr->data; 8242 8243 while (remaining) { 8244 off = position_memwin(sc, win, addr); 8245 8246 /* number of bytes that we'll copy in the inner loop */ 8247 n = min(remaining, mw_aperture - off); 8248 for (i = 0; i < n; i += 4) 8249 *b++ = t4_read_reg(sc, mw_base + off + i); 8250 8251 rc = copyout(buf, dst, n); 8252 if (rc != 0) 8253 break; 8254 8255 b = buf; 8256 dst += n; 8257 remaining -= n; 8258 addr += n; 8259 } 8260 8261 free(buf, M_CXGBE); 8262 return (rc); 8263 } 8264 8265 static int 8266 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 8267 { 8268 int rc; 8269 8270 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 8271 return (EINVAL); 8272 8273 if (i2cd->len > sizeof(i2cd->data)) 8274 return (EFBIG); 8275 8276 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 8277 if (rc) 8278 return (rc); 8279 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 8280 i2cd->offset, i2cd->len, &i2cd->data[0]); 8281 end_synchronized_op(sc, 0); 8282 8283 return (rc); 8284 } 8285 8286 static int 8287 in_range(int val, int lo, int hi) 8288 { 8289 8290 return (val < 0 || (val <= hi && val >= lo)); 8291 } 8292 8293 static int 8294 set_sched_class(struct adapter *sc, struct t4_sched_params *p) 8295 { 8296 int fw_subcmd, fw_type, rc; 8297 8298 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc"); 8299 if (rc) 8300 return (rc); 8301 8302 if (!(sc->flags & FULL_INIT_DONE)) { 8303 rc = EAGAIN; 8304 goto done; 8305 } 8306 8307 /* 8308 * Translate the cxgbetool parameters into T4 firmware parameters. (The 8309 * sub-command and type are in common locations.) 8310 */ 8311 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG) 8312 fw_subcmd = FW_SCHED_SC_CONFIG; 8313 else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS) 8314 fw_subcmd = FW_SCHED_SC_PARAMS; 8315 else { 8316 rc = EINVAL; 8317 goto done; 8318 } 8319 if (p->type == SCHED_CLASS_TYPE_PACKET) 8320 fw_type = FW_SCHED_TYPE_PKTSCHED; 8321 else { 8322 rc = EINVAL; 8323 goto done; 8324 } 8325 8326 if (fw_subcmd == FW_SCHED_SC_CONFIG) { 8327 /* Vet our parameters ..*/ 8328 if (p->u.config.minmax < 0) { 8329 rc = EINVAL; 8330 goto done; 8331 } 8332 8333 /* And pass the request to the firmware ...*/ 8334 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1); 8335 goto done; 8336 } 8337 8338 if (fw_subcmd == FW_SCHED_SC_PARAMS) { 8339 int fw_level; 8340 int fw_mode; 8341 int fw_rateunit; 8342 int fw_ratemode; 8343 8344 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL) 8345 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL; 8346 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) 8347 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR; 8348 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) 8349 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL; 8350 else { 8351 rc = EINVAL; 8352 goto done; 8353 } 8354 8355 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS) 8356 fw_mode = FW_SCHED_PARAMS_MODE_CLASS; 8357 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW) 8358 fw_mode = FW_SCHED_PARAMS_MODE_FLOW; 8359 else { 8360 rc = EINVAL; 8361 goto done; 8362 } 8363 8364 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS) 8365 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 8366 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS) 8367 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE; 8368 else { 8369 rc = EINVAL; 8370 goto done; 8371 } 8372 8373 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL) 8374 fw_ratemode = FW_SCHED_PARAMS_RATE_REL; 8375 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS) 8376 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS; 8377 else { 8378 rc = EINVAL; 8379 goto done; 8380 } 8381 8382 /* Vet our parameters ... */ 8383 if (!in_range(p->u.params.channel, 0, 3) || 8384 !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) || 8385 !in_range(p->u.params.minrate, 0, 10000000) || 8386 !in_range(p->u.params.maxrate, 0, 10000000) || 8387 !in_range(p->u.params.weight, 0, 100)) { 8388 rc = ERANGE; 8389 goto done; 8390 } 8391 8392 /* 8393 * Translate any unset parameters into the firmware's 8394 * nomenclature and/or fail the call if the parameters 8395 * are required ... 8396 */ 8397 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 || 8398 p->u.params.channel < 0 || p->u.params.cl < 0) { 8399 rc = EINVAL; 8400 goto done; 8401 } 8402 if (p->u.params.minrate < 0) 8403 p->u.params.minrate = 0; 8404 if (p->u.params.maxrate < 0) { 8405 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 8406 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 8407 rc = EINVAL; 8408 goto done; 8409 } else 8410 p->u.params.maxrate = 0; 8411 } 8412 if (p->u.params.weight < 0) { 8413 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) { 8414 rc = EINVAL; 8415 goto done; 8416 } else 8417 p->u.params.weight = 0; 8418 } 8419 if (p->u.params.pktsize < 0) { 8420 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 8421 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 8422 rc = EINVAL; 8423 goto done; 8424 } else 8425 p->u.params.pktsize = 0; 8426 } 8427 8428 /* See what the firmware thinks of the request ... */ 8429 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode, 8430 fw_rateunit, fw_ratemode, p->u.params.channel, 8431 p->u.params.cl, p->u.params.minrate, p->u.params.maxrate, 8432 p->u.params.weight, p->u.params.pktsize, 1); 8433 goto done; 8434 } 8435 8436 rc = EINVAL; 8437 done: 8438 end_synchronized_op(sc, 0); 8439 return (rc); 8440 } 8441 8442 static int 8443 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p) 8444 { 8445 struct port_info *pi = NULL; 8446 struct vi_info *vi; 8447 struct sge_txq *txq; 8448 uint32_t fw_mnem, fw_queue, fw_class; 8449 int i, rc; 8450 8451 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq"); 8452 if (rc) 8453 return (rc); 8454 8455 if (!(sc->flags & FULL_INIT_DONE)) { 8456 rc = EAGAIN; 8457 goto done; 8458 } 8459 8460 if (p->port >= sc->params.nports) { 8461 rc = EINVAL; 8462 goto done; 8463 } 8464 8465 /* XXX: Only supported for the main VI. */ 8466 pi = sc->port[p->port]; 8467 vi = &pi->vi[0]; 8468 if (!in_range(p->queue, 0, vi->ntxq - 1) || !in_range(p->cl, 0, 7)) { 8469 rc = EINVAL; 8470 goto done; 8471 } 8472 8473 /* 8474 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX 8475 * Scheduling Class in this case). 8476 */ 8477 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 8478 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH)); 8479 fw_class = p->cl < 0 ? 0xffffffff : p->cl; 8480 8481 /* 8482 * If op.queue is non-negative, then we're only changing the scheduling 8483 * on a single specified TX queue. 8484 */ 8485 if (p->queue >= 0) { 8486 txq = &sc->sge.txq[vi->first_txq + p->queue]; 8487 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 8488 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 8489 &fw_class); 8490 goto done; 8491 } 8492 8493 /* 8494 * Change the scheduling on all the TX queues for the 8495 * interface. 8496 */ 8497 for_each_txq(vi, i, txq) { 8498 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 8499 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 8500 &fw_class); 8501 if (rc) 8502 goto done; 8503 } 8504 8505 rc = 0; 8506 done: 8507 end_synchronized_op(sc, 0); 8508 return (rc); 8509 } 8510 8511 int 8512 t4_os_find_pci_capability(struct adapter *sc, int cap) 8513 { 8514 int i; 8515 8516 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 8517 } 8518 8519 int 8520 t4_os_pci_save_state(struct adapter *sc) 8521 { 8522 device_t dev; 8523 struct pci_devinfo *dinfo; 8524 8525 dev = sc->dev; 8526 dinfo = device_get_ivars(dev); 8527 8528 pci_cfg_save(dev, dinfo, 0); 8529 return (0); 8530 } 8531 8532 int 8533 t4_os_pci_restore_state(struct adapter *sc) 8534 { 8535 device_t dev; 8536 struct pci_devinfo *dinfo; 8537 8538 dev = sc->dev; 8539 dinfo = device_get_ivars(dev); 8540 8541 pci_cfg_restore(dev, dinfo); 8542 return (0); 8543 } 8544 8545 void 8546 t4_os_portmod_changed(const struct adapter *sc, int idx) 8547 { 8548 struct port_info *pi = sc->port[idx]; 8549 struct vi_info *vi; 8550 struct ifnet *ifp; 8551 int v; 8552 static const char *mod_str[] = { 8553 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 8554 }; 8555 8556 for_each_vi(pi, v, vi) { 8557 build_medialist(pi, &vi->media); 8558 } 8559 8560 ifp = pi->vi[0].ifp; 8561 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 8562 if_printf(ifp, "transceiver unplugged.\n"); 8563 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 8564 if_printf(ifp, "unknown transceiver inserted.\n"); 8565 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 8566 if_printf(ifp, "unsupported transceiver inserted.\n"); 8567 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 8568 if_printf(ifp, "%s transceiver inserted.\n", 8569 mod_str[pi->mod_type]); 8570 } else { 8571 if_printf(ifp, "transceiver (type %d) inserted.\n", 8572 pi->mod_type); 8573 } 8574 } 8575 8576 void 8577 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason) 8578 { 8579 struct port_info *pi = sc->port[idx]; 8580 struct vi_info *vi; 8581 struct ifnet *ifp; 8582 int v; 8583 8584 if (link_stat) 8585 pi->linkdnrc = -1; 8586 else { 8587 if (reason >= 0) 8588 pi->linkdnrc = reason; 8589 } 8590 for_each_vi(pi, v, vi) { 8591 ifp = vi->ifp; 8592 if (ifp == NULL) 8593 continue; 8594 8595 if (link_stat) { 8596 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 8597 if_link_state_change(ifp, LINK_STATE_UP); 8598 } else { 8599 if_link_state_change(ifp, LINK_STATE_DOWN); 8600 } 8601 } 8602 } 8603 8604 void 8605 t4_iterate(void (*func)(struct adapter *, void *), void *arg) 8606 { 8607 struct adapter *sc; 8608 8609 sx_slock(&t4_list_lock); 8610 SLIST_FOREACH(sc, &t4_list, link) { 8611 /* 8612 * func should not make any assumptions about what state sc is 8613 * in - the only guarantee is that sc->sc_lock is a valid lock. 8614 */ 8615 func(sc, arg); 8616 } 8617 sx_sunlock(&t4_list_lock); 8618 } 8619 8620 static int 8621 t4_open(struct cdev *dev, int flags, int type, struct thread *td) 8622 { 8623 return (0); 8624 } 8625 8626 static int 8627 t4_close(struct cdev *dev, int flags, int type, struct thread *td) 8628 { 8629 return (0); 8630 } 8631 8632 static int 8633 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 8634 struct thread *td) 8635 { 8636 int rc; 8637 struct adapter *sc = dev->si_drv1; 8638 8639 rc = priv_check(td, PRIV_DRIVER); 8640 if (rc != 0) 8641 return (rc); 8642 8643 switch (cmd) { 8644 case CHELSIO_T4_GETREG: { 8645 struct t4_reg *edata = (struct t4_reg *)data; 8646 8647 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8648 return (EFAULT); 8649 8650 if (edata->size == 4) 8651 edata->val = t4_read_reg(sc, edata->addr); 8652 else if (edata->size == 8) 8653 edata->val = t4_read_reg64(sc, edata->addr); 8654 else 8655 return (EINVAL); 8656 8657 break; 8658 } 8659 case CHELSIO_T4_SETREG: { 8660 struct t4_reg *edata = (struct t4_reg *)data; 8661 8662 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8663 return (EFAULT); 8664 8665 if (edata->size == 4) { 8666 if (edata->val & 0xffffffff00000000) 8667 return (EINVAL); 8668 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 8669 } else if (edata->size == 8) 8670 t4_write_reg64(sc, edata->addr, edata->val); 8671 else 8672 return (EINVAL); 8673 break; 8674 } 8675 case CHELSIO_T4_REGDUMP: { 8676 struct t4_regdump *regs = (struct t4_regdump *)data; 8677 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 8678 uint8_t *buf; 8679 8680 if (regs->len < reglen) { 8681 regs->len = reglen; /* hint to the caller */ 8682 return (ENOBUFS); 8683 } 8684 8685 regs->len = reglen; 8686 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 8687 t4_get_regs(sc, regs, buf); 8688 rc = copyout(buf, regs->data, reglen); 8689 free(buf, M_CXGBE); 8690 break; 8691 } 8692 case CHELSIO_T4_GET_FILTER_MODE: 8693 rc = get_filter_mode(sc, (uint32_t *)data); 8694 break; 8695 case CHELSIO_T4_SET_FILTER_MODE: 8696 rc = set_filter_mode(sc, *(uint32_t *)data); 8697 break; 8698 case CHELSIO_T4_GET_FILTER: 8699 rc = get_filter(sc, (struct t4_filter *)data); 8700 break; 8701 case CHELSIO_T4_SET_FILTER: 8702 rc = set_filter(sc, (struct t4_filter *)data); 8703 break; 8704 case CHELSIO_T4_DEL_FILTER: 8705 rc = del_filter(sc, (struct t4_filter *)data); 8706 break; 8707 case CHELSIO_T4_GET_SGE_CONTEXT: 8708 rc = get_sge_context(sc, (struct t4_sge_context *)data); 8709 break; 8710 case CHELSIO_T4_LOAD_FW: 8711 rc = load_fw(sc, (struct t4_data *)data); 8712 break; 8713 case CHELSIO_T4_GET_MEM: 8714 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 8715 break; 8716 case CHELSIO_T4_GET_I2C: 8717 rc = read_i2c(sc, (struct t4_i2c_data *)data); 8718 break; 8719 case CHELSIO_T4_CLEAR_STATS: { 8720 int i, v; 8721 u_int port_id = *(uint32_t *)data; 8722 struct port_info *pi; 8723 struct vi_info *vi; 8724 8725 if (port_id >= sc->params.nports) 8726 return (EINVAL); 8727 pi = sc->port[port_id]; 8728 8729 /* MAC stats */ 8730 t4_clr_port_stats(sc, pi->tx_chan); 8731 pi->tx_parse_error = 0; 8732 mtx_lock(&sc->regwin_lock); 8733 for_each_vi(pi, v, vi) { 8734 if (vi->flags & VI_INIT_DONE) 8735 t4_clr_vi_stats(sc, vi->viid); 8736 } 8737 mtx_unlock(&sc->regwin_lock); 8738 8739 /* 8740 * Since this command accepts a port, clear stats for 8741 * all VIs on this port. 8742 */ 8743 for_each_vi(pi, v, vi) { 8744 if (vi->flags & VI_INIT_DONE) { 8745 struct sge_rxq *rxq; 8746 struct sge_txq *txq; 8747 struct sge_wrq *wrq; 8748 8749 if (vi->flags & VI_NETMAP) 8750 continue; 8751 8752 for_each_rxq(vi, i, rxq) { 8753 #if defined(INET) || defined(INET6) 8754 rxq->lro.lro_queued = 0; 8755 rxq->lro.lro_flushed = 0; 8756 #endif 8757 rxq->rxcsum = 0; 8758 rxq->vlan_extraction = 0; 8759 } 8760 8761 for_each_txq(vi, i, txq) { 8762 txq->txcsum = 0; 8763 txq->tso_wrs = 0; 8764 txq->vlan_insertion = 0; 8765 txq->imm_wrs = 0; 8766 txq->sgl_wrs = 0; 8767 txq->txpkt_wrs = 0; 8768 txq->txpkts0_wrs = 0; 8769 txq->txpkts1_wrs = 0; 8770 txq->txpkts0_pkts = 0; 8771 txq->txpkts1_pkts = 0; 8772 mp_ring_reset_stats(txq->r); 8773 } 8774 8775 #ifdef TCP_OFFLOAD 8776 /* nothing to clear for each ofld_rxq */ 8777 8778 for_each_ofld_txq(vi, i, wrq) { 8779 wrq->tx_wrs_direct = 0; 8780 wrq->tx_wrs_copied = 0; 8781 } 8782 #endif 8783 8784 if (IS_MAIN_VI(vi)) { 8785 wrq = &sc->sge.ctrlq[pi->port_id]; 8786 wrq->tx_wrs_direct = 0; 8787 wrq->tx_wrs_copied = 0; 8788 } 8789 } 8790 } 8791 break; 8792 } 8793 case CHELSIO_T4_SCHED_CLASS: 8794 rc = set_sched_class(sc, (struct t4_sched_params *)data); 8795 break; 8796 case CHELSIO_T4_SCHED_QUEUE: 8797 rc = set_sched_queue(sc, (struct t4_sched_queue *)data); 8798 break; 8799 case CHELSIO_T4_GET_TRACER: 8800 rc = t4_get_tracer(sc, (struct t4_tracer *)data); 8801 break; 8802 case CHELSIO_T4_SET_TRACER: 8803 rc = t4_set_tracer(sc, (struct t4_tracer *)data); 8804 break; 8805 default: 8806 rc = EINVAL; 8807 } 8808 8809 return (rc); 8810 } 8811 8812 #ifdef TCP_OFFLOAD 8813 void 8814 t4_iscsi_init(struct adapter *sc, u_int tag_mask, const u_int *pgsz_order) 8815 { 8816 8817 t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask); 8818 t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) | 8819 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) | 8820 V_HPZ3(pgsz_order[3])); 8821 } 8822 8823 static int 8824 toe_capability(struct vi_info *vi, int enable) 8825 { 8826 int rc; 8827 struct port_info *pi = vi->pi; 8828 struct adapter *sc = pi->adapter; 8829 8830 ASSERT_SYNCHRONIZED_OP(sc); 8831 8832 if (!is_offload(sc)) 8833 return (ENODEV); 8834 8835 if (enable) { 8836 if ((vi->ifp->if_capenable & IFCAP_TOE) != 0) { 8837 /* TOE is already enabled. */ 8838 return (0); 8839 } 8840 8841 /* 8842 * We need the port's queues around so that we're able to send 8843 * and receive CPLs to/from the TOE even if the ifnet for this 8844 * port has never been UP'd administratively. 8845 */ 8846 if (!(vi->flags & VI_INIT_DONE)) { 8847 rc = cxgbe_init_synchronized(vi); 8848 if (rc) 8849 return (rc); 8850 } 8851 if (!(pi->vi[0].flags & VI_INIT_DONE)) { 8852 rc = cxgbe_init_synchronized(&pi->vi[0]); 8853 if (rc) 8854 return (rc); 8855 } 8856 8857 if (isset(&sc->offload_map, pi->port_id)) { 8858 /* TOE is enabled on another VI of this port. */ 8859 pi->uld_vis++; 8860 return (0); 8861 } 8862 8863 if (!uld_active(sc, ULD_TOM)) { 8864 rc = t4_activate_uld(sc, ULD_TOM); 8865 if (rc == EAGAIN) { 8866 log(LOG_WARNING, 8867 "You must kldload t4_tom.ko before trying " 8868 "to enable TOE on a cxgbe interface.\n"); 8869 } 8870 if (rc != 0) 8871 return (rc); 8872 KASSERT(sc->tom_softc != NULL, 8873 ("%s: TOM activated but softc NULL", __func__)); 8874 KASSERT(uld_active(sc, ULD_TOM), 8875 ("%s: TOM activated but flag not set", __func__)); 8876 } 8877 8878 /* Activate iWARP and iSCSI too, if the modules are loaded. */ 8879 if (!uld_active(sc, ULD_IWARP)) 8880 (void) t4_activate_uld(sc, ULD_IWARP); 8881 if (!uld_active(sc, ULD_ISCSI)) 8882 (void) t4_activate_uld(sc, ULD_ISCSI); 8883 8884 pi->uld_vis++; 8885 setbit(&sc->offload_map, pi->port_id); 8886 } else { 8887 pi->uld_vis--; 8888 8889 if (!isset(&sc->offload_map, pi->port_id) || pi->uld_vis > 0) 8890 return (0); 8891 8892 KASSERT(uld_active(sc, ULD_TOM), 8893 ("%s: TOM never initialized?", __func__)); 8894 clrbit(&sc->offload_map, pi->port_id); 8895 } 8896 8897 return (0); 8898 } 8899 8900 /* 8901 * Add an upper layer driver to the global list. 8902 */ 8903 int 8904 t4_register_uld(struct uld_info *ui) 8905 { 8906 int rc = 0; 8907 struct uld_info *u; 8908 8909 sx_xlock(&t4_uld_list_lock); 8910 SLIST_FOREACH(u, &t4_uld_list, link) { 8911 if (u->uld_id == ui->uld_id) { 8912 rc = EEXIST; 8913 goto done; 8914 } 8915 } 8916 8917 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 8918 ui->refcount = 0; 8919 done: 8920 sx_xunlock(&t4_uld_list_lock); 8921 return (rc); 8922 } 8923 8924 int 8925 t4_unregister_uld(struct uld_info *ui) 8926 { 8927 int rc = EINVAL; 8928 struct uld_info *u; 8929 8930 sx_xlock(&t4_uld_list_lock); 8931 8932 SLIST_FOREACH(u, &t4_uld_list, link) { 8933 if (u == ui) { 8934 if (ui->refcount > 0) { 8935 rc = EBUSY; 8936 goto done; 8937 } 8938 8939 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 8940 rc = 0; 8941 goto done; 8942 } 8943 } 8944 done: 8945 sx_xunlock(&t4_uld_list_lock); 8946 return (rc); 8947 } 8948 8949 int 8950 t4_activate_uld(struct adapter *sc, int id) 8951 { 8952 int rc; 8953 struct uld_info *ui; 8954 8955 ASSERT_SYNCHRONIZED_OP(sc); 8956 8957 if (id < 0 || id > ULD_MAX) 8958 return (EINVAL); 8959 rc = EAGAIN; /* kldoad the module with this ULD and try again. */ 8960 8961 sx_slock(&t4_uld_list_lock); 8962 8963 SLIST_FOREACH(ui, &t4_uld_list, link) { 8964 if (ui->uld_id == id) { 8965 if (!(sc->flags & FULL_INIT_DONE)) { 8966 rc = adapter_full_init(sc); 8967 if (rc != 0) 8968 break; 8969 } 8970 8971 rc = ui->activate(sc); 8972 if (rc == 0) { 8973 setbit(&sc->active_ulds, id); 8974 ui->refcount++; 8975 } 8976 break; 8977 } 8978 } 8979 8980 sx_sunlock(&t4_uld_list_lock); 8981 8982 return (rc); 8983 } 8984 8985 int 8986 t4_deactivate_uld(struct adapter *sc, int id) 8987 { 8988 int rc; 8989 struct uld_info *ui; 8990 8991 ASSERT_SYNCHRONIZED_OP(sc); 8992 8993 if (id < 0 || id > ULD_MAX) 8994 return (EINVAL); 8995 rc = ENXIO; 8996 8997 sx_slock(&t4_uld_list_lock); 8998 8999 SLIST_FOREACH(ui, &t4_uld_list, link) { 9000 if (ui->uld_id == id) { 9001 rc = ui->deactivate(sc); 9002 if (rc == 0) { 9003 clrbit(&sc->active_ulds, id); 9004 ui->refcount--; 9005 } 9006 break; 9007 } 9008 } 9009 9010 sx_sunlock(&t4_uld_list_lock); 9011 9012 return (rc); 9013 } 9014 9015 int 9016 uld_active(struct adapter *sc, int uld_id) 9017 { 9018 9019 MPASS(uld_id >= 0 && uld_id <= ULD_MAX); 9020 9021 return (isset(&sc->active_ulds, uld_id)); 9022 } 9023 #endif 9024 9025 /* 9026 * Come up with reasonable defaults for some of the tunables, provided they're 9027 * not set by the user (in which case we'll use the values as is). 9028 */ 9029 static void 9030 tweak_tunables(void) 9031 { 9032 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 9033 9034 if (t4_ntxq10g < 1) { 9035 #ifdef RSS 9036 t4_ntxq10g = rss_getnumbuckets(); 9037 #else 9038 t4_ntxq10g = min(nc, NTXQ_10G); 9039 #endif 9040 } 9041 9042 if (t4_ntxq1g < 1) { 9043 #ifdef RSS 9044 /* XXX: way too many for 1GbE? */ 9045 t4_ntxq1g = rss_getnumbuckets(); 9046 #else 9047 t4_ntxq1g = min(nc, NTXQ_1G); 9048 #endif 9049 } 9050 9051 if (t4_nrxq10g < 1) { 9052 #ifdef RSS 9053 t4_nrxq10g = rss_getnumbuckets(); 9054 #else 9055 t4_nrxq10g = min(nc, NRXQ_10G); 9056 #endif 9057 } 9058 9059 if (t4_nrxq1g < 1) { 9060 #ifdef RSS 9061 /* XXX: way too many for 1GbE? */ 9062 t4_nrxq1g = rss_getnumbuckets(); 9063 #else 9064 t4_nrxq1g = min(nc, NRXQ_1G); 9065 #endif 9066 } 9067 9068 #ifdef TCP_OFFLOAD 9069 if (t4_nofldtxq10g < 1) 9070 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 9071 9072 if (t4_nofldtxq1g < 1) 9073 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 9074 9075 if (t4_nofldrxq10g < 1) 9076 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 9077 9078 if (t4_nofldrxq1g < 1) 9079 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 9080 9081 if (t4_toecaps_allowed == -1) 9082 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 9083 #else 9084 if (t4_toecaps_allowed == -1) 9085 t4_toecaps_allowed = 0; 9086 #endif 9087 9088 #ifdef DEV_NETMAP 9089 if (t4_nnmtxq10g < 1) 9090 t4_nnmtxq10g = min(nc, NNMTXQ_10G); 9091 9092 if (t4_nnmtxq1g < 1) 9093 t4_nnmtxq1g = min(nc, NNMTXQ_1G); 9094 9095 if (t4_nnmrxq10g < 1) 9096 t4_nnmrxq10g = min(nc, NNMRXQ_10G); 9097 9098 if (t4_nnmrxq1g < 1) 9099 t4_nnmrxq1g = min(nc, NNMRXQ_1G); 9100 #endif 9101 9102 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 9103 t4_tmr_idx_10g = TMR_IDX_10G; 9104 9105 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 9106 t4_pktc_idx_10g = PKTC_IDX_10G; 9107 9108 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 9109 t4_tmr_idx_1g = TMR_IDX_1G; 9110 9111 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 9112 t4_pktc_idx_1g = PKTC_IDX_1G; 9113 9114 if (t4_qsize_txq < 128) 9115 t4_qsize_txq = 128; 9116 9117 if (t4_qsize_rxq < 128) 9118 t4_qsize_rxq = 128; 9119 while (t4_qsize_rxq & 7) 9120 t4_qsize_rxq++; 9121 9122 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 9123 } 9124 9125 static struct sx mlu; /* mod load unload */ 9126 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload"); 9127 9128 static int 9129 mod_event(module_t mod, int cmd, void *arg) 9130 { 9131 int rc = 0; 9132 static int loaded = 0; 9133 9134 switch (cmd) { 9135 case MOD_LOAD: 9136 sx_xlock(&mlu); 9137 if (loaded++ == 0) { 9138 t4_sge_modload(); 9139 sx_init(&t4_list_lock, "T4/T5 adapters"); 9140 SLIST_INIT(&t4_list); 9141 #ifdef TCP_OFFLOAD 9142 sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); 9143 SLIST_INIT(&t4_uld_list); 9144 #endif 9145 t4_tracer_modload(); 9146 tweak_tunables(); 9147 } 9148 sx_xunlock(&mlu); 9149 break; 9150 9151 case MOD_UNLOAD: 9152 sx_xlock(&mlu); 9153 if (--loaded == 0) { 9154 int tries; 9155 9156 sx_slock(&t4_list_lock); 9157 if (!SLIST_EMPTY(&t4_list)) { 9158 rc = EBUSY; 9159 sx_sunlock(&t4_list_lock); 9160 goto done_unload; 9161 } 9162 #ifdef TCP_OFFLOAD 9163 sx_slock(&t4_uld_list_lock); 9164 if (!SLIST_EMPTY(&t4_uld_list)) { 9165 rc = EBUSY; 9166 sx_sunlock(&t4_uld_list_lock); 9167 sx_sunlock(&t4_list_lock); 9168 goto done_unload; 9169 } 9170 #endif 9171 tries = 0; 9172 while (tries++ < 5 && t4_sge_extfree_refs() != 0) { 9173 uprintf("%ju clusters with custom free routine " 9174 "still is use.\n", t4_sge_extfree_refs()); 9175 pause("t4unload", 2 * hz); 9176 } 9177 #ifdef TCP_OFFLOAD 9178 sx_sunlock(&t4_uld_list_lock); 9179 #endif 9180 sx_sunlock(&t4_list_lock); 9181 9182 if (t4_sge_extfree_refs() == 0) { 9183 t4_tracer_modunload(); 9184 #ifdef TCP_OFFLOAD 9185 sx_destroy(&t4_uld_list_lock); 9186 #endif 9187 sx_destroy(&t4_list_lock); 9188 t4_sge_modunload(); 9189 loaded = 0; 9190 } else { 9191 rc = EBUSY; 9192 loaded++; /* undo earlier decrement */ 9193 } 9194 } 9195 done_unload: 9196 sx_xunlock(&mlu); 9197 break; 9198 } 9199 9200 return (rc); 9201 } 9202 9203 static devclass_t t4_devclass, t5_devclass; 9204 static devclass_t cxgbe_devclass, cxl_devclass; 9205 static devclass_t vcxgbe_devclass, vcxl_devclass; 9206 9207 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 9208 MODULE_VERSION(t4nex, 1); 9209 MODULE_DEPEND(t4nex, firmware, 1, 1, 1); 9210 #ifdef DEV_NETMAP 9211 MODULE_DEPEND(t4nex, netmap, 1, 1, 1); 9212 #endif /* DEV_NETMAP */ 9213 9214 9215 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 9216 MODULE_VERSION(t5nex, 1); 9217 MODULE_DEPEND(t5nex, firmware, 1, 1, 1); 9218 #ifdef DEV_NETMAP 9219 MODULE_DEPEND(t5nex, netmap, 1, 1, 1); 9220 #endif /* DEV_NETMAP */ 9221 9222 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 9223 MODULE_VERSION(cxgbe, 1); 9224 9225 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 9226 MODULE_VERSION(cxl, 1); 9227 9228 DRIVER_MODULE(vcxgbe, cxgbe, vcxgbe_driver, vcxgbe_devclass, 0, 0); 9229 MODULE_VERSION(vcxgbe, 1); 9230 9231 DRIVER_MODULE(vcxl, cxl, vcxl_driver, vcxl_devclass, 0, 0); 9232 MODULE_VERSION(vcxl, 1); 9233