1 /*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_inet.h" 32 #include "opt_inet6.h" 33 #include "opt_rss.h" 34 35 #include <sys/param.h> 36 #include <sys/conf.h> 37 #include <sys/priv.h> 38 #include <sys/kernel.h> 39 #include <sys/bus.h> 40 #include <sys/module.h> 41 #include <sys/malloc.h> 42 #include <sys/queue.h> 43 #include <sys/taskqueue.h> 44 #include <sys/pciio.h> 45 #include <dev/pci/pcireg.h> 46 #include <dev/pci/pcivar.h> 47 #include <dev/pci/pci_private.h> 48 #include <sys/firmware.h> 49 #include <sys/sbuf.h> 50 #include <sys/smp.h> 51 #include <sys/socket.h> 52 #include <sys/sockio.h> 53 #include <sys/sysctl.h> 54 #include <net/ethernet.h> 55 #include <net/if.h> 56 #include <net/if_types.h> 57 #include <net/if_dl.h> 58 #include <net/if_vlan_var.h> 59 #ifdef RSS 60 #include <net/rss_config.h> 61 #endif 62 #if defined(__i386__) || defined(__amd64__) 63 #include <vm/vm.h> 64 #include <vm/pmap.h> 65 #endif 66 67 #include "common/common.h" 68 #include "common/t4_msg.h" 69 #include "common/t4_regs.h" 70 #include "common/t4_regs_values.h" 71 #include "t4_ioctl.h" 72 #include "t4_l2t.h" 73 #include "t4_mp_ring.h" 74 75 /* T4 bus driver interface */ 76 static int t4_probe(device_t); 77 static int t4_attach(device_t); 78 static int t4_detach(device_t); 79 static device_method_t t4_methods[] = { 80 DEVMETHOD(device_probe, t4_probe), 81 DEVMETHOD(device_attach, t4_attach), 82 DEVMETHOD(device_detach, t4_detach), 83 84 DEVMETHOD_END 85 }; 86 static driver_t t4_driver = { 87 "t4nex", 88 t4_methods, 89 sizeof(struct adapter) 90 }; 91 92 93 /* T4 port (cxgbe) interface */ 94 static int cxgbe_probe(device_t); 95 static int cxgbe_attach(device_t); 96 static int cxgbe_detach(device_t); 97 static device_method_t cxgbe_methods[] = { 98 DEVMETHOD(device_probe, cxgbe_probe), 99 DEVMETHOD(device_attach, cxgbe_attach), 100 DEVMETHOD(device_detach, cxgbe_detach), 101 { 0, 0 } 102 }; 103 static driver_t cxgbe_driver = { 104 "cxgbe", 105 cxgbe_methods, 106 sizeof(struct port_info) 107 }; 108 109 /* T4 VI (vcxgbe) interface */ 110 static int vcxgbe_probe(device_t); 111 static int vcxgbe_attach(device_t); 112 static int vcxgbe_detach(device_t); 113 static device_method_t vcxgbe_methods[] = { 114 DEVMETHOD(device_probe, vcxgbe_probe), 115 DEVMETHOD(device_attach, vcxgbe_attach), 116 DEVMETHOD(device_detach, vcxgbe_detach), 117 { 0, 0 } 118 }; 119 static driver_t vcxgbe_driver = { 120 "vcxgbe", 121 vcxgbe_methods, 122 sizeof(struct vi_info) 123 }; 124 125 static d_ioctl_t t4_ioctl; 126 static d_open_t t4_open; 127 static d_close_t t4_close; 128 129 static struct cdevsw t4_cdevsw = { 130 .d_version = D_VERSION, 131 .d_flags = 0, 132 .d_open = t4_open, 133 .d_close = t4_close, 134 .d_ioctl = t4_ioctl, 135 .d_name = "t4nex", 136 }; 137 138 /* T5 bus driver interface */ 139 static int t5_probe(device_t); 140 static device_method_t t5_methods[] = { 141 DEVMETHOD(device_probe, t5_probe), 142 DEVMETHOD(device_attach, t4_attach), 143 DEVMETHOD(device_detach, t4_detach), 144 145 DEVMETHOD_END 146 }; 147 static driver_t t5_driver = { 148 "t5nex", 149 t5_methods, 150 sizeof(struct adapter) 151 }; 152 153 154 /* T5 port (cxl) interface */ 155 static driver_t cxl_driver = { 156 "cxl", 157 cxgbe_methods, 158 sizeof(struct port_info) 159 }; 160 161 /* T5 VI (vcxl) interface */ 162 static driver_t vcxl_driver = { 163 "vcxl", 164 vcxgbe_methods, 165 sizeof(struct vi_info) 166 }; 167 168 static struct cdevsw t5_cdevsw = { 169 .d_version = D_VERSION, 170 .d_flags = 0, 171 .d_open = t4_open, 172 .d_close = t4_close, 173 .d_ioctl = t4_ioctl, 174 .d_name = "t5nex", 175 }; 176 177 /* ifnet + media interface */ 178 static void cxgbe_init(void *); 179 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 180 static int cxgbe_transmit(struct ifnet *, struct mbuf *); 181 static void cxgbe_qflush(struct ifnet *); 182 static int cxgbe_media_change(struct ifnet *); 183 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 184 185 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 186 187 /* 188 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 189 * then ADAPTER_LOCK, then t4_uld_list_lock. 190 */ 191 static struct sx t4_list_lock; 192 SLIST_HEAD(, adapter) t4_list; 193 #ifdef TCP_OFFLOAD 194 static struct sx t4_uld_list_lock; 195 SLIST_HEAD(, uld_info) t4_uld_list; 196 #endif 197 198 /* 199 * Tunables. See tweak_tunables() too. 200 * 201 * Each tunable is set to a default value here if it's known at compile-time. 202 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 203 * provide a reasonable default when the driver is loaded. 204 * 205 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 206 * T5 are under hw.cxl. 207 */ 208 209 /* 210 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 211 */ 212 #define NTXQ_10G 16 213 static int t4_ntxq10g = -1; 214 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 215 216 #define NRXQ_10G 8 217 static int t4_nrxq10g = -1; 218 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 219 220 #define NTXQ_1G 4 221 static int t4_ntxq1g = -1; 222 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 223 224 #define NRXQ_1G 2 225 static int t4_nrxq1g = -1; 226 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 227 228 static int t4_rsrv_noflowq = 0; 229 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq); 230 231 #ifdef TCP_OFFLOAD 232 #define NOFLDTXQ_10G 8 233 static int t4_nofldtxq10g = -1; 234 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 235 236 #define NOFLDRXQ_10G 2 237 static int t4_nofldrxq10g = -1; 238 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 239 240 #define NOFLDTXQ_1G 2 241 static int t4_nofldtxq1g = -1; 242 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 243 244 #define NOFLDRXQ_1G 1 245 static int t4_nofldrxq1g = -1; 246 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 247 #endif 248 249 #ifdef DEV_NETMAP 250 #define NNMTXQ_10G 2 251 static int t4_nnmtxq10g = -1; 252 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g); 253 254 #define NNMRXQ_10G 2 255 static int t4_nnmrxq10g = -1; 256 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g); 257 258 #define NNMTXQ_1G 1 259 static int t4_nnmtxq1g = -1; 260 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g); 261 262 #define NNMRXQ_1G 1 263 static int t4_nnmrxq1g = -1; 264 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g); 265 #endif 266 267 /* 268 * Holdoff parameters for 10G and 1G ports. 269 */ 270 #define TMR_IDX_10G 1 271 static int t4_tmr_idx_10g = TMR_IDX_10G; 272 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 273 274 #define PKTC_IDX_10G (-1) 275 static int t4_pktc_idx_10g = PKTC_IDX_10G; 276 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 277 278 #define TMR_IDX_1G 1 279 static int t4_tmr_idx_1g = TMR_IDX_1G; 280 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 281 282 #define PKTC_IDX_1G (-1) 283 static int t4_pktc_idx_1g = PKTC_IDX_1G; 284 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 285 286 /* 287 * Size (# of entries) of each tx and rx queue. 288 */ 289 static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 290 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 291 292 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 293 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 294 295 /* 296 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 297 */ 298 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 299 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 300 301 /* 302 * Configuration file. 303 */ 304 #define DEFAULT_CF "default" 305 #define FLASH_CF "flash" 306 #define UWIRE_CF "uwire" 307 #define FPGA_CF "fpga" 308 static char t4_cfg_file[32] = DEFAULT_CF; 309 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 310 311 /* 312 * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively). 313 * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them. 314 * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water 315 * mark or when signalled to do so, 0 to never emit PAUSE. 316 */ 317 static int t4_pause_settings = PAUSE_TX | PAUSE_RX; 318 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings); 319 320 /* 321 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 322 * encouraged respectively). 323 */ 324 static unsigned int t4_fw_install = 1; 325 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 326 327 /* 328 * ASIC features that will be used. Disable the ones you don't want so that the 329 * chip resources aren't wasted on features that will not be used. 330 */ 331 static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 332 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 333 334 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 335 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 336 337 static int t4_toecaps_allowed = -1; 338 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 339 340 static int t4_rdmacaps_allowed = 0; 341 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 342 343 static int t4_iscsicaps_allowed = 0; 344 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 345 346 static int t4_fcoecaps_allowed = 0; 347 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 348 349 static int t5_write_combine = 0; 350 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 351 352 static int t4_num_vis = 1; 353 TUNABLE_INT("hw.cxgbe.num_vis", &t4_num_vis); 354 355 /* Functions used by extra VIs to obtain unique MAC addresses for each VI. */ 356 static int vi_mac_funcs[] = { 357 FW_VI_FUNC_OFLD, 358 FW_VI_FUNC_IWARP, 359 FW_VI_FUNC_OPENISCSI, 360 FW_VI_FUNC_OPENFCOE, 361 FW_VI_FUNC_FOISCSI, 362 FW_VI_FUNC_FOFCOE, 363 }; 364 365 struct intrs_and_queues { 366 uint16_t intr_type; /* INTx, MSI, or MSI-X */ 367 uint16_t nirq; /* Total # of vectors */ 368 uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */ 369 uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */ 370 uint16_t ntxq10g; /* # of NIC txq's for each 10G port */ 371 uint16_t nrxq10g; /* # of NIC rxq's for each 10G port */ 372 uint16_t ntxq1g; /* # of NIC txq's for each 1G port */ 373 uint16_t nrxq1g; /* # of NIC rxq's for each 1G port */ 374 uint16_t rsrv_noflowq; /* Flag whether to reserve queue 0 */ 375 #ifdef TCP_OFFLOAD 376 uint16_t nofldtxq10g; /* # of TOE txq's for each 10G port */ 377 uint16_t nofldrxq10g; /* # of TOE rxq's for each 10G port */ 378 uint16_t nofldtxq1g; /* # of TOE txq's for each 1G port */ 379 uint16_t nofldrxq1g; /* # of TOE rxq's for each 1G port */ 380 #endif 381 #ifdef DEV_NETMAP 382 uint16_t nnmtxq10g; /* # of netmap txq's for each 10G port */ 383 uint16_t nnmrxq10g; /* # of netmap rxq's for each 10G port */ 384 uint16_t nnmtxq1g; /* # of netmap txq's for each 1G port */ 385 uint16_t nnmrxq1g; /* # of netmap rxq's for each 1G port */ 386 #endif 387 }; 388 389 struct filter_entry { 390 uint32_t valid:1; /* filter allocated and valid */ 391 uint32_t locked:1; /* filter is administratively locked */ 392 uint32_t pending:1; /* filter action is pending firmware reply */ 393 uint32_t smtidx:8; /* Source MAC Table index for smac */ 394 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 395 396 struct t4_filter_specification fs; 397 }; 398 399 static int map_bars_0_and_4(struct adapter *); 400 static int map_bar_2(struct adapter *); 401 static void setup_memwin(struct adapter *); 402 static int validate_mem_range(struct adapter *, uint32_t, int); 403 static int fwmtype_to_hwmtype(int); 404 static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 405 uint32_t *); 406 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 407 static uint32_t position_memwin(struct adapter *, int, uint32_t); 408 static int cfg_itype_and_nqueues(struct adapter *, int, int, int, 409 struct intrs_and_queues *); 410 static int prep_firmware(struct adapter *); 411 static int partition_resources(struct adapter *, const struct firmware *, 412 const char *); 413 static int get_params__pre_init(struct adapter *); 414 static int get_params__post_init(struct adapter *); 415 static int set_params__post_init(struct adapter *); 416 static void t4_set_desc(struct adapter *); 417 static void build_medialist(struct port_info *, struct ifmedia *); 418 static int cxgbe_init_synchronized(struct vi_info *); 419 static int cxgbe_uninit_synchronized(struct vi_info *); 420 static int setup_intr_handlers(struct adapter *); 421 static void quiesce_txq(struct adapter *, struct sge_txq *); 422 static void quiesce_wrq(struct adapter *, struct sge_wrq *); 423 static void quiesce_iq(struct adapter *, struct sge_iq *); 424 static void quiesce_fl(struct adapter *, struct sge_fl *); 425 static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 426 driver_intr_t *, void *, char *); 427 static int t4_free_irq(struct adapter *, struct irq *); 428 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 429 unsigned int); 430 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 431 static void vi_refresh_stats(struct adapter *, struct vi_info *); 432 static void cxgbe_refresh_stats(struct adapter *, struct port_info *); 433 static void cxgbe_tick(void *); 434 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 435 static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 436 struct mbuf *); 437 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 438 static int fw_msg_not_handled(struct adapter *, const __be64 *); 439 static void t4_sysctls(struct adapter *); 440 static void cxgbe_sysctls(struct port_info *); 441 static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 442 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 443 static int sysctl_btphy(SYSCTL_HANDLER_ARGS); 444 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); 445 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 446 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 447 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 448 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 449 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS); 450 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 451 static int sysctl_temperature(SYSCTL_HANDLER_ARGS); 452 #ifdef SBUF_DRAIN 453 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 454 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 455 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 456 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); 457 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); 458 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 459 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 460 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 461 static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 462 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 463 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 464 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 465 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); 466 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 467 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); 468 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 469 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 470 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 471 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 472 static int sysctl_tids(SYSCTL_HANDLER_ARGS); 473 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 474 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 475 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 476 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 477 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 478 #endif 479 static uint32_t fconf_to_mode(uint32_t); 480 static uint32_t mode_to_fconf(uint32_t); 481 static uint32_t fspec_to_fconf(struct t4_filter_specification *); 482 static int get_filter_mode(struct adapter *, uint32_t *); 483 static int set_filter_mode(struct adapter *, uint32_t); 484 static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 485 static int get_filter(struct adapter *, struct t4_filter *); 486 static int set_filter(struct adapter *, struct t4_filter *); 487 static int del_filter(struct adapter *, struct t4_filter *); 488 static void clear_filter(struct filter_entry *); 489 static int set_filter_wr(struct adapter *, int); 490 static int del_filter_wr(struct adapter *, int); 491 static int get_sge_context(struct adapter *, struct t4_sge_context *); 492 static int load_fw(struct adapter *, struct t4_data *); 493 static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 494 static int read_i2c(struct adapter *, struct t4_i2c_data *); 495 static int set_sched_class(struct adapter *, struct t4_sched_params *); 496 static int set_sched_queue(struct adapter *, struct t4_sched_queue *); 497 #ifdef TCP_OFFLOAD 498 static int toe_capability(struct vi_info *, int); 499 #endif 500 static int mod_event(module_t, int, void *); 501 502 struct { 503 uint16_t device; 504 char *desc; 505 } t4_pciids[] = { 506 {0xa000, "Chelsio Terminator 4 FPGA"}, 507 {0x4400, "Chelsio T440-dbg"}, 508 {0x4401, "Chelsio T420-CR"}, 509 {0x4402, "Chelsio T422-CR"}, 510 {0x4403, "Chelsio T440-CR"}, 511 {0x4404, "Chelsio T420-BCH"}, 512 {0x4405, "Chelsio T440-BCH"}, 513 {0x4406, "Chelsio T440-CH"}, 514 {0x4407, "Chelsio T420-SO"}, 515 {0x4408, "Chelsio T420-CX"}, 516 {0x4409, "Chelsio T420-BT"}, 517 {0x440a, "Chelsio T404-BT"}, 518 {0x440e, "Chelsio T440-LP-CR"}, 519 }, t5_pciids[] = { 520 {0xb000, "Chelsio Terminator 5 FPGA"}, 521 {0x5400, "Chelsio T580-dbg"}, 522 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ 523 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ 524 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ 525 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ 526 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ 527 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ 528 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ 529 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ 530 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 531 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ 532 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ 533 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ 534 {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */ 535 #ifdef notyet 536 {0x5404, "Chelsio T520-BCH"}, 537 {0x5405, "Chelsio T540-BCH"}, 538 {0x5406, "Chelsio T540-CH"}, 539 {0x5408, "Chelsio T520-CX"}, 540 {0x540b, "Chelsio B520-SR"}, 541 {0x540c, "Chelsio B504-BT"}, 542 {0x540f, "Chelsio Amsterdam"}, 543 {0x5413, "Chelsio T580-CHR"}, 544 #endif 545 }; 546 547 #ifdef TCP_OFFLOAD 548 /* 549 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 550 * exactly the same for both rxq and ofld_rxq. 551 */ 552 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 553 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 554 #endif 555 556 /* No easy way to include t4_msg.h before adapter.h so we check this way */ 557 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 558 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 559 560 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); 561 562 static int 563 t4_probe(device_t dev) 564 { 565 int i; 566 uint16_t v = pci_get_vendor(dev); 567 uint16_t d = pci_get_device(dev); 568 uint8_t f = pci_get_function(dev); 569 570 if (v != PCI_VENDOR_ID_CHELSIO) 571 return (ENXIO); 572 573 /* Attach only to PF0 of the FPGA */ 574 if (d == 0xa000 && f != 0) 575 return (ENXIO); 576 577 for (i = 0; i < nitems(t4_pciids); i++) { 578 if (d == t4_pciids[i].device) { 579 device_set_desc(dev, t4_pciids[i].desc); 580 return (BUS_PROBE_DEFAULT); 581 } 582 } 583 584 return (ENXIO); 585 } 586 587 static int 588 t5_probe(device_t dev) 589 { 590 int i; 591 uint16_t v = pci_get_vendor(dev); 592 uint16_t d = pci_get_device(dev); 593 uint8_t f = pci_get_function(dev); 594 595 if (v != PCI_VENDOR_ID_CHELSIO) 596 return (ENXIO); 597 598 /* Attach only to PF0 of the FPGA */ 599 if (d == 0xb000 && f != 0) 600 return (ENXIO); 601 602 for (i = 0; i < nitems(t5_pciids); i++) { 603 if (d == t5_pciids[i].device) { 604 device_set_desc(dev, t5_pciids[i].desc); 605 return (BUS_PROBE_DEFAULT); 606 } 607 } 608 609 return (ENXIO); 610 } 611 612 static void 613 t5_attribute_workaround(device_t dev) 614 { 615 device_t root_port; 616 uint32_t v; 617 618 /* 619 * The T5 chips do not properly echo the No Snoop and Relaxed 620 * Ordering attributes when replying to a TLP from a Root 621 * Port. As a workaround, find the parent Root Port and 622 * disable No Snoop and Relaxed Ordering. Note that this 623 * affects all devices under this root port. 624 */ 625 root_port = pci_find_pcie_root_port(dev); 626 if (root_port == NULL) { 627 device_printf(dev, "Unable to find parent root port\n"); 628 return; 629 } 630 631 v = pcie_adjust_config(root_port, PCIER_DEVICE_CTL, 632 PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE, 0, 2); 633 if ((v & (PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE)) != 634 0) 635 device_printf(dev, "Disabled No Snoop/Relaxed Ordering on %s\n", 636 device_get_nameunit(root_port)); 637 } 638 639 static int 640 t4_attach(device_t dev) 641 { 642 struct adapter *sc; 643 int rc = 0, i, j, n10g, n1g, rqidx, tqidx; 644 struct intrs_and_queues iaq; 645 struct sge *s; 646 #ifdef TCP_OFFLOAD 647 int ofld_rqidx, ofld_tqidx; 648 #endif 649 #ifdef DEV_NETMAP 650 int nm_rqidx, nm_tqidx; 651 #endif 652 int num_vis; 653 654 sc = device_get_softc(dev); 655 sc->dev = dev; 656 TUNABLE_INT_FETCH("hw.cxgbe.debug_flags", &sc->debug_flags); 657 658 if ((pci_get_device(dev) & 0xff00) == 0x5400) 659 t5_attribute_workaround(dev); 660 pci_enable_busmaster(dev); 661 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 662 uint32_t v; 663 664 pci_set_max_read_req(dev, 4096); 665 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 666 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 667 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 668 669 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5); 670 } 671 672 sc->traceq = -1; 673 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); 674 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", 675 device_get_nameunit(dev)); 676 677 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 678 device_get_nameunit(dev)); 679 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 680 sx_xlock(&t4_list_lock); 681 SLIST_INSERT_HEAD(&t4_list, sc, link); 682 sx_xunlock(&t4_list_lock); 683 684 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 685 TAILQ_INIT(&sc->sfl); 686 callout_init_mtx(&sc->sfl_callout, &sc->sfl_lock, 0); 687 688 mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF); 689 690 rc = map_bars_0_and_4(sc); 691 if (rc != 0) 692 goto done; /* error message displayed already */ 693 694 /* 695 * This is the real PF# to which we're attaching. Works from within PCI 696 * passthrough environments too, where pci_get_function() could return a 697 * different PF# depending on the passthrough configuration. We need to 698 * use the real PF# in all our communication with the firmware. 699 */ 700 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 701 sc->mbox = sc->pf; 702 703 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 704 sc->an_handler = an_not_handled; 705 for (i = 0; i < nitems(sc->cpl_handler); i++) 706 sc->cpl_handler[i] = cpl_not_handled; 707 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 708 sc->fw_msg_handler[i] = fw_msg_not_handled; 709 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 710 t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt); 711 t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt); 712 t4_init_sge_cpl_handlers(sc); 713 714 /* Prepare the adapter for operation */ 715 rc = -t4_prep_adapter(sc); 716 if (rc != 0) { 717 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 718 goto done; 719 } 720 721 /* 722 * Do this really early, with the memory windows set up even before the 723 * character device. The userland tool's register i/o and mem read 724 * will work even in "recovery mode". 725 */ 726 setup_memwin(sc); 727 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 728 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 729 device_get_nameunit(dev)); 730 if (sc->cdev == NULL) 731 device_printf(dev, "failed to create nexus char device.\n"); 732 else 733 sc->cdev->si_drv1 = sc; 734 735 /* Go no further if recovery mode has been requested. */ 736 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 737 device_printf(dev, "recovery mode.\n"); 738 goto done; 739 } 740 741 #if defined(__i386__) 742 if ((cpu_feature & CPUID_CX8) == 0) { 743 device_printf(dev, "64 bit atomics not available.\n"); 744 rc = ENOTSUP; 745 goto done; 746 } 747 #endif 748 749 /* Prepare the firmware for operation */ 750 rc = prep_firmware(sc); 751 if (rc != 0) 752 goto done; /* error message displayed already */ 753 754 rc = get_params__post_init(sc); 755 if (rc != 0) 756 goto done; /* error message displayed already */ 757 758 rc = set_params__post_init(sc); 759 if (rc != 0) 760 goto done; /* error message displayed already */ 761 762 rc = map_bar_2(sc); 763 if (rc != 0) 764 goto done; /* error message displayed already */ 765 766 rc = t4_create_dma_tag(sc); 767 if (rc != 0) 768 goto done; /* error message displayed already */ 769 770 /* 771 * Number of VIs to create per-port. The first VI is the 772 * "main" regular VI for the port. The second VI is used for 773 * netmap if present, and any remaining VIs are used for 774 * additional virtual interfaces. 775 * 776 * Limit the number of VIs per port to the number of available 777 * MAC addresses per port. 778 */ 779 if (t4_num_vis >= 1) 780 num_vis = t4_num_vis; 781 else 782 num_vis = 1; 783 #ifdef DEV_NETMAP 784 num_vis++; 785 #endif 786 if (num_vis > nitems(vi_mac_funcs)) { 787 num_vis = nitems(vi_mac_funcs); 788 device_printf(dev, "Number of VIs limited to %d\n", num_vis); 789 } 790 791 /* 792 * First pass over all the ports - allocate VIs and initialize some 793 * basic parameters like mac address, port type, etc. We also figure 794 * out whether a port is 10G or 1G and use that information when 795 * calculating how many interrupts to attempt to allocate. 796 */ 797 n10g = n1g = 0; 798 for_each_port(sc, i) { 799 struct port_info *pi; 800 struct vi_info *vi; 801 802 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 803 sc->port[i] = pi; 804 805 /* These must be set before t4_port_init */ 806 pi->adapter = sc; 807 pi->port_id = i; 808 pi->nvi = num_vis; 809 pi->vi = malloc(sizeof(struct vi_info) * num_vis, M_CXGBE, 810 M_ZERO | M_WAITOK); 811 812 /* 813 * Allocate the "main" VI and initialize parameters 814 * like mac addr. 815 */ 816 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 817 if (rc != 0) { 818 device_printf(dev, "unable to initialize port %d: %d\n", 819 i, rc); 820 free(pi->vi, M_CXGBE); 821 free(pi, M_CXGBE); 822 sc->port[i] = NULL; 823 goto done; 824 } 825 826 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX); 827 pi->link_cfg.requested_fc |= t4_pause_settings; 828 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX); 829 pi->link_cfg.fc |= t4_pause_settings; 830 831 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 832 if (rc != 0) { 833 device_printf(dev, "port %d l1cfg failed: %d\n", i, rc); 834 free(pi->vi, M_CXGBE); 835 free(pi, M_CXGBE); 836 sc->port[i] = NULL; 837 goto done; 838 } 839 840 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 841 device_get_nameunit(dev), i); 842 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 843 sc->chan_map[pi->tx_chan] = i; 844 845 if (is_10G_port(pi) || is_40G_port(pi)) { 846 n10g++; 847 for_each_vi(pi, j, vi) { 848 vi->tmr_idx = t4_tmr_idx_10g; 849 vi->pktc_idx = t4_pktc_idx_10g; 850 } 851 } else { 852 n1g++; 853 for_each_vi(pi, j, vi) { 854 vi->tmr_idx = t4_tmr_idx_1g; 855 vi->pktc_idx = t4_pktc_idx_1g; 856 } 857 } 858 859 pi->linkdnrc = -1; 860 861 for_each_vi(pi, j, vi) { 862 vi->qsize_rxq = t4_qsize_rxq; 863 vi->qsize_txq = t4_qsize_txq; 864 vi->pi = pi; 865 } 866 867 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 868 if (pi->dev == NULL) { 869 device_printf(dev, 870 "failed to add device for port %d.\n", i); 871 rc = ENXIO; 872 goto done; 873 } 874 pi->vi[0].dev = pi->dev; 875 device_set_softc(pi->dev, pi); 876 } 877 878 /* 879 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 880 */ 881 #ifdef DEV_NETMAP 882 num_vis--; 883 #endif 884 rc = cfg_itype_and_nqueues(sc, n10g, n1g, num_vis, &iaq); 885 if (rc != 0) 886 goto done; /* error message displayed already */ 887 888 sc->intr_type = iaq.intr_type; 889 sc->intr_count = iaq.nirq; 890 891 s = &sc->sge; 892 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 893 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 894 if (num_vis > 1) { 895 s->nrxq += (n10g + n1g) * (num_vis - 1); 896 s->ntxq += (n10g + n1g) * (num_vis - 1); 897 } 898 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 899 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 900 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 901 #ifdef TCP_OFFLOAD 902 if (is_offload(sc)) { 903 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 904 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 905 if (num_vis > 1) { 906 s->nofldrxq += (n10g + n1g) * (num_vis - 1); 907 s->nofldtxq += (n10g + n1g) * (num_vis - 1); 908 } 909 s->neq += s->nofldtxq + s->nofldrxq; 910 s->niq += s->nofldrxq; 911 912 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 913 M_CXGBE, M_ZERO | M_WAITOK); 914 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 915 M_CXGBE, M_ZERO | M_WAITOK); 916 } 917 #endif 918 #ifdef DEV_NETMAP 919 s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g; 920 s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g; 921 s->neq += s->nnmtxq + s->nnmrxq; 922 s->niq += s->nnmrxq; 923 924 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq), 925 M_CXGBE, M_ZERO | M_WAITOK); 926 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq), 927 M_CXGBE, M_ZERO | M_WAITOK); 928 #endif 929 930 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 931 M_ZERO | M_WAITOK); 932 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 933 M_ZERO | M_WAITOK); 934 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 935 M_ZERO | M_WAITOK); 936 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 937 M_ZERO | M_WAITOK); 938 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 939 M_ZERO | M_WAITOK); 940 941 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 942 M_ZERO | M_WAITOK); 943 944 t4_init_l2t(sc, M_WAITOK); 945 946 /* 947 * Second pass over the ports. This time we know the number of rx and 948 * tx queues that each port should get. 949 */ 950 rqidx = tqidx = 0; 951 #ifdef TCP_OFFLOAD 952 ofld_rqidx = ofld_tqidx = 0; 953 #endif 954 #ifdef DEV_NETMAP 955 nm_rqidx = nm_tqidx = 0; 956 #endif 957 for_each_port(sc, i) { 958 struct port_info *pi = sc->port[i]; 959 struct vi_info *vi; 960 961 if (pi == NULL) 962 continue; 963 964 for_each_vi(pi, j, vi) { 965 #ifdef DEV_NETMAP 966 if (j == 1) { 967 vi->flags |= VI_NETMAP | INTR_RXQ; 968 vi->first_rxq = nm_rqidx; 969 vi->first_txq = nm_tqidx; 970 if (is_10G_port(pi) || is_40G_port(pi)) { 971 vi->nrxq = iaq.nnmrxq10g; 972 vi->ntxq = iaq.nnmtxq10g; 973 } else { 974 vi->nrxq = iaq.nnmrxq1g; 975 vi->ntxq = iaq.nnmtxq1g; 976 } 977 nm_rqidx += vi->nrxq; 978 nm_tqidx += vi->ntxq; 979 continue; 980 } 981 #endif 982 983 vi->first_rxq = rqidx; 984 vi->first_txq = tqidx; 985 if (is_10G_port(pi) || is_40G_port(pi)) { 986 vi->flags |= iaq.intr_flags_10g & INTR_RXQ; 987 vi->nrxq = j == 0 ? iaq.nrxq10g : 1; 988 vi->ntxq = j == 0 ? iaq.ntxq10g : 1; 989 } else { 990 vi->flags |= iaq.intr_flags_1g & INTR_RXQ; 991 vi->nrxq = j == 0 ? iaq.nrxq1g : 1; 992 vi->ntxq = j == 0 ? iaq.ntxq1g : 1; 993 } 994 995 if (vi->ntxq > 1) 996 vi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0; 997 else 998 vi->rsrv_noflowq = 0; 999 1000 rqidx += vi->nrxq; 1001 tqidx += vi->ntxq; 1002 1003 #ifdef TCP_OFFLOAD 1004 if (!is_offload(sc)) 1005 continue; 1006 vi->first_ofld_rxq = ofld_rqidx; 1007 vi->first_ofld_txq = ofld_tqidx; 1008 if (is_10G_port(pi) || is_40G_port(pi)) { 1009 vi->flags |= iaq.intr_flags_10g & INTR_OFLD_RXQ; 1010 vi->nofldrxq = j == 0 ? iaq.nofldrxq10g : 1; 1011 vi->nofldtxq = j == 0 ? iaq.nofldtxq10g : 1; 1012 } else { 1013 vi->flags |= iaq.intr_flags_1g & INTR_OFLD_RXQ; 1014 vi->nofldrxq = j == 0 ? iaq.nofldrxq1g : 1; 1015 vi->nofldtxq = j == 0 ? iaq.nofldtxq1g : 1; 1016 } 1017 ofld_rqidx += vi->nofldrxq; 1018 ofld_tqidx += vi->nofldtxq; 1019 #endif 1020 } 1021 } 1022 1023 rc = setup_intr_handlers(sc); 1024 if (rc != 0) { 1025 device_printf(dev, 1026 "failed to setup interrupt handlers: %d\n", rc); 1027 goto done; 1028 } 1029 1030 rc = bus_generic_attach(dev); 1031 if (rc != 0) { 1032 device_printf(dev, 1033 "failed to attach all child ports: %d\n", rc); 1034 goto done; 1035 } 1036 1037 device_printf(dev, 1038 "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 1039 sc->params.pci.speed, sc->params.pci.width, sc->params.nports, 1040 sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" : 1041 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 1042 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 1043 1044 t4_set_desc(sc); 1045 1046 done: 1047 if (rc != 0 && sc->cdev) { 1048 /* cdev was created and so cxgbetool works; recover that way. */ 1049 device_printf(dev, 1050 "error during attach, adapter is now in recovery mode.\n"); 1051 rc = 0; 1052 } 1053 1054 if (rc != 0) 1055 t4_detach(dev); 1056 else 1057 t4_sysctls(sc); 1058 1059 return (rc); 1060 } 1061 1062 /* 1063 * Idempotent 1064 */ 1065 static int 1066 t4_detach(device_t dev) 1067 { 1068 struct adapter *sc; 1069 struct port_info *pi; 1070 int i, rc; 1071 1072 sc = device_get_softc(dev); 1073 1074 if (sc->flags & FULL_INIT_DONE) 1075 t4_intr_disable(sc); 1076 1077 if (sc->cdev) { 1078 destroy_dev(sc->cdev); 1079 sc->cdev = NULL; 1080 } 1081 1082 rc = bus_generic_detach(dev); 1083 if (rc) { 1084 device_printf(dev, 1085 "failed to detach child devices: %d\n", rc); 1086 return (rc); 1087 } 1088 1089 for (i = 0; i < sc->intr_count; i++) 1090 t4_free_irq(sc, &sc->irq[i]); 1091 1092 for (i = 0; i < MAX_NPORTS; i++) { 1093 pi = sc->port[i]; 1094 if (pi) { 1095 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->vi[0].viid); 1096 if (pi->dev) 1097 device_delete_child(dev, pi->dev); 1098 1099 mtx_destroy(&pi->pi_lock); 1100 free(pi->vi, M_CXGBE); 1101 free(pi, M_CXGBE); 1102 } 1103 } 1104 1105 if (sc->flags & FULL_INIT_DONE) 1106 adapter_full_uninit(sc); 1107 1108 if (sc->flags & FW_OK) 1109 t4_fw_bye(sc, sc->mbox); 1110 1111 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 1112 pci_release_msi(dev); 1113 1114 if (sc->regs_res) 1115 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 1116 sc->regs_res); 1117 1118 if (sc->udbs_res) 1119 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 1120 sc->udbs_res); 1121 1122 if (sc->msix_res) 1123 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 1124 sc->msix_res); 1125 1126 if (sc->l2t) 1127 t4_free_l2t(sc->l2t); 1128 1129 #ifdef TCP_OFFLOAD 1130 free(sc->sge.ofld_rxq, M_CXGBE); 1131 free(sc->sge.ofld_txq, M_CXGBE); 1132 #endif 1133 #ifdef DEV_NETMAP 1134 free(sc->sge.nm_rxq, M_CXGBE); 1135 free(sc->sge.nm_txq, M_CXGBE); 1136 #endif 1137 free(sc->irq, M_CXGBE); 1138 free(sc->sge.rxq, M_CXGBE); 1139 free(sc->sge.txq, M_CXGBE); 1140 free(sc->sge.ctrlq, M_CXGBE); 1141 free(sc->sge.iqmap, M_CXGBE); 1142 free(sc->sge.eqmap, M_CXGBE); 1143 free(sc->tids.ftid_tab, M_CXGBE); 1144 t4_destroy_dma_tag(sc); 1145 if (mtx_initialized(&sc->sc_lock)) { 1146 sx_xlock(&t4_list_lock); 1147 SLIST_REMOVE(&t4_list, sc, adapter, link); 1148 sx_xunlock(&t4_list_lock); 1149 mtx_destroy(&sc->sc_lock); 1150 } 1151 1152 callout_drain(&sc->sfl_callout); 1153 if (mtx_initialized(&sc->tids.ftid_lock)) 1154 mtx_destroy(&sc->tids.ftid_lock); 1155 if (mtx_initialized(&sc->sfl_lock)) 1156 mtx_destroy(&sc->sfl_lock); 1157 if (mtx_initialized(&sc->ifp_lock)) 1158 mtx_destroy(&sc->ifp_lock); 1159 if (mtx_initialized(&sc->regwin_lock)) 1160 mtx_destroy(&sc->regwin_lock); 1161 1162 bzero(sc, sizeof(*sc)); 1163 1164 return (0); 1165 } 1166 1167 static int 1168 cxgbe_probe(device_t dev) 1169 { 1170 char buf[128]; 1171 struct port_info *pi = device_get_softc(dev); 1172 1173 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 1174 device_set_desc_copy(dev, buf); 1175 1176 return (BUS_PROBE_DEFAULT); 1177 } 1178 1179 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 1180 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 1181 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS) 1182 #define T4_CAP_ENABLE (T4_CAP) 1183 1184 static int 1185 cxgbe_vi_attach(device_t dev, struct vi_info *vi) 1186 { 1187 struct ifnet *ifp; 1188 struct sbuf *sb; 1189 1190 vi->xact_addr_filt = -1; 1191 callout_init(&vi->tick, 1); 1192 1193 /* Allocate an ifnet and set it up */ 1194 ifp = if_alloc(IFT_ETHER); 1195 if (ifp == NULL) { 1196 device_printf(dev, "Cannot allocate ifnet\n"); 1197 return (ENOMEM); 1198 } 1199 vi->ifp = ifp; 1200 ifp->if_softc = vi; 1201 1202 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1203 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1204 1205 ifp->if_init = cxgbe_init; 1206 ifp->if_ioctl = cxgbe_ioctl; 1207 ifp->if_transmit = cxgbe_transmit; 1208 ifp->if_qflush = cxgbe_qflush; 1209 ifp->if_get_counter = cxgbe_get_counter; 1210 1211 ifp->if_capabilities = T4_CAP; 1212 #ifdef TCP_OFFLOAD 1213 if (vi->nofldrxq != 0) 1214 ifp->if_capabilities |= IFCAP_TOE; 1215 #endif 1216 ifp->if_capenable = T4_CAP_ENABLE; 1217 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1218 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1219 1220 ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1221 ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS; 1222 ifp->if_hw_tsomaxsegsize = 65536; 1223 1224 /* Initialize ifmedia for this VI */ 1225 ifmedia_init(&vi->media, IFM_IMASK, cxgbe_media_change, 1226 cxgbe_media_status); 1227 build_medialist(vi->pi, &vi->media); 1228 1229 vi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 1230 EVENTHANDLER_PRI_ANY); 1231 1232 ether_ifattach(ifp, vi->hw_addr); 1233 1234 sb = sbuf_new_auto(); 1235 sbuf_printf(sb, "%d txq, %d rxq (NIC)", vi->ntxq, vi->nrxq); 1236 #ifdef TCP_OFFLOAD 1237 if (ifp->if_capabilities & IFCAP_TOE) 1238 sbuf_printf(sb, "; %d txq, %d rxq (TOE)", 1239 vi->nofldtxq, vi->nofldrxq); 1240 #endif 1241 sbuf_finish(sb); 1242 device_printf(dev, "%s\n", sbuf_data(sb)); 1243 sbuf_delete(sb); 1244 1245 vi_sysctls(vi); 1246 1247 return (0); 1248 } 1249 1250 static int 1251 cxgbe_attach(device_t dev) 1252 { 1253 struct port_info *pi = device_get_softc(dev); 1254 struct vi_info *vi; 1255 int i, rc; 1256 1257 callout_init_mtx(&pi->tick, &pi->pi_lock, 0); 1258 1259 rc = cxgbe_vi_attach(dev, &pi->vi[0]); 1260 if (rc) 1261 return (rc); 1262 1263 for_each_vi(pi, i, vi) { 1264 if (i == 0) 1265 continue; 1266 #ifdef DEV_NETMAP 1267 if (vi->flags & VI_NETMAP) { 1268 /* 1269 * media handled here to keep 1270 * implementation private to this file 1271 */ 1272 ifmedia_init(&vi->media, IFM_IMASK, cxgbe_media_change, 1273 cxgbe_media_status); 1274 build_medialist(pi, &vi->media); 1275 vi->dev = device_add_child(dev, is_t4(pi->adapter) ? 1276 "ncxgbe" : "ncxl", device_get_unit(dev)); 1277 } else 1278 #endif 1279 vi->dev = device_add_child(dev, is_t4(pi->adapter) ? 1280 "vcxgbe" : "vcxl", -1); 1281 if (vi->dev == NULL) { 1282 device_printf(dev, "failed to add VI %d\n", i); 1283 continue; 1284 } 1285 device_set_softc(vi->dev, vi); 1286 } 1287 1288 cxgbe_sysctls(pi); 1289 1290 bus_generic_attach(dev); 1291 1292 return (0); 1293 } 1294 1295 static void 1296 cxgbe_vi_detach(struct vi_info *vi) 1297 { 1298 struct ifnet *ifp = vi->ifp; 1299 1300 ether_ifdetach(ifp); 1301 1302 if (vi->vlan_c) 1303 EVENTHANDLER_DEREGISTER(vlan_config, vi->vlan_c); 1304 1305 /* Let detach proceed even if these fail. */ 1306 cxgbe_uninit_synchronized(vi); 1307 callout_drain(&vi->tick); 1308 vi_full_uninit(vi); 1309 1310 ifmedia_removeall(&vi->media); 1311 if_free(vi->ifp); 1312 vi->ifp = NULL; 1313 } 1314 1315 static int 1316 cxgbe_detach(device_t dev) 1317 { 1318 struct port_info *pi = device_get_softc(dev); 1319 struct adapter *sc = pi->adapter; 1320 int rc; 1321 1322 /* Detach the extra VIs first. */ 1323 rc = bus_generic_detach(dev); 1324 if (rc) 1325 return (rc); 1326 device_delete_children(dev); 1327 1328 doom_vi(sc, &pi->vi[0]); 1329 1330 if (pi->flags & HAS_TRACEQ) { 1331 sc->traceq = -1; /* cloner should not create ifnet */ 1332 t4_tracer_port_detach(sc); 1333 } 1334 1335 cxgbe_vi_detach(&pi->vi[0]); 1336 callout_drain(&pi->tick); 1337 1338 end_synchronized_op(sc, 0); 1339 1340 return (0); 1341 } 1342 1343 static void 1344 cxgbe_init(void *arg) 1345 { 1346 struct vi_info *vi = arg; 1347 struct adapter *sc = vi->pi->adapter; 1348 1349 if (begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4init") != 0) 1350 return; 1351 cxgbe_init_synchronized(vi); 1352 end_synchronized_op(sc, 0); 1353 } 1354 1355 static int 1356 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1357 { 1358 int rc = 0, mtu, flags, can_sleep; 1359 struct vi_info *vi = ifp->if_softc; 1360 struct adapter *sc = vi->pi->adapter; 1361 struct ifreq *ifr = (struct ifreq *)data; 1362 uint32_t mask; 1363 1364 switch (cmd) { 1365 case SIOCSIFMTU: 1366 mtu = ifr->ifr_mtu; 1367 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1368 return (EINVAL); 1369 1370 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4mtu"); 1371 if (rc) 1372 return (rc); 1373 ifp->if_mtu = mtu; 1374 if (vi->flags & VI_INIT_DONE) { 1375 t4_update_fl_bufsize(ifp); 1376 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1377 rc = update_mac_settings(ifp, XGMAC_MTU); 1378 } 1379 end_synchronized_op(sc, 0); 1380 break; 1381 1382 case SIOCSIFFLAGS: 1383 can_sleep = 0; 1384 redo_sifflags: 1385 rc = begin_synchronized_op(sc, vi, 1386 can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg"); 1387 if (rc) 1388 return (rc); 1389 1390 if (ifp->if_flags & IFF_UP) { 1391 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1392 flags = vi->if_flags; 1393 if ((ifp->if_flags ^ flags) & 1394 (IFF_PROMISC | IFF_ALLMULTI)) { 1395 if (can_sleep == 1) { 1396 end_synchronized_op(sc, 0); 1397 can_sleep = 0; 1398 goto redo_sifflags; 1399 } 1400 rc = update_mac_settings(ifp, 1401 XGMAC_PROMISC | XGMAC_ALLMULTI); 1402 } 1403 } else { 1404 if (can_sleep == 0) { 1405 end_synchronized_op(sc, LOCK_HELD); 1406 can_sleep = 1; 1407 goto redo_sifflags; 1408 } 1409 rc = cxgbe_init_synchronized(vi); 1410 } 1411 vi->if_flags = ifp->if_flags; 1412 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1413 if (can_sleep == 0) { 1414 end_synchronized_op(sc, LOCK_HELD); 1415 can_sleep = 1; 1416 goto redo_sifflags; 1417 } 1418 rc = cxgbe_uninit_synchronized(vi); 1419 } 1420 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD); 1421 break; 1422 1423 case SIOCADDMULTI: 1424 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1425 rc = begin_synchronized_op(sc, vi, HOLD_LOCK, "t4multi"); 1426 if (rc) 1427 return (rc); 1428 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1429 rc = update_mac_settings(ifp, XGMAC_MCADDRS); 1430 end_synchronized_op(sc, LOCK_HELD); 1431 break; 1432 1433 case SIOCSIFCAP: 1434 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4cap"); 1435 if (rc) 1436 return (rc); 1437 1438 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1439 if (mask & IFCAP_TXCSUM) { 1440 ifp->if_capenable ^= IFCAP_TXCSUM; 1441 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1442 1443 if (IFCAP_TSO4 & ifp->if_capenable && 1444 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1445 ifp->if_capenable &= ~IFCAP_TSO4; 1446 if_printf(ifp, 1447 "tso4 disabled due to -txcsum.\n"); 1448 } 1449 } 1450 if (mask & IFCAP_TXCSUM_IPV6) { 1451 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1452 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1453 1454 if (IFCAP_TSO6 & ifp->if_capenable && 1455 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1456 ifp->if_capenable &= ~IFCAP_TSO6; 1457 if_printf(ifp, 1458 "tso6 disabled due to -txcsum6.\n"); 1459 } 1460 } 1461 if (mask & IFCAP_RXCSUM) 1462 ifp->if_capenable ^= IFCAP_RXCSUM; 1463 if (mask & IFCAP_RXCSUM_IPV6) 1464 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1465 1466 /* 1467 * Note that we leave CSUM_TSO alone (it is always set). The 1468 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1469 * sending a TSO request our way, so it's sufficient to toggle 1470 * IFCAP_TSOx only. 1471 */ 1472 if (mask & IFCAP_TSO4) { 1473 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1474 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1475 if_printf(ifp, "enable txcsum first.\n"); 1476 rc = EAGAIN; 1477 goto fail; 1478 } 1479 ifp->if_capenable ^= IFCAP_TSO4; 1480 } 1481 if (mask & IFCAP_TSO6) { 1482 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1483 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1484 if_printf(ifp, "enable txcsum6 first.\n"); 1485 rc = EAGAIN; 1486 goto fail; 1487 } 1488 ifp->if_capenable ^= IFCAP_TSO6; 1489 } 1490 if (mask & IFCAP_LRO) { 1491 #if defined(INET) || defined(INET6) 1492 int i; 1493 struct sge_rxq *rxq; 1494 1495 ifp->if_capenable ^= IFCAP_LRO; 1496 for_each_rxq(vi, i, rxq) { 1497 if (ifp->if_capenable & IFCAP_LRO) 1498 rxq->iq.flags |= IQ_LRO_ENABLED; 1499 else 1500 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1501 } 1502 #endif 1503 } 1504 #ifdef TCP_OFFLOAD 1505 if (mask & IFCAP_TOE) { 1506 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1507 1508 rc = toe_capability(vi, enable); 1509 if (rc != 0) 1510 goto fail; 1511 1512 ifp->if_capenable ^= mask; 1513 } 1514 #endif 1515 if (mask & IFCAP_VLAN_HWTAGGING) { 1516 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1517 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1518 rc = update_mac_settings(ifp, XGMAC_VLANEX); 1519 } 1520 if (mask & IFCAP_VLAN_MTU) { 1521 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1522 1523 /* Need to find out how to disable auto-mtu-inflation */ 1524 } 1525 if (mask & IFCAP_VLAN_HWTSO) 1526 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1527 if (mask & IFCAP_VLAN_HWCSUM) 1528 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1529 1530 #ifdef VLAN_CAPABILITIES 1531 VLAN_CAPABILITIES(ifp); 1532 #endif 1533 fail: 1534 end_synchronized_op(sc, 0); 1535 break; 1536 1537 case SIOCSIFMEDIA: 1538 case SIOCGIFMEDIA: 1539 ifmedia_ioctl(ifp, ifr, &vi->media, cmd); 1540 break; 1541 1542 case SIOCGI2C: { 1543 struct ifi2creq i2c; 1544 1545 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); 1546 if (rc != 0) 1547 break; 1548 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) { 1549 rc = EPERM; 1550 break; 1551 } 1552 if (i2c.len > sizeof(i2c.data)) { 1553 rc = EINVAL; 1554 break; 1555 } 1556 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4i2c"); 1557 if (rc) 1558 return (rc); 1559 rc = -t4_i2c_rd(sc, sc->mbox, vi->pi->port_id, i2c.dev_addr, 1560 i2c.offset, i2c.len, &i2c.data[0]); 1561 end_synchronized_op(sc, 0); 1562 if (rc == 0) 1563 rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); 1564 break; 1565 } 1566 1567 default: 1568 rc = ether_ioctl(ifp, cmd, data); 1569 } 1570 1571 return (rc); 1572 } 1573 1574 static int 1575 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1576 { 1577 struct vi_info *vi = ifp->if_softc; 1578 struct port_info *pi = vi->pi; 1579 struct adapter *sc = pi->adapter; 1580 struct sge_txq *txq; 1581 void *items[1]; 1582 int rc; 1583 1584 M_ASSERTPKTHDR(m); 1585 MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */ 1586 1587 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1588 m_freem(m); 1589 return (ENETDOWN); 1590 } 1591 1592 rc = parse_pkt(&m); 1593 if (__predict_false(rc != 0)) { 1594 MPASS(m == NULL); /* was freed already */ 1595 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */ 1596 return (rc); 1597 } 1598 1599 /* Select a txq. */ 1600 txq = &sc->sge.txq[vi->first_txq]; 1601 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 1602 txq += ((m->m_pkthdr.flowid % (vi->ntxq - vi->rsrv_noflowq)) + 1603 vi->rsrv_noflowq); 1604 1605 items[0] = m; 1606 rc = mp_ring_enqueue(txq->r, items, 1, 4096); 1607 if (__predict_false(rc != 0)) 1608 m_freem(m); 1609 1610 return (rc); 1611 } 1612 1613 static void 1614 cxgbe_qflush(struct ifnet *ifp) 1615 { 1616 struct vi_info *vi = ifp->if_softc; 1617 struct sge_txq *txq; 1618 int i; 1619 1620 /* queues do not exist if !VI_INIT_DONE. */ 1621 if (vi->flags & VI_INIT_DONE) { 1622 for_each_txq(vi, i, txq) { 1623 TXQ_LOCK(txq); 1624 txq->eq.flags &= ~EQ_ENABLED; 1625 TXQ_UNLOCK(txq); 1626 while (!mp_ring_is_idle(txq->r)) { 1627 mp_ring_check_drainage(txq->r, 0); 1628 pause("qflush", 1); 1629 } 1630 } 1631 } 1632 if_qflush(ifp); 1633 } 1634 1635 static uint64_t 1636 vi_get_counter(struct ifnet *ifp, ift_counter c) 1637 { 1638 struct vi_info *vi = ifp->if_softc; 1639 struct fw_vi_stats_vf *s = &vi->stats; 1640 1641 vi_refresh_stats(vi->pi->adapter, vi); 1642 1643 switch (c) { 1644 case IFCOUNTER_IPACKETS: 1645 return (s->rx_bcast_frames + s->rx_mcast_frames + 1646 s->rx_ucast_frames); 1647 case IFCOUNTER_IERRORS: 1648 return (s->rx_err_frames); 1649 case IFCOUNTER_OPACKETS: 1650 return (s->tx_bcast_frames + s->tx_mcast_frames + 1651 s->tx_ucast_frames + s->tx_offload_frames); 1652 case IFCOUNTER_OERRORS: 1653 return (s->tx_drop_frames); 1654 case IFCOUNTER_IBYTES: 1655 return (s->rx_bcast_bytes + s->rx_mcast_bytes + 1656 s->rx_ucast_bytes); 1657 case IFCOUNTER_OBYTES: 1658 return (s->tx_bcast_bytes + s->tx_mcast_bytes + 1659 s->tx_ucast_bytes + s->tx_offload_bytes); 1660 case IFCOUNTER_IMCASTS: 1661 return (s->rx_mcast_frames); 1662 case IFCOUNTER_OMCASTS: 1663 return (s->tx_mcast_frames); 1664 case IFCOUNTER_OQDROPS: { 1665 uint64_t drops; 1666 1667 drops = 0; 1668 if ((vi->flags & (VI_INIT_DONE | VI_NETMAP)) == VI_INIT_DONE) { 1669 int i; 1670 struct sge_txq *txq; 1671 1672 for_each_txq(vi, i, txq) 1673 drops += counter_u64_fetch(txq->r->drops); 1674 } 1675 1676 return (drops); 1677 1678 } 1679 1680 default: 1681 return (if_get_counter_default(ifp, c)); 1682 } 1683 } 1684 1685 uint64_t 1686 cxgbe_get_counter(struct ifnet *ifp, ift_counter c) 1687 { 1688 struct vi_info *vi = ifp->if_softc; 1689 struct port_info *pi = vi->pi; 1690 struct adapter *sc = pi->adapter; 1691 struct port_stats *s = &pi->stats; 1692 1693 if (pi->nvi > 1) 1694 return (vi_get_counter(ifp, c)); 1695 1696 cxgbe_refresh_stats(sc, pi); 1697 1698 switch (c) { 1699 case IFCOUNTER_IPACKETS: 1700 return (s->rx_frames - s->rx_pause); 1701 1702 case IFCOUNTER_IERRORS: 1703 return (s->rx_jabber + s->rx_runt + s->rx_too_long + 1704 s->rx_fcs_err + s->rx_len_err); 1705 1706 case IFCOUNTER_OPACKETS: 1707 return (s->tx_frames - s->tx_pause); 1708 1709 case IFCOUNTER_OERRORS: 1710 return (s->tx_error_frames); 1711 1712 case IFCOUNTER_IBYTES: 1713 return (s->rx_octets - s->rx_pause * 64); 1714 1715 case IFCOUNTER_OBYTES: 1716 return (s->tx_octets - s->tx_pause * 64); 1717 1718 case IFCOUNTER_IMCASTS: 1719 return (s->rx_mcast_frames - s->rx_pause); 1720 1721 case IFCOUNTER_OMCASTS: 1722 return (s->tx_mcast_frames - s->tx_pause); 1723 1724 case IFCOUNTER_IQDROPS: 1725 return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 1726 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 1727 s->rx_trunc3 + pi->tnl_cong_drops); 1728 1729 case IFCOUNTER_OQDROPS: { 1730 uint64_t drops; 1731 1732 drops = s->tx_drop; 1733 if (vi->flags & VI_INIT_DONE) { 1734 int i; 1735 struct sge_txq *txq; 1736 1737 for_each_txq(vi, i, txq) 1738 drops += counter_u64_fetch(txq->r->drops); 1739 } 1740 1741 return (drops); 1742 1743 } 1744 1745 default: 1746 return (if_get_counter_default(ifp, c)); 1747 } 1748 } 1749 1750 static int 1751 cxgbe_media_change(struct ifnet *ifp) 1752 { 1753 struct vi_info *vi = ifp->if_softc; 1754 1755 device_printf(vi->dev, "%s unimplemented.\n", __func__); 1756 1757 return (EOPNOTSUPP); 1758 } 1759 1760 static void 1761 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1762 { 1763 struct vi_info *vi = ifp->if_softc; 1764 struct port_info *pi = vi->pi; 1765 struct ifmedia_entry *cur; 1766 int speed = pi->link_cfg.speed; 1767 1768 cur = vi->media.ifm_cur; 1769 1770 ifmr->ifm_status = IFM_AVALID; 1771 if (!pi->link_cfg.link_ok) 1772 return; 1773 1774 ifmr->ifm_status |= IFM_ACTIVE; 1775 1776 /* active and current will differ iff current media is autoselect. */ 1777 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1778 return; 1779 1780 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1781 if (speed == SPEED_10000) 1782 ifmr->ifm_active |= IFM_10G_T; 1783 else if (speed == SPEED_1000) 1784 ifmr->ifm_active |= IFM_1000_T; 1785 else if (speed == SPEED_100) 1786 ifmr->ifm_active |= IFM_100_TX; 1787 else if (speed == SPEED_10) 1788 ifmr->ifm_active |= IFM_10_T; 1789 else 1790 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1791 speed)); 1792 } 1793 1794 static int 1795 vcxgbe_probe(device_t dev) 1796 { 1797 char buf[128]; 1798 struct vi_info *vi = device_get_softc(dev); 1799 1800 snprintf(buf, sizeof(buf), "port %d vi %td", vi->pi->port_id, 1801 vi - vi->pi->vi); 1802 device_set_desc_copy(dev, buf); 1803 1804 return (BUS_PROBE_DEFAULT); 1805 } 1806 1807 static int 1808 vcxgbe_attach(device_t dev) 1809 { 1810 struct vi_info *vi; 1811 struct port_info *pi; 1812 struct adapter *sc; 1813 int func, index, rc; 1814 u32 param, val; 1815 1816 vi = device_get_softc(dev); 1817 pi = vi->pi; 1818 sc = pi->adapter; 1819 1820 index = vi - pi->vi; 1821 KASSERT(index < nitems(vi_mac_funcs), 1822 ("%s: VI %s doesn't have a MAC func", __func__, 1823 device_get_nameunit(dev))); 1824 func = vi_mac_funcs[index]; 1825 rc = t4_alloc_vi_func(sc, sc->mbox, pi->tx_chan, sc->pf, 0, 1, 1826 vi->hw_addr, &vi->rss_size, func, 0); 1827 if (rc < 0) { 1828 device_printf(dev, "Failed to allocate virtual interface " 1829 "for port %d: %d\n", pi->port_id, -rc); 1830 return (-rc); 1831 } 1832 vi->viid = rc; 1833 1834 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 1835 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_RSSINFO) | 1836 V_FW_PARAMS_PARAM_YZ(vi->viid); 1837 rc = t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 1838 if (rc) 1839 vi->rss_base = 0xffff; 1840 else { 1841 /* MPASS((val >> 16) == rss_size); */ 1842 vi->rss_base = val & 0xffff; 1843 } 1844 1845 rc = cxgbe_vi_attach(dev, vi); 1846 if (rc) { 1847 t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid); 1848 return (rc); 1849 } 1850 return (0); 1851 } 1852 1853 static int 1854 vcxgbe_detach(device_t dev) 1855 { 1856 struct vi_info *vi; 1857 struct adapter *sc; 1858 1859 vi = device_get_softc(dev); 1860 sc = vi->pi->adapter; 1861 1862 doom_vi(sc, vi); 1863 1864 cxgbe_vi_detach(vi); 1865 t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid); 1866 1867 end_synchronized_op(sc, 0); 1868 1869 return (0); 1870 } 1871 1872 void 1873 t4_fatal_err(struct adapter *sc) 1874 { 1875 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1876 t4_intr_disable(sc); 1877 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1878 device_get_nameunit(sc->dev)); 1879 } 1880 1881 static int 1882 map_bars_0_and_4(struct adapter *sc) 1883 { 1884 sc->regs_rid = PCIR_BAR(0); 1885 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1886 &sc->regs_rid, RF_ACTIVE); 1887 if (sc->regs_res == NULL) { 1888 device_printf(sc->dev, "cannot map registers.\n"); 1889 return (ENXIO); 1890 } 1891 sc->bt = rman_get_bustag(sc->regs_res); 1892 sc->bh = rman_get_bushandle(sc->regs_res); 1893 sc->mmio_len = rman_get_size(sc->regs_res); 1894 setbit(&sc->doorbells, DOORBELL_KDB); 1895 1896 sc->msix_rid = PCIR_BAR(4); 1897 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1898 &sc->msix_rid, RF_ACTIVE); 1899 if (sc->msix_res == NULL) { 1900 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1901 return (ENXIO); 1902 } 1903 1904 return (0); 1905 } 1906 1907 static int 1908 map_bar_2(struct adapter *sc) 1909 { 1910 1911 /* 1912 * T4: only iWARP driver uses the userspace doorbells. There is no need 1913 * to map it if RDMA is disabled. 1914 */ 1915 if (is_t4(sc) && sc->rdmacaps == 0) 1916 return (0); 1917 1918 sc->udbs_rid = PCIR_BAR(2); 1919 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1920 &sc->udbs_rid, RF_ACTIVE); 1921 if (sc->udbs_res == NULL) { 1922 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1923 return (ENXIO); 1924 } 1925 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1926 1927 if (is_t5(sc)) { 1928 setbit(&sc->doorbells, DOORBELL_UDB); 1929 #if defined(__i386__) || defined(__amd64__) 1930 if (t5_write_combine) { 1931 int rc; 1932 1933 /* 1934 * Enable write combining on BAR2. This is the 1935 * userspace doorbell BAR and is split into 128B 1936 * (UDBS_SEG_SIZE) doorbell regions, each associated 1937 * with an egress queue. The first 64B has the doorbell 1938 * and the second 64B can be used to submit a tx work 1939 * request with an implicit doorbell. 1940 */ 1941 1942 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1943 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1944 if (rc == 0) { 1945 clrbit(&sc->doorbells, DOORBELL_UDB); 1946 setbit(&sc->doorbells, DOORBELL_WCWR); 1947 setbit(&sc->doorbells, DOORBELL_UDBWC); 1948 } else { 1949 device_printf(sc->dev, 1950 "couldn't enable write combining: %d\n", 1951 rc); 1952 } 1953 1954 t4_write_reg(sc, A_SGE_STAT_CFG, 1955 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1956 } 1957 #endif 1958 } 1959 1960 return (0); 1961 } 1962 1963 static const struct memwin t4_memwin[] = { 1964 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1965 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1966 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1967 }; 1968 1969 static const struct memwin t5_memwin[] = { 1970 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1971 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1972 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1973 }; 1974 1975 static void 1976 setup_memwin(struct adapter *sc) 1977 { 1978 const struct memwin *mw; 1979 int i, n; 1980 uint32_t bar0; 1981 1982 if (is_t4(sc)) { 1983 /* 1984 * Read low 32b of bar0 indirectly via the hardware backdoor 1985 * mechanism. Works from within PCI passthrough environments 1986 * too, where rman_get_start() can return a different value. We 1987 * need to program the T4 memory window decoders with the actual 1988 * addresses that will be coming across the PCIe link. 1989 */ 1990 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1991 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1992 1993 mw = &t4_memwin[0]; 1994 n = nitems(t4_memwin); 1995 } else { 1996 /* T5 uses the relative offset inside the PCIe BAR */ 1997 bar0 = 0; 1998 1999 mw = &t5_memwin[0]; 2000 n = nitems(t5_memwin); 2001 } 2002 2003 for (i = 0; i < n; i++, mw++) { 2004 t4_write_reg(sc, 2005 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 2006 (mw->base + bar0) | V_BIR(0) | 2007 V_WINDOW(ilog2(mw->aperture) - 10)); 2008 } 2009 2010 /* flush */ 2011 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 2012 } 2013 2014 /* 2015 * Verify that the memory range specified by the addr/len pair is valid and lies 2016 * entirely within a single region (EDCx or MCx). 2017 */ 2018 static int 2019 validate_mem_range(struct adapter *sc, uint32_t addr, int len) 2020 { 2021 uint32_t em, addr_len, maddr, mlen; 2022 2023 /* Memory can only be accessed in naturally aligned 4 byte units */ 2024 if (addr & 3 || len & 3 || len == 0) 2025 return (EINVAL); 2026 2027 /* Enabled memories */ 2028 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 2029 if (em & F_EDRAM0_ENABLE) { 2030 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 2031 maddr = G_EDRAM0_BASE(addr_len) << 20; 2032 mlen = G_EDRAM0_SIZE(addr_len) << 20; 2033 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 2034 addr + len <= maddr + mlen) 2035 return (0); 2036 } 2037 if (em & F_EDRAM1_ENABLE) { 2038 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 2039 maddr = G_EDRAM1_BASE(addr_len) << 20; 2040 mlen = G_EDRAM1_SIZE(addr_len) << 20; 2041 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 2042 addr + len <= maddr + mlen) 2043 return (0); 2044 } 2045 if (em & F_EXT_MEM_ENABLE) { 2046 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 2047 maddr = G_EXT_MEM_BASE(addr_len) << 20; 2048 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 2049 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 2050 addr + len <= maddr + mlen) 2051 return (0); 2052 } 2053 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 2054 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 2055 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 2056 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 2057 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 2058 addr + len <= maddr + mlen) 2059 return (0); 2060 } 2061 2062 return (EFAULT); 2063 } 2064 2065 static int 2066 fwmtype_to_hwmtype(int mtype) 2067 { 2068 2069 switch (mtype) { 2070 case FW_MEMTYPE_EDC0: 2071 return (MEM_EDC0); 2072 case FW_MEMTYPE_EDC1: 2073 return (MEM_EDC1); 2074 case FW_MEMTYPE_EXTMEM: 2075 return (MEM_MC0); 2076 case FW_MEMTYPE_EXTMEM1: 2077 return (MEM_MC1); 2078 default: 2079 panic("%s: cannot translate fw mtype %d.", __func__, mtype); 2080 } 2081 } 2082 2083 /* 2084 * Verify that the memory range specified by the memtype/offset/len pair is 2085 * valid and lies entirely within the memtype specified. The global address of 2086 * the start of the range is returned in addr. 2087 */ 2088 static int 2089 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 2090 uint32_t *addr) 2091 { 2092 uint32_t em, addr_len, maddr, mlen; 2093 2094 /* Memory can only be accessed in naturally aligned 4 byte units */ 2095 if (off & 3 || len & 3 || len == 0) 2096 return (EINVAL); 2097 2098 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 2099 switch (fwmtype_to_hwmtype(mtype)) { 2100 case MEM_EDC0: 2101 if (!(em & F_EDRAM0_ENABLE)) 2102 return (EINVAL); 2103 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 2104 maddr = G_EDRAM0_BASE(addr_len) << 20; 2105 mlen = G_EDRAM0_SIZE(addr_len) << 20; 2106 break; 2107 case MEM_EDC1: 2108 if (!(em & F_EDRAM1_ENABLE)) 2109 return (EINVAL); 2110 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 2111 maddr = G_EDRAM1_BASE(addr_len) << 20; 2112 mlen = G_EDRAM1_SIZE(addr_len) << 20; 2113 break; 2114 case MEM_MC: 2115 if (!(em & F_EXT_MEM_ENABLE)) 2116 return (EINVAL); 2117 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 2118 maddr = G_EXT_MEM_BASE(addr_len) << 20; 2119 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 2120 break; 2121 case MEM_MC1: 2122 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 2123 return (EINVAL); 2124 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 2125 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 2126 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 2127 break; 2128 default: 2129 return (EINVAL); 2130 } 2131 2132 if (mlen > 0 && off < mlen && off + len <= mlen) { 2133 *addr = maddr + off; /* global address */ 2134 return (0); 2135 } 2136 2137 return (EFAULT); 2138 } 2139 2140 static void 2141 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 2142 { 2143 const struct memwin *mw; 2144 2145 if (is_t4(sc)) { 2146 KASSERT(win >= 0 && win < nitems(t4_memwin), 2147 ("%s: incorrect memwin# (%d)", __func__, win)); 2148 mw = &t4_memwin[win]; 2149 } else { 2150 KASSERT(win >= 0 && win < nitems(t5_memwin), 2151 ("%s: incorrect memwin# (%d)", __func__, win)); 2152 mw = &t5_memwin[win]; 2153 } 2154 2155 if (base != NULL) 2156 *base = mw->base; 2157 if (aperture != NULL) 2158 *aperture = mw->aperture; 2159 } 2160 2161 /* 2162 * Positions the memory window such that it can be used to access the specified 2163 * address in the chip's address space. The return value is the offset of addr 2164 * from the start of the window. 2165 */ 2166 static uint32_t 2167 position_memwin(struct adapter *sc, int n, uint32_t addr) 2168 { 2169 uint32_t start, pf; 2170 uint32_t reg; 2171 2172 KASSERT(n >= 0 && n <= 3, 2173 ("%s: invalid window %d.", __func__, n)); 2174 KASSERT((addr & 3) == 0, 2175 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 2176 2177 if (is_t4(sc)) { 2178 pf = 0; 2179 start = addr & ~0xf; /* start must be 16B aligned */ 2180 } else { 2181 pf = V_PFNUM(sc->pf); 2182 start = addr & ~0x7f; /* start must be 128B aligned */ 2183 } 2184 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 2185 2186 t4_write_reg(sc, reg, start | pf); 2187 t4_read_reg(sc, reg); 2188 2189 return (addr - start); 2190 } 2191 2192 static int 2193 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, int num_vis, 2194 struct intrs_and_queues *iaq) 2195 { 2196 int rc, itype, navail, nrxq10g, nrxq1g, n; 2197 int nofldrxq10g = 0, nofldrxq1g = 0; 2198 int nnmrxq10g = 0, nnmrxq1g = 0; 2199 2200 bzero(iaq, sizeof(*iaq)); 2201 2202 iaq->ntxq10g = t4_ntxq10g; 2203 iaq->ntxq1g = t4_ntxq1g; 2204 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 2205 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 2206 iaq->rsrv_noflowq = t4_rsrv_noflowq; 2207 #ifdef TCP_OFFLOAD 2208 if (is_offload(sc)) { 2209 iaq->nofldtxq10g = t4_nofldtxq10g; 2210 iaq->nofldtxq1g = t4_nofldtxq1g; 2211 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 2212 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 2213 } 2214 #endif 2215 #ifdef DEV_NETMAP 2216 iaq->nnmtxq10g = t4_nnmtxq10g; 2217 iaq->nnmtxq1g = t4_nnmtxq1g; 2218 iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g; 2219 iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g; 2220 #endif 2221 2222 for (itype = INTR_MSIX; itype; itype >>= 1) { 2223 2224 if ((itype & t4_intr_types) == 0) 2225 continue; /* not allowed */ 2226 2227 if (itype == INTR_MSIX) 2228 navail = pci_msix_count(sc->dev); 2229 else if (itype == INTR_MSI) 2230 navail = pci_msi_count(sc->dev); 2231 else 2232 navail = 1; 2233 restart: 2234 if (navail == 0) 2235 continue; 2236 2237 iaq->intr_type = itype; 2238 iaq->intr_flags_10g = 0; 2239 iaq->intr_flags_1g = 0; 2240 2241 /* 2242 * Best option: an interrupt vector for errors, one for the 2243 * firmware event queue, and one for every rxq (NIC, TOE, and 2244 * netmap). 2245 */ 2246 iaq->nirq = T4_EXTRA_INTR; 2247 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g); 2248 iaq->nirq += n10g * 2 * (num_vis - 1); 2249 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g); 2250 iaq->nirq += n1g * 2 * (num_vis - 1); 2251 if (iaq->nirq <= navail && 2252 (itype != INTR_MSI || powerof2(iaq->nirq))) { 2253 iaq->intr_flags_10g = INTR_ALL; 2254 iaq->intr_flags_1g = INTR_ALL; 2255 goto allocate; 2256 } 2257 2258 /* 2259 * Second best option: a vector for errors, one for the firmware 2260 * event queue, and vectors for either all the NIC rx queues or 2261 * all the TOE rx queues. The queues that don't get vectors 2262 * will forward their interrupts to those that do. 2263 * 2264 * Note: netmap rx queues cannot be created early and so they 2265 * can't be setup to receive forwarded interrupts for others. 2266 */ 2267 iaq->nirq = T4_EXTRA_INTR; 2268 if (nrxq10g >= nofldrxq10g) { 2269 iaq->intr_flags_10g = INTR_RXQ; 2270 iaq->nirq += n10g * nrxq10g; 2271 iaq->nirq += n10g * (num_vis - 1); 2272 #ifdef DEV_NETMAP 2273 iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g); 2274 #endif 2275 } else { 2276 iaq->intr_flags_10g = INTR_OFLD_RXQ; 2277 iaq->nirq += n10g * nofldrxq10g; 2278 #ifdef DEV_NETMAP 2279 iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g); 2280 #endif 2281 } 2282 if (nrxq1g >= nofldrxq1g) { 2283 iaq->intr_flags_1g = INTR_RXQ; 2284 iaq->nirq += n1g * nrxq1g; 2285 iaq->nirq += n1g * (num_vis - 1); 2286 #ifdef DEV_NETMAP 2287 iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g); 2288 #endif 2289 } else { 2290 iaq->intr_flags_1g = INTR_OFLD_RXQ; 2291 iaq->nirq += n1g * nofldrxq1g; 2292 #ifdef DEV_NETMAP 2293 iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g); 2294 #endif 2295 } 2296 if (iaq->nirq <= navail && 2297 (itype != INTR_MSI || powerof2(iaq->nirq))) 2298 goto allocate; 2299 2300 /* 2301 * Next best option: an interrupt vector for errors, one for the 2302 * firmware event queue, and at least one per VI. At this 2303 * point we know we'll have to downsize nrxq and/or nofldrxq 2304 * and/or nnmrxq to fit what's available to us. 2305 */ 2306 iaq->nirq = T4_EXTRA_INTR; 2307 iaq->nirq += (n10g + n1g) * num_vis; 2308 if (iaq->nirq <= navail) { 2309 int leftover = navail - iaq->nirq; 2310 2311 if (n10g > 0) { 2312 int target = max(nrxq10g, nofldrxq10g); 2313 2314 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ? 2315 INTR_RXQ : INTR_OFLD_RXQ; 2316 2317 n = 1; 2318 while (n < target && leftover >= n10g) { 2319 leftover -= n10g; 2320 iaq->nirq += n10g; 2321 n++; 2322 } 2323 iaq->nrxq10g = min(n, nrxq10g); 2324 #ifdef TCP_OFFLOAD 2325 iaq->nofldrxq10g = min(n, nofldrxq10g); 2326 #endif 2327 #ifdef DEV_NETMAP 2328 iaq->nnmrxq10g = min(n, nnmrxq10g); 2329 #endif 2330 } 2331 2332 if (n1g > 0) { 2333 int target = max(nrxq1g, nofldrxq1g); 2334 2335 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ? 2336 INTR_RXQ : INTR_OFLD_RXQ; 2337 2338 n = 1; 2339 while (n < target && leftover >= n1g) { 2340 leftover -= n1g; 2341 iaq->nirq += n1g; 2342 n++; 2343 } 2344 iaq->nrxq1g = min(n, nrxq1g); 2345 #ifdef TCP_OFFLOAD 2346 iaq->nofldrxq1g = min(n, nofldrxq1g); 2347 #endif 2348 #ifdef DEV_NETMAP 2349 iaq->nnmrxq1g = min(n, nnmrxq1g); 2350 #endif 2351 } 2352 2353 if (itype != INTR_MSI || powerof2(iaq->nirq)) 2354 goto allocate; 2355 } 2356 2357 /* 2358 * Least desirable option: one interrupt vector for everything. 2359 */ 2360 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 2361 iaq->intr_flags_10g = iaq->intr_flags_1g = 0; 2362 #ifdef TCP_OFFLOAD 2363 if (is_offload(sc)) 2364 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 2365 #endif 2366 #ifdef DEV_NETMAP 2367 iaq->nnmrxq10g = iaq->nnmrxq1g = 1; 2368 #endif 2369 2370 allocate: 2371 navail = iaq->nirq; 2372 rc = 0; 2373 if (itype == INTR_MSIX) 2374 rc = pci_alloc_msix(sc->dev, &navail); 2375 else if (itype == INTR_MSI) 2376 rc = pci_alloc_msi(sc->dev, &navail); 2377 2378 if (rc == 0) { 2379 if (navail == iaq->nirq) 2380 return (0); 2381 2382 /* 2383 * Didn't get the number requested. Use whatever number 2384 * the kernel is willing to allocate (it's in navail). 2385 */ 2386 device_printf(sc->dev, "fewer vectors than requested, " 2387 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 2388 itype, iaq->nirq, navail); 2389 pci_release_msi(sc->dev); 2390 goto restart; 2391 } 2392 2393 device_printf(sc->dev, 2394 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 2395 itype, rc, iaq->nirq, navail); 2396 } 2397 2398 device_printf(sc->dev, 2399 "failed to find a usable interrupt type. " 2400 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 2401 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 2402 2403 return (ENXIO); 2404 } 2405 2406 #define FW_VERSION(chip) ( \ 2407 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ 2408 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ 2409 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ 2410 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) 2411 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) 2412 2413 struct fw_info { 2414 uint8_t chip; 2415 char *kld_name; 2416 char *fw_mod_name; 2417 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 2418 } fw_info[] = { 2419 { 2420 .chip = CHELSIO_T4, 2421 .kld_name = "t4fw_cfg", 2422 .fw_mod_name = "t4fw", 2423 .fw_hdr = { 2424 .chip = FW_HDR_CHIP_T4, 2425 .fw_ver = htobe32_const(FW_VERSION(T4)), 2426 .intfver_nic = FW_INTFVER(T4, NIC), 2427 .intfver_vnic = FW_INTFVER(T4, VNIC), 2428 .intfver_ofld = FW_INTFVER(T4, OFLD), 2429 .intfver_ri = FW_INTFVER(T4, RI), 2430 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 2431 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 2432 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 2433 .intfver_fcoe = FW_INTFVER(T4, FCOE), 2434 }, 2435 }, { 2436 .chip = CHELSIO_T5, 2437 .kld_name = "t5fw_cfg", 2438 .fw_mod_name = "t5fw", 2439 .fw_hdr = { 2440 .chip = FW_HDR_CHIP_T5, 2441 .fw_ver = htobe32_const(FW_VERSION(T5)), 2442 .intfver_nic = FW_INTFVER(T5, NIC), 2443 .intfver_vnic = FW_INTFVER(T5, VNIC), 2444 .intfver_ofld = FW_INTFVER(T5, OFLD), 2445 .intfver_ri = FW_INTFVER(T5, RI), 2446 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 2447 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 2448 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 2449 .intfver_fcoe = FW_INTFVER(T5, FCOE), 2450 }, 2451 } 2452 }; 2453 2454 static struct fw_info * 2455 find_fw_info(int chip) 2456 { 2457 int i; 2458 2459 for (i = 0; i < nitems(fw_info); i++) { 2460 if (fw_info[i].chip == chip) 2461 return (&fw_info[i]); 2462 } 2463 return (NULL); 2464 } 2465 2466 /* 2467 * Is the given firmware API compatible with the one the driver was compiled 2468 * with? 2469 */ 2470 static int 2471 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 2472 { 2473 2474 /* short circuit if it's the exact same firmware version */ 2475 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 2476 return (1); 2477 2478 /* 2479 * XXX: Is this too conservative? Perhaps I should limit this to the 2480 * features that are supported in the driver. 2481 */ 2482 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 2483 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 2484 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 2485 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 2486 return (1); 2487 #undef SAME_INTF 2488 2489 return (0); 2490 } 2491 2492 /* 2493 * The firmware in the KLD is usable, but should it be installed? This routine 2494 * explains itself in detail if it indicates the KLD firmware should be 2495 * installed. 2496 */ 2497 static int 2498 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 2499 { 2500 const char *reason; 2501 2502 if (!card_fw_usable) { 2503 reason = "incompatible or unusable"; 2504 goto install; 2505 } 2506 2507 if (k > c) { 2508 reason = "older than the version bundled with this driver"; 2509 goto install; 2510 } 2511 2512 if (t4_fw_install == 2 && k != c) { 2513 reason = "different than the version bundled with this driver"; 2514 goto install; 2515 } 2516 2517 return (0); 2518 2519 install: 2520 if (t4_fw_install == 0) { 2521 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2522 "but the driver is prohibited from installing a different " 2523 "firmware on the card.\n", 2524 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2525 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 2526 2527 return (0); 2528 } 2529 2530 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2531 "installing firmware %u.%u.%u.%u on card.\n", 2532 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2533 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 2534 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2535 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2536 2537 return (1); 2538 } 2539 /* 2540 * Establish contact with the firmware and determine if we are the master driver 2541 * or not, and whether we are responsible for chip initialization. 2542 */ 2543 static int 2544 prep_firmware(struct adapter *sc) 2545 { 2546 const struct firmware *fw = NULL, *default_cfg; 2547 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 2548 enum dev_state state; 2549 struct fw_info *fw_info; 2550 struct fw_hdr *card_fw; /* fw on the card */ 2551 const struct fw_hdr *kld_fw; /* fw in the KLD */ 2552 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 2553 against */ 2554 2555 /* Contact firmware. */ 2556 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 2557 if (rc < 0 || state == DEV_STATE_ERR) { 2558 rc = -rc; 2559 device_printf(sc->dev, 2560 "failed to connect to the firmware: %d, %d.\n", rc, state); 2561 return (rc); 2562 } 2563 pf = rc; 2564 if (pf == sc->mbox) 2565 sc->flags |= MASTER_PF; 2566 else if (state == DEV_STATE_UNINIT) { 2567 /* 2568 * We didn't get to be the master so we definitely won't be 2569 * configuring the chip. It's a bug if someone else hasn't 2570 * configured it already. 2571 */ 2572 device_printf(sc->dev, "couldn't be master(%d), " 2573 "device not already initialized either(%d).\n", rc, state); 2574 return (EDOOFUS); 2575 } 2576 2577 /* This is the firmware whose headers the driver was compiled against */ 2578 fw_info = find_fw_info(chip_id(sc)); 2579 if (fw_info == NULL) { 2580 device_printf(sc->dev, 2581 "unable to look up firmware information for chip %d.\n", 2582 chip_id(sc)); 2583 return (EINVAL); 2584 } 2585 drv_fw = &fw_info->fw_hdr; 2586 2587 /* 2588 * The firmware KLD contains many modules. The KLD name is also the 2589 * name of the module that contains the default config file. 2590 */ 2591 default_cfg = firmware_get(fw_info->kld_name); 2592 2593 /* Read the header of the firmware on the card */ 2594 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 2595 rc = -t4_read_flash(sc, FLASH_FW_START, 2596 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 2597 if (rc == 0) 2598 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 2599 else { 2600 device_printf(sc->dev, 2601 "Unable to read card's firmware header: %d\n", rc); 2602 card_fw_usable = 0; 2603 } 2604 2605 /* This is the firmware in the KLD */ 2606 fw = firmware_get(fw_info->fw_mod_name); 2607 if (fw != NULL) { 2608 kld_fw = (const void *)fw->data; 2609 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 2610 } else { 2611 kld_fw = NULL; 2612 kld_fw_usable = 0; 2613 } 2614 2615 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2616 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) { 2617 /* 2618 * Common case: the firmware on the card is an exact match and 2619 * the KLD is an exact match too, or the KLD is 2620 * absent/incompatible. Note that t4_fw_install = 2 is ignored 2621 * here -- use cxgbetool loadfw if you want to reinstall the 2622 * same firmware as the one on the card. 2623 */ 2624 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2625 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2626 be32toh(card_fw->fw_ver))) { 2627 2628 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 2629 if (rc != 0) { 2630 device_printf(sc->dev, 2631 "failed to install firmware: %d\n", rc); 2632 goto done; 2633 } 2634 2635 /* Installed successfully, update the cached header too. */ 2636 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2637 card_fw_usable = 1; 2638 need_fw_reset = 0; /* already reset as part of load_fw */ 2639 } 2640 2641 if (!card_fw_usable) { 2642 uint32_t d, c, k; 2643 2644 d = ntohl(drv_fw->fw_ver); 2645 c = ntohl(card_fw->fw_ver); 2646 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2647 2648 device_printf(sc->dev, "Cannot find a usable firmware: " 2649 "fw_install %d, chip state %d, " 2650 "driver compiled with %d.%d.%d.%d, " 2651 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2652 t4_fw_install, state, 2653 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2654 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2655 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2656 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2657 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2658 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2659 rc = EINVAL; 2660 goto done; 2661 } 2662 2663 /* We're using whatever's on the card and it's known to be good. */ 2664 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2665 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2666 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2667 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2668 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2669 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2670 t4_get_tp_version(sc, &sc->params.tp_vers); 2671 2672 /* Reset device */ 2673 if (need_fw_reset && 2674 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2675 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2676 if (rc != ETIMEDOUT && rc != EIO) 2677 t4_fw_bye(sc, sc->mbox); 2678 goto done; 2679 } 2680 sc->flags |= FW_OK; 2681 2682 rc = get_params__pre_init(sc); 2683 if (rc != 0) 2684 goto done; /* error message displayed already */ 2685 2686 /* Partition adapter resources as specified in the config file. */ 2687 if (state == DEV_STATE_UNINIT) { 2688 2689 KASSERT(sc->flags & MASTER_PF, 2690 ("%s: trying to change chip settings when not master.", 2691 __func__)); 2692 2693 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2694 if (rc != 0) 2695 goto done; /* error message displayed already */ 2696 2697 t4_tweak_chip_settings(sc); 2698 2699 /* get basic stuff going */ 2700 rc = -t4_fw_initialize(sc, sc->mbox); 2701 if (rc != 0) { 2702 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2703 goto done; 2704 } 2705 } else { 2706 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2707 sc->cfcsum = 0; 2708 } 2709 2710 done: 2711 free(card_fw, M_CXGBE); 2712 if (fw != NULL) 2713 firmware_put(fw, FIRMWARE_UNLOAD); 2714 if (default_cfg != NULL) 2715 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2716 2717 return (rc); 2718 } 2719 2720 #define FW_PARAM_DEV(param) \ 2721 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2722 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2723 #define FW_PARAM_PFVF(param) \ 2724 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2725 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2726 2727 /* 2728 * Partition chip resources for use between various PFs, VFs, etc. 2729 */ 2730 static int 2731 partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2732 const char *name_prefix) 2733 { 2734 const struct firmware *cfg = NULL; 2735 int rc = 0; 2736 struct fw_caps_config_cmd caps; 2737 uint32_t mtype, moff, finicsum, cfcsum; 2738 2739 /* 2740 * Figure out what configuration file to use. Pick the default config 2741 * file for the card if the user hasn't specified one explicitly. 2742 */ 2743 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2744 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2745 /* Card specific overrides go here. */ 2746 if (pci_get_device(sc->dev) == 0x440a) 2747 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2748 if (is_fpga(sc)) 2749 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2750 } 2751 2752 /* 2753 * We need to load another module if the profile is anything except 2754 * "default" or "flash". 2755 */ 2756 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2757 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2758 char s[32]; 2759 2760 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2761 cfg = firmware_get(s); 2762 if (cfg == NULL) { 2763 if (default_cfg != NULL) { 2764 device_printf(sc->dev, 2765 "unable to load module \"%s\" for " 2766 "configuration profile \"%s\", will use " 2767 "the default config file instead.\n", 2768 s, sc->cfg_file); 2769 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2770 "%s", DEFAULT_CF); 2771 } else { 2772 device_printf(sc->dev, 2773 "unable to load module \"%s\" for " 2774 "configuration profile \"%s\", will use " 2775 "the config file on the card's flash " 2776 "instead.\n", s, sc->cfg_file); 2777 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2778 "%s", FLASH_CF); 2779 } 2780 } 2781 } 2782 2783 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2784 default_cfg == NULL) { 2785 device_printf(sc->dev, 2786 "default config file not available, will use the config " 2787 "file on the card's flash instead.\n"); 2788 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2789 } 2790 2791 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2792 u_int cflen, i, n; 2793 const uint32_t *cfdata; 2794 uint32_t param, val, addr, off, mw_base, mw_aperture; 2795 2796 KASSERT(cfg != NULL || default_cfg != NULL, 2797 ("%s: no config to upload", __func__)); 2798 2799 /* 2800 * Ask the firmware where it wants us to upload the config file. 2801 */ 2802 param = FW_PARAM_DEV(CF); 2803 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2804 if (rc != 0) { 2805 /* No support for config file? Shouldn't happen. */ 2806 device_printf(sc->dev, 2807 "failed to query config file location: %d.\n", rc); 2808 goto done; 2809 } 2810 mtype = G_FW_PARAMS_PARAM_Y(val); 2811 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2812 2813 /* 2814 * XXX: sheer laziness. We deliberately added 4 bytes of 2815 * useless stuffing/comments at the end of the config file so 2816 * it's ok to simply throw away the last remaining bytes when 2817 * the config file is not an exact multiple of 4. This also 2818 * helps with the validate_mt_off_len check. 2819 */ 2820 if (cfg != NULL) { 2821 cflen = cfg->datasize & ~3; 2822 cfdata = cfg->data; 2823 } else { 2824 cflen = default_cfg->datasize & ~3; 2825 cfdata = default_cfg->data; 2826 } 2827 2828 if (cflen > FLASH_CFG_MAX_SIZE) { 2829 device_printf(sc->dev, 2830 "config file too long (%d, max allowed is %d). " 2831 "Will try to use the config on the card, if any.\n", 2832 cflen, FLASH_CFG_MAX_SIZE); 2833 goto use_config_on_flash; 2834 } 2835 2836 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2837 if (rc != 0) { 2838 device_printf(sc->dev, 2839 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2840 "Will try to use the config on the card, if any.\n", 2841 __func__, mtype, moff, cflen, rc); 2842 goto use_config_on_flash; 2843 } 2844 2845 memwin_info(sc, 2, &mw_base, &mw_aperture); 2846 while (cflen) { 2847 off = position_memwin(sc, 2, addr); 2848 n = min(cflen, mw_aperture - off); 2849 for (i = 0; i < n; i += 4) 2850 t4_write_reg(sc, mw_base + off + i, *cfdata++); 2851 cflen -= n; 2852 addr += n; 2853 } 2854 } else { 2855 use_config_on_flash: 2856 mtype = FW_MEMTYPE_FLASH; 2857 moff = t4_flash_cfg_addr(sc); 2858 } 2859 2860 bzero(&caps, sizeof(caps)); 2861 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2862 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2863 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2864 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2865 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2866 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2867 if (rc != 0) { 2868 device_printf(sc->dev, 2869 "failed to pre-process config file: %d " 2870 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2871 goto done; 2872 } 2873 2874 finicsum = be32toh(caps.finicsum); 2875 cfcsum = be32toh(caps.cfcsum); 2876 if (finicsum != cfcsum) { 2877 device_printf(sc->dev, 2878 "WARNING: config file checksum mismatch: %08x %08x\n", 2879 finicsum, cfcsum); 2880 } 2881 sc->cfcsum = cfcsum; 2882 2883 #define LIMIT_CAPS(x) do { \ 2884 caps.x &= htobe16(t4_##x##_allowed); \ 2885 } while (0) 2886 2887 /* 2888 * Let the firmware know what features will (not) be used so it can tune 2889 * things accordingly. 2890 */ 2891 LIMIT_CAPS(linkcaps); 2892 LIMIT_CAPS(niccaps); 2893 LIMIT_CAPS(toecaps); 2894 LIMIT_CAPS(rdmacaps); 2895 LIMIT_CAPS(iscsicaps); 2896 LIMIT_CAPS(fcoecaps); 2897 #undef LIMIT_CAPS 2898 2899 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2900 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2901 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2902 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2903 if (rc != 0) { 2904 device_printf(sc->dev, 2905 "failed to process config file: %d.\n", rc); 2906 } 2907 done: 2908 if (cfg != NULL) 2909 firmware_put(cfg, FIRMWARE_UNLOAD); 2910 return (rc); 2911 } 2912 2913 /* 2914 * Retrieve parameters that are needed (or nice to have) very early. 2915 */ 2916 static int 2917 get_params__pre_init(struct adapter *sc) 2918 { 2919 int rc; 2920 uint32_t param[2], val[2]; 2921 struct fw_devlog_cmd cmd; 2922 struct devlog_params *dlog = &sc->params.devlog; 2923 2924 param[0] = FW_PARAM_DEV(PORTVEC); 2925 param[1] = FW_PARAM_DEV(CCLK); 2926 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2927 if (rc != 0) { 2928 device_printf(sc->dev, 2929 "failed to query parameters (pre_init): %d.\n", rc); 2930 return (rc); 2931 } 2932 2933 sc->params.portvec = val[0]; 2934 sc->params.nports = bitcount32(val[0]); 2935 sc->params.vpd.cclk = val[1]; 2936 2937 /* Read device log parameters. */ 2938 bzero(&cmd, sizeof(cmd)); 2939 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2940 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2941 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2942 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2943 if (rc != 0) { 2944 device_printf(sc->dev, 2945 "failed to get devlog parameters: %d.\n", rc); 2946 bzero(dlog, sizeof (*dlog)); 2947 rc = 0; /* devlog isn't critical for device operation */ 2948 } else { 2949 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2950 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2951 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2952 dlog->size = be32toh(cmd.memsize_devlog); 2953 } 2954 2955 return (rc); 2956 } 2957 2958 /* 2959 * Retrieve various parameters that are of interest to the driver. The device 2960 * has been initialized by the firmware at this point. 2961 */ 2962 static int 2963 get_params__post_init(struct adapter *sc) 2964 { 2965 int rc; 2966 uint32_t param[7], val[7]; 2967 struct fw_caps_config_cmd caps; 2968 2969 param[0] = FW_PARAM_PFVF(IQFLINT_START); 2970 param[1] = FW_PARAM_PFVF(EQ_START); 2971 param[2] = FW_PARAM_PFVF(FILTER_START); 2972 param[3] = FW_PARAM_PFVF(FILTER_END); 2973 param[4] = FW_PARAM_PFVF(L2T_START); 2974 param[5] = FW_PARAM_PFVF(L2T_END); 2975 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2976 if (rc != 0) { 2977 device_printf(sc->dev, 2978 "failed to query parameters (post_init): %d.\n", rc); 2979 return (rc); 2980 } 2981 2982 sc->sge.iq_start = val[0]; 2983 sc->sge.eq_start = val[1]; 2984 sc->tids.ftid_base = val[2]; 2985 sc->tids.nftids = val[3] - val[2] + 1; 2986 sc->params.ftid_min = val[2]; 2987 sc->params.ftid_max = val[3]; 2988 sc->vres.l2t.start = val[4]; 2989 sc->vres.l2t.size = val[5] - val[4] + 1; 2990 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2991 ("%s: L2 table size (%u) larger than expected (%u)", 2992 __func__, sc->vres.l2t.size, L2T_SIZE)); 2993 2994 /* get capabilites */ 2995 bzero(&caps, sizeof(caps)); 2996 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2997 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2998 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2999 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 3000 if (rc != 0) { 3001 device_printf(sc->dev, 3002 "failed to get card capabilities: %d.\n", rc); 3003 return (rc); 3004 } 3005 3006 #define READ_CAPS(x) do { \ 3007 sc->x = htobe16(caps.x); \ 3008 } while (0) 3009 READ_CAPS(linkcaps); 3010 READ_CAPS(niccaps); 3011 READ_CAPS(toecaps); 3012 READ_CAPS(rdmacaps); 3013 READ_CAPS(iscsicaps); 3014 READ_CAPS(fcoecaps); 3015 3016 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { 3017 param[0] = FW_PARAM_PFVF(ETHOFLD_START); 3018 param[1] = FW_PARAM_PFVF(ETHOFLD_END); 3019 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 3020 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); 3021 if (rc != 0) { 3022 device_printf(sc->dev, 3023 "failed to query NIC parameters: %d.\n", rc); 3024 return (rc); 3025 } 3026 sc->tids.etid_base = val[0]; 3027 sc->params.etid_min = val[0]; 3028 sc->tids.netids = val[1] - val[0] + 1; 3029 sc->params.netids = sc->tids.netids; 3030 sc->params.eo_wr_cred = val[2]; 3031 sc->params.ethoffload = 1; 3032 } 3033 3034 if (sc->toecaps) { 3035 /* query offload-related parameters */ 3036 param[0] = FW_PARAM_DEV(NTID); 3037 param[1] = FW_PARAM_PFVF(SERVER_START); 3038 param[2] = FW_PARAM_PFVF(SERVER_END); 3039 param[3] = FW_PARAM_PFVF(TDDP_START); 3040 param[4] = FW_PARAM_PFVF(TDDP_END); 3041 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 3042 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 3043 if (rc != 0) { 3044 device_printf(sc->dev, 3045 "failed to query TOE parameters: %d.\n", rc); 3046 return (rc); 3047 } 3048 sc->tids.ntids = val[0]; 3049 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 3050 sc->tids.stid_base = val[1]; 3051 sc->tids.nstids = val[2] - val[1] + 1; 3052 sc->vres.ddp.start = val[3]; 3053 sc->vres.ddp.size = val[4] - val[3] + 1; 3054 sc->params.ofldq_wr_cred = val[5]; 3055 sc->params.offload = 1; 3056 } 3057 if (sc->rdmacaps) { 3058 param[0] = FW_PARAM_PFVF(STAG_START); 3059 param[1] = FW_PARAM_PFVF(STAG_END); 3060 param[2] = FW_PARAM_PFVF(RQ_START); 3061 param[3] = FW_PARAM_PFVF(RQ_END); 3062 param[4] = FW_PARAM_PFVF(PBL_START); 3063 param[5] = FW_PARAM_PFVF(PBL_END); 3064 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 3065 if (rc != 0) { 3066 device_printf(sc->dev, 3067 "failed to query RDMA parameters(1): %d.\n", rc); 3068 return (rc); 3069 } 3070 sc->vres.stag.start = val[0]; 3071 sc->vres.stag.size = val[1] - val[0] + 1; 3072 sc->vres.rq.start = val[2]; 3073 sc->vres.rq.size = val[3] - val[2] + 1; 3074 sc->vres.pbl.start = val[4]; 3075 sc->vres.pbl.size = val[5] - val[4] + 1; 3076 3077 param[0] = FW_PARAM_PFVF(SQRQ_START); 3078 param[1] = FW_PARAM_PFVF(SQRQ_END); 3079 param[2] = FW_PARAM_PFVF(CQ_START); 3080 param[3] = FW_PARAM_PFVF(CQ_END); 3081 param[4] = FW_PARAM_PFVF(OCQ_START); 3082 param[5] = FW_PARAM_PFVF(OCQ_END); 3083 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 3084 if (rc != 0) { 3085 device_printf(sc->dev, 3086 "failed to query RDMA parameters(2): %d.\n", rc); 3087 return (rc); 3088 } 3089 sc->vres.qp.start = val[0]; 3090 sc->vres.qp.size = val[1] - val[0] + 1; 3091 sc->vres.cq.start = val[2]; 3092 sc->vres.cq.size = val[3] - val[2] + 1; 3093 sc->vres.ocq.start = val[4]; 3094 sc->vres.ocq.size = val[5] - val[4] + 1; 3095 } 3096 if (sc->iscsicaps) { 3097 param[0] = FW_PARAM_PFVF(ISCSI_START); 3098 param[1] = FW_PARAM_PFVF(ISCSI_END); 3099 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 3100 if (rc != 0) { 3101 device_printf(sc->dev, 3102 "failed to query iSCSI parameters: %d.\n", rc); 3103 return (rc); 3104 } 3105 sc->vres.iscsi.start = val[0]; 3106 sc->vres.iscsi.size = val[1] - val[0] + 1; 3107 } 3108 3109 /* 3110 * We've got the params we wanted to query via the firmware. Now grab 3111 * some others directly from the chip. 3112 */ 3113 rc = t4_read_chip_settings(sc); 3114 3115 return (rc); 3116 } 3117 3118 static int 3119 set_params__post_init(struct adapter *sc) 3120 { 3121 uint32_t param, val; 3122 3123 /* ask for encapsulated CPLs */ 3124 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 3125 val = 1; 3126 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 3127 3128 return (0); 3129 } 3130 3131 #undef FW_PARAM_PFVF 3132 #undef FW_PARAM_DEV 3133 3134 static void 3135 t4_set_desc(struct adapter *sc) 3136 { 3137 char buf[128]; 3138 struct adapter_params *p = &sc->params; 3139 3140 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, " 3141 "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "", 3142 chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec); 3143 3144 device_set_desc_copy(sc->dev, buf); 3145 } 3146 3147 static void 3148 build_medialist(struct port_info *pi, struct ifmedia *media) 3149 { 3150 int m; 3151 3152 PORT_LOCK(pi); 3153 3154 ifmedia_removeall(media); 3155 3156 m = IFM_ETHER | IFM_FDX; 3157 3158 switch(pi->port_type) { 3159 case FW_PORT_TYPE_BT_XFI: 3160 case FW_PORT_TYPE_BT_XAUI: 3161 ifmedia_add(media, m | IFM_10G_T, 0, NULL); 3162 /* fall through */ 3163 3164 case FW_PORT_TYPE_BT_SGMII: 3165 ifmedia_add(media, m | IFM_1000_T, 0, NULL); 3166 ifmedia_add(media, m | IFM_100_TX, 0, NULL); 3167 ifmedia_add(media, IFM_ETHER | IFM_AUTO, 0, NULL); 3168 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 3169 break; 3170 3171 case FW_PORT_TYPE_CX4: 3172 ifmedia_add(media, m | IFM_10G_CX4, 0, NULL); 3173 ifmedia_set(media, m | IFM_10G_CX4); 3174 break; 3175 3176 case FW_PORT_TYPE_QSFP_10G: 3177 case FW_PORT_TYPE_SFP: 3178 case FW_PORT_TYPE_FIBER_XFI: 3179 case FW_PORT_TYPE_FIBER_XAUI: 3180 switch (pi->mod_type) { 3181 3182 case FW_PORT_MOD_TYPE_LR: 3183 ifmedia_add(media, m | IFM_10G_LR, 0, NULL); 3184 ifmedia_set(media, m | IFM_10G_LR); 3185 break; 3186 3187 case FW_PORT_MOD_TYPE_SR: 3188 ifmedia_add(media, m | IFM_10G_SR, 0, NULL); 3189 ifmedia_set(media, m | IFM_10G_SR); 3190 break; 3191 3192 case FW_PORT_MOD_TYPE_LRM: 3193 ifmedia_add(media, m | IFM_10G_LRM, 0, NULL); 3194 ifmedia_set(media, m | IFM_10G_LRM); 3195 break; 3196 3197 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 3198 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 3199 ifmedia_add(media, m | IFM_10G_TWINAX, 0, NULL); 3200 ifmedia_set(media, m | IFM_10G_TWINAX); 3201 break; 3202 3203 case FW_PORT_MOD_TYPE_NONE: 3204 m &= ~IFM_FDX; 3205 ifmedia_add(media, m | IFM_NONE, 0, NULL); 3206 ifmedia_set(media, m | IFM_NONE); 3207 break; 3208 3209 case FW_PORT_MOD_TYPE_NA: 3210 case FW_PORT_MOD_TYPE_ER: 3211 default: 3212 device_printf(pi->dev, 3213 "unknown port_type (%d), mod_type (%d)\n", 3214 pi->port_type, pi->mod_type); 3215 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 3216 ifmedia_set(media, m | IFM_UNKNOWN); 3217 break; 3218 } 3219 break; 3220 3221 case FW_PORT_TYPE_QSFP: 3222 switch (pi->mod_type) { 3223 3224 case FW_PORT_MOD_TYPE_LR: 3225 ifmedia_add(media, m | IFM_40G_LR4, 0, NULL); 3226 ifmedia_set(media, m | IFM_40G_LR4); 3227 break; 3228 3229 case FW_PORT_MOD_TYPE_SR: 3230 ifmedia_add(media, m | IFM_40G_SR4, 0, NULL); 3231 ifmedia_set(media, m | IFM_40G_SR4); 3232 break; 3233 3234 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 3235 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 3236 ifmedia_add(media, m | IFM_40G_CR4, 0, NULL); 3237 ifmedia_set(media, m | IFM_40G_CR4); 3238 break; 3239 3240 case FW_PORT_MOD_TYPE_NONE: 3241 m &= ~IFM_FDX; 3242 ifmedia_add(media, m | IFM_NONE, 0, NULL); 3243 ifmedia_set(media, m | IFM_NONE); 3244 break; 3245 3246 default: 3247 device_printf(pi->dev, 3248 "unknown port_type (%d), mod_type (%d)\n", 3249 pi->port_type, pi->mod_type); 3250 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 3251 ifmedia_set(media, m | IFM_UNKNOWN); 3252 break; 3253 } 3254 break; 3255 3256 default: 3257 device_printf(pi->dev, 3258 "unknown port_type (%d), mod_type (%d)\n", pi->port_type, 3259 pi->mod_type); 3260 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 3261 ifmedia_set(media, m | IFM_UNKNOWN); 3262 break; 3263 } 3264 3265 PORT_UNLOCK(pi); 3266 } 3267 3268 #define FW_MAC_EXACT_CHUNK 7 3269 3270 /* 3271 * Program the port's XGMAC based on parameters in ifnet. The caller also 3272 * indicates which parameters should be programmed (the rest are left alone). 3273 */ 3274 int 3275 update_mac_settings(struct ifnet *ifp, int flags) 3276 { 3277 int rc = 0; 3278 struct vi_info *vi = ifp->if_softc; 3279 struct port_info *pi = vi->pi; 3280 struct adapter *sc = pi->adapter; 3281 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 3282 3283 ASSERT_SYNCHRONIZED_OP(sc); 3284 KASSERT(flags, ("%s: not told what to update.", __func__)); 3285 3286 if (flags & XGMAC_MTU) 3287 mtu = ifp->if_mtu; 3288 3289 if (flags & XGMAC_PROMISC) 3290 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 3291 3292 if (flags & XGMAC_ALLMULTI) 3293 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 3294 3295 if (flags & XGMAC_VLANEX) 3296 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 3297 3298 if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) { 3299 rc = -t4_set_rxmode(sc, sc->mbox, vi->viid, mtu, promisc, 3300 allmulti, 1, vlanex, false); 3301 if (rc) { 3302 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, 3303 rc); 3304 return (rc); 3305 } 3306 } 3307 3308 if (flags & XGMAC_UCADDR) { 3309 uint8_t ucaddr[ETHER_ADDR_LEN]; 3310 3311 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 3312 rc = t4_change_mac(sc, sc->mbox, vi->viid, vi->xact_addr_filt, 3313 ucaddr, true, true); 3314 if (rc < 0) { 3315 rc = -rc; 3316 if_printf(ifp, "change_mac failed: %d\n", rc); 3317 return (rc); 3318 } else { 3319 vi->xact_addr_filt = rc; 3320 rc = 0; 3321 } 3322 } 3323 3324 if (flags & XGMAC_MCADDRS) { 3325 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 3326 int del = 1; 3327 uint64_t hash = 0; 3328 struct ifmultiaddr *ifma; 3329 int i = 0, j; 3330 3331 if_maddr_rlock(ifp); 3332 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3333 if (ifma->ifma_addr->sa_family != AF_LINK) 3334 continue; 3335 mcaddr[i] = 3336 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 3337 MPASS(ETHER_IS_MULTICAST(mcaddr[i])); 3338 i++; 3339 3340 if (i == FW_MAC_EXACT_CHUNK) { 3341 rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid, 3342 del, i, mcaddr, NULL, &hash, 0); 3343 if (rc < 0) { 3344 rc = -rc; 3345 for (j = 0; j < i; j++) { 3346 if_printf(ifp, 3347 "failed to add mc address" 3348 " %02x:%02x:%02x:" 3349 "%02x:%02x:%02x rc=%d\n", 3350 mcaddr[j][0], mcaddr[j][1], 3351 mcaddr[j][2], mcaddr[j][3], 3352 mcaddr[j][4], mcaddr[j][5], 3353 rc); 3354 } 3355 goto mcfail; 3356 } 3357 del = 0; 3358 i = 0; 3359 } 3360 } 3361 if (i > 0) { 3362 rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid, del, i, 3363 mcaddr, NULL, &hash, 0); 3364 if (rc < 0) { 3365 rc = -rc; 3366 for (j = 0; j < i; j++) { 3367 if_printf(ifp, 3368 "failed to add mc address" 3369 " %02x:%02x:%02x:" 3370 "%02x:%02x:%02x rc=%d\n", 3371 mcaddr[j][0], mcaddr[j][1], 3372 mcaddr[j][2], mcaddr[j][3], 3373 mcaddr[j][4], mcaddr[j][5], 3374 rc); 3375 } 3376 goto mcfail; 3377 } 3378 } 3379 3380 rc = -t4_set_addr_hash(sc, sc->mbox, vi->viid, 0, hash, 0); 3381 if (rc != 0) 3382 if_printf(ifp, "failed to set mc address hash: %d", rc); 3383 mcfail: 3384 if_maddr_runlock(ifp); 3385 } 3386 3387 return (rc); 3388 } 3389 3390 /* 3391 * {begin|end}_synchronized_op must be called from the same thread. 3392 */ 3393 int 3394 begin_synchronized_op(struct adapter *sc, struct vi_info *vi, int flags, 3395 char *wmesg) 3396 { 3397 int rc, pri; 3398 3399 #ifdef WITNESS 3400 /* the caller thinks it's ok to sleep, but is it really? */ 3401 if (flags & SLEEP_OK) 3402 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 3403 "begin_synchronized_op"); 3404 #endif 3405 3406 if (INTR_OK) 3407 pri = PCATCH; 3408 else 3409 pri = 0; 3410 3411 ADAPTER_LOCK(sc); 3412 for (;;) { 3413 3414 if (vi && IS_DOOMED(vi)) { 3415 rc = ENXIO; 3416 goto done; 3417 } 3418 3419 if (!IS_BUSY(sc)) { 3420 rc = 0; 3421 break; 3422 } 3423 3424 if (!(flags & SLEEP_OK)) { 3425 rc = EBUSY; 3426 goto done; 3427 } 3428 3429 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 3430 rc = EINTR; 3431 goto done; 3432 } 3433 } 3434 3435 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 3436 SET_BUSY(sc); 3437 #ifdef INVARIANTS 3438 sc->last_op = wmesg; 3439 sc->last_op_thr = curthread; 3440 sc->last_op_flags = flags; 3441 #endif 3442 3443 done: 3444 if (!(flags & HOLD_LOCK) || rc) 3445 ADAPTER_UNLOCK(sc); 3446 3447 return (rc); 3448 } 3449 3450 /* 3451 * Tell if_ioctl and if_init that the VI is going away. This is 3452 * special variant of begin_synchronized_op and must be paired with a 3453 * call to end_synchronized_op. 3454 */ 3455 void 3456 doom_vi(struct adapter *sc, struct vi_info *vi) 3457 { 3458 3459 ADAPTER_LOCK(sc); 3460 SET_DOOMED(vi); 3461 wakeup(&sc->flags); 3462 while (IS_BUSY(sc)) 3463 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 3464 SET_BUSY(sc); 3465 #ifdef INVARIANTS 3466 sc->last_op = "t4detach"; 3467 sc->last_op_thr = curthread; 3468 sc->last_op_flags = 0; 3469 #endif 3470 ADAPTER_UNLOCK(sc); 3471 } 3472 3473 /* 3474 * {begin|end}_synchronized_op must be called from the same thread. 3475 */ 3476 void 3477 end_synchronized_op(struct adapter *sc, int flags) 3478 { 3479 3480 if (flags & LOCK_HELD) 3481 ADAPTER_LOCK_ASSERT_OWNED(sc); 3482 else 3483 ADAPTER_LOCK(sc); 3484 3485 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 3486 CLR_BUSY(sc); 3487 wakeup(&sc->flags); 3488 ADAPTER_UNLOCK(sc); 3489 } 3490 3491 static int 3492 cxgbe_init_synchronized(struct vi_info *vi) 3493 { 3494 struct port_info *pi = vi->pi; 3495 struct adapter *sc = pi->adapter; 3496 struct ifnet *ifp = vi->ifp; 3497 int rc = 0, i; 3498 struct sge_txq *txq; 3499 3500 ASSERT_SYNCHRONIZED_OP(sc); 3501 3502 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 3503 return (0); /* already running */ 3504 3505 if (!(sc->flags & FULL_INIT_DONE) && 3506 ((rc = adapter_full_init(sc)) != 0)) 3507 return (rc); /* error message displayed already */ 3508 3509 if (!(vi->flags & VI_INIT_DONE) && 3510 ((rc = vi_full_init(vi)) != 0)) 3511 return (rc); /* error message displayed already */ 3512 3513 rc = update_mac_settings(ifp, XGMAC_ALL); 3514 if (rc) 3515 goto done; /* error message displayed already */ 3516 3517 rc = -t4_enable_vi(sc, sc->mbox, vi->viid, true, true); 3518 if (rc != 0) { 3519 if_printf(ifp, "enable_vi failed: %d\n", rc); 3520 goto done; 3521 } 3522 3523 /* 3524 * Can't fail from this point onwards. Review cxgbe_uninit_synchronized 3525 * if this changes. 3526 */ 3527 3528 for_each_txq(vi, i, txq) { 3529 TXQ_LOCK(txq); 3530 txq->eq.flags |= EQ_ENABLED; 3531 TXQ_UNLOCK(txq); 3532 } 3533 3534 /* 3535 * The first iq of the first port to come up is used for tracing. 3536 */ 3537 if (sc->traceq < 0 && IS_MAIN_VI(vi)) { 3538 sc->traceq = sc->sge.rxq[vi->first_rxq].iq.abs_id; 3539 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : 3540 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | 3541 V_QUEUENUMBER(sc->traceq)); 3542 pi->flags |= HAS_TRACEQ; 3543 } 3544 3545 /* all ok */ 3546 PORT_LOCK(pi); 3547 ifp->if_drv_flags |= IFF_DRV_RUNNING; 3548 pi->up_vis++; 3549 3550 if (pi->nvi > 1) 3551 callout_reset(&vi->tick, hz, vi_tick, vi); 3552 else 3553 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 3554 PORT_UNLOCK(pi); 3555 done: 3556 if (rc != 0) 3557 cxgbe_uninit_synchronized(vi); 3558 3559 return (rc); 3560 } 3561 3562 /* 3563 * Idempotent. 3564 */ 3565 static int 3566 cxgbe_uninit_synchronized(struct vi_info *vi) 3567 { 3568 struct port_info *pi = vi->pi; 3569 struct adapter *sc = pi->adapter; 3570 struct ifnet *ifp = vi->ifp; 3571 int rc, i; 3572 struct sge_txq *txq; 3573 3574 ASSERT_SYNCHRONIZED_OP(sc); 3575 3576 if (!(vi->flags & VI_INIT_DONE)) { 3577 KASSERT(!(ifp->if_drv_flags & IFF_DRV_RUNNING), 3578 ("uninited VI is running")); 3579 return (0); 3580 } 3581 3582 /* 3583 * Disable the VI so that all its data in either direction is discarded 3584 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 3585 * tick) intact as the TP can deliver negative advice or data that it's 3586 * holding in its RAM (for an offloaded connection) even after the VI is 3587 * disabled. 3588 */ 3589 rc = -t4_enable_vi(sc, sc->mbox, vi->viid, false, false); 3590 if (rc) { 3591 if_printf(ifp, "disable_vi failed: %d\n", rc); 3592 return (rc); 3593 } 3594 3595 for_each_txq(vi, i, txq) { 3596 TXQ_LOCK(txq); 3597 txq->eq.flags &= ~EQ_ENABLED; 3598 TXQ_UNLOCK(txq); 3599 } 3600 3601 PORT_LOCK(pi); 3602 if (pi->nvi == 1) 3603 callout_stop(&pi->tick); 3604 else 3605 callout_stop(&vi->tick); 3606 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 3607 PORT_UNLOCK(pi); 3608 return (0); 3609 } 3610 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3611 pi->up_vis--; 3612 if (pi->up_vis > 0) { 3613 PORT_UNLOCK(pi); 3614 return (0); 3615 } 3616 PORT_UNLOCK(pi); 3617 3618 pi->link_cfg.link_ok = 0; 3619 pi->link_cfg.speed = 0; 3620 pi->linkdnrc = -1; 3621 t4_os_link_changed(sc, pi->port_id, 0, -1); 3622 3623 return (0); 3624 } 3625 3626 /* 3627 * It is ok for this function to fail midway and return right away. t4_detach 3628 * will walk the entire sc->irq list and clean up whatever is valid. 3629 */ 3630 static int 3631 setup_intr_handlers(struct adapter *sc) 3632 { 3633 int rc, rid, p, q, v; 3634 char s[8]; 3635 struct irq *irq; 3636 struct port_info *pi; 3637 struct vi_info *vi; 3638 struct sge_rxq *rxq; 3639 #ifdef TCP_OFFLOAD 3640 struct sge_ofld_rxq *ofld_rxq; 3641 #endif 3642 #ifdef DEV_NETMAP 3643 struct sge_nm_rxq *nm_rxq; 3644 #endif 3645 #ifdef RSS 3646 int nbuckets = rss_getnumbuckets(); 3647 #endif 3648 3649 /* 3650 * Setup interrupts. 3651 */ 3652 irq = &sc->irq[0]; 3653 rid = sc->intr_type == INTR_INTX ? 0 : 1; 3654 if (sc->intr_count == 1) 3655 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all")); 3656 3657 /* Multiple interrupts. */ 3658 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 3659 ("%s: too few intr.", __func__)); 3660 3661 /* The first one is always error intr */ 3662 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 3663 if (rc != 0) 3664 return (rc); 3665 irq++; 3666 rid++; 3667 3668 /* The second one is always the firmware event queue */ 3669 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt"); 3670 if (rc != 0) 3671 return (rc); 3672 irq++; 3673 rid++; 3674 3675 for_each_port(sc, p) { 3676 pi = sc->port[p]; 3677 for_each_vi(pi, v, vi) { 3678 vi->first_intr = rid - 1; 3679 #ifdef DEV_NETMAP 3680 if (vi->flags & VI_NETMAP) { 3681 for_each_nm_rxq(vi, q, nm_rxq) { 3682 snprintf(s, sizeof(s), "%d-%d", p, q); 3683 rc = t4_alloc_irq(sc, irq, rid, 3684 t4_nm_intr, nm_rxq, s); 3685 if (rc != 0) 3686 return (rc); 3687 irq++; 3688 rid++; 3689 vi->nintr++; 3690 } 3691 continue; 3692 } 3693 #endif 3694 if (vi->flags & INTR_RXQ) { 3695 for_each_rxq(vi, q, rxq) { 3696 if (v == 0) 3697 snprintf(s, sizeof(s), "%d.%d", 3698 p, q); 3699 else 3700 snprintf(s, sizeof(s), 3701 "%d(%d).%d", p, v, q); 3702 rc = t4_alloc_irq(sc, irq, rid, 3703 t4_intr, rxq, s); 3704 if (rc != 0) 3705 return (rc); 3706 #ifdef RSS 3707 bus_bind_intr(sc->dev, irq->res, 3708 rss_getcpu(q % nbuckets)); 3709 #endif 3710 irq++; 3711 rid++; 3712 vi->nintr++; 3713 } 3714 } 3715 #ifdef TCP_OFFLOAD 3716 if (vi->flags & INTR_OFLD_RXQ) { 3717 for_each_ofld_rxq(vi, q, ofld_rxq) { 3718 snprintf(s, sizeof(s), "%d,%d", p, q); 3719 rc = t4_alloc_irq(sc, irq, rid, 3720 t4_intr, ofld_rxq, s); 3721 if (rc != 0) 3722 return (rc); 3723 irq++; 3724 rid++; 3725 vi->nintr++; 3726 } 3727 } 3728 #endif 3729 } 3730 } 3731 MPASS(irq == &sc->irq[sc->intr_count]); 3732 3733 return (0); 3734 } 3735 3736 int 3737 adapter_full_init(struct adapter *sc) 3738 { 3739 int rc, i; 3740 3741 ASSERT_SYNCHRONIZED_OP(sc); 3742 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3743 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 3744 ("%s: FULL_INIT_DONE already", __func__)); 3745 3746 /* 3747 * queues that belong to the adapter (not any particular port). 3748 */ 3749 rc = t4_setup_adapter_queues(sc); 3750 if (rc != 0) 3751 goto done; 3752 3753 for (i = 0; i < nitems(sc->tq); i++) { 3754 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 3755 taskqueue_thread_enqueue, &sc->tq[i]); 3756 if (sc->tq[i] == NULL) { 3757 device_printf(sc->dev, 3758 "failed to allocate task queue %d\n", i); 3759 rc = ENOMEM; 3760 goto done; 3761 } 3762 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3763 device_get_nameunit(sc->dev), i); 3764 } 3765 3766 t4_intr_enable(sc); 3767 sc->flags |= FULL_INIT_DONE; 3768 done: 3769 if (rc != 0) 3770 adapter_full_uninit(sc); 3771 3772 return (rc); 3773 } 3774 3775 int 3776 adapter_full_uninit(struct adapter *sc) 3777 { 3778 int i; 3779 3780 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3781 3782 t4_teardown_adapter_queues(sc); 3783 3784 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3785 taskqueue_free(sc->tq[i]); 3786 sc->tq[i] = NULL; 3787 } 3788 3789 sc->flags &= ~FULL_INIT_DONE; 3790 3791 return (0); 3792 } 3793 3794 #ifdef RSS 3795 #define SUPPORTED_RSS_HASHTYPES (RSS_HASHTYPE_RSS_IPV4 | \ 3796 RSS_HASHTYPE_RSS_TCP_IPV4 | RSS_HASHTYPE_RSS_IPV6 | \ 3797 RSS_HASHTYPE_RSS_TCP_IPV6 | RSS_HASHTYPE_RSS_UDP_IPV4 | \ 3798 RSS_HASHTYPE_RSS_UDP_IPV6) 3799 3800 /* Translates kernel hash types to hardware. */ 3801 static int 3802 hashconfig_to_hashen(int hashconfig) 3803 { 3804 int hashen = 0; 3805 3806 if (hashconfig & RSS_HASHTYPE_RSS_IPV4) 3807 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN; 3808 if (hashconfig & RSS_HASHTYPE_RSS_IPV6) 3809 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN; 3810 if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV4) { 3811 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN | 3812 F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN; 3813 } 3814 if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV6) { 3815 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN | 3816 F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN; 3817 } 3818 if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV4) 3819 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN; 3820 if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV6) 3821 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN; 3822 3823 return (hashen); 3824 } 3825 3826 /* Translates hardware hash types to kernel. */ 3827 static int 3828 hashen_to_hashconfig(int hashen) 3829 { 3830 int hashconfig = 0; 3831 3832 if (hashen & F_FW_RSS_VI_CONFIG_CMD_UDPEN) { 3833 /* 3834 * If UDP hashing was enabled it must have been enabled for 3835 * either IPv4 or IPv6 (inclusive or). Enabling UDP without 3836 * enabling any 4-tuple hash is nonsense configuration. 3837 */ 3838 MPASS(hashen & (F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN | 3839 F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)); 3840 3841 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) 3842 hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV4; 3843 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) 3844 hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV6; 3845 } 3846 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) 3847 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV4; 3848 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) 3849 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV6; 3850 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN) 3851 hashconfig |= RSS_HASHTYPE_RSS_IPV4; 3852 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN) 3853 hashconfig |= RSS_HASHTYPE_RSS_IPV6; 3854 3855 return (hashconfig); 3856 } 3857 #endif 3858 3859 int 3860 vi_full_init(struct vi_info *vi) 3861 { 3862 struct adapter *sc = vi->pi->adapter; 3863 struct ifnet *ifp = vi->ifp; 3864 uint16_t *rss; 3865 struct sge_rxq *rxq; 3866 int rc, i, j, hashen; 3867 #ifdef RSS 3868 int nbuckets = rss_getnumbuckets(); 3869 int hashconfig = rss_gethashconfig(); 3870 int extra; 3871 uint32_t raw_rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; 3872 uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; 3873 #endif 3874 3875 ASSERT_SYNCHRONIZED_OP(sc); 3876 KASSERT((vi->flags & VI_INIT_DONE) == 0, 3877 ("%s: VI_INIT_DONE already", __func__)); 3878 3879 sysctl_ctx_init(&vi->ctx); 3880 vi->flags |= VI_SYSCTL_CTX; 3881 3882 /* 3883 * Allocate tx/rx/fl queues for this VI. 3884 */ 3885 rc = t4_setup_vi_queues(vi); 3886 if (rc != 0) 3887 goto done; /* error message displayed already */ 3888 3889 #ifdef DEV_NETMAP 3890 /* Netmap VIs configure RSS when netmap is enabled. */ 3891 if (vi->flags & VI_NETMAP) { 3892 vi->flags |= VI_INIT_DONE; 3893 return (0); 3894 } 3895 #endif 3896 3897 /* 3898 * Setup RSS for this VI. Save a copy of the RSS table for later use. 3899 */ 3900 if (vi->nrxq > vi->rss_size) { 3901 if_printf(ifp, "nrxq (%d) > hw RSS table size (%d); " 3902 "some queues will never receive traffic.\n", vi->nrxq, 3903 vi->rss_size); 3904 } else if (vi->rss_size % vi->nrxq) { 3905 if_printf(ifp, "nrxq (%d), hw RSS table size (%d); " 3906 "expect uneven traffic distribution.\n", vi->nrxq, 3907 vi->rss_size); 3908 } 3909 #ifdef RSS 3910 MPASS(RSS_KEYSIZE == 40); 3911 if (vi->nrxq != nbuckets) { 3912 if_printf(ifp, "nrxq (%d) != kernel RSS buckets (%d);" 3913 "performance will be impacted.\n", vi->nrxq, nbuckets); 3914 } 3915 3916 rss_getkey((void *)&raw_rss_key[0]); 3917 for (i = 0; i < nitems(rss_key); i++) { 3918 rss_key[i] = htobe32(raw_rss_key[nitems(rss_key) - 1 - i]); 3919 } 3920 t4_write_rss_key(sc, (void *)&rss_key[0], -1); 3921 #endif 3922 rss = malloc(vi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); 3923 for (i = 0; i < vi->rss_size;) { 3924 #ifdef RSS 3925 j = rss_get_indirection_to_bucket(i); 3926 j %= vi->nrxq; 3927 rxq = &sc->sge.rxq[vi->first_rxq + j]; 3928 rss[i++] = rxq->iq.abs_id; 3929 #else 3930 for_each_rxq(vi, j, rxq) { 3931 rss[i++] = rxq->iq.abs_id; 3932 if (i == vi->rss_size) 3933 break; 3934 } 3935 #endif 3936 } 3937 3938 rc = -t4_config_rss_range(sc, sc->mbox, vi->viid, 0, vi->rss_size, rss, 3939 vi->rss_size); 3940 if (rc != 0) { 3941 if_printf(ifp, "rss_config failed: %d\n", rc); 3942 goto done; 3943 } 3944 3945 #ifdef RSS 3946 hashen = hashconfig_to_hashen(hashconfig); 3947 3948 /* 3949 * We may have had to enable some hashes even though the global config 3950 * wants them disabled. This is a potential problem that must be 3951 * reported to the user. 3952 */ 3953 extra = hashen_to_hashconfig(hashen) ^ hashconfig; 3954 3955 /* 3956 * If we consider only the supported hash types, then the enabled hashes 3957 * are a superset of the requested hashes. In other words, there cannot 3958 * be any supported hash that was requested but not enabled, but there 3959 * can be hashes that were not requested but had to be enabled. 3960 */ 3961 extra &= SUPPORTED_RSS_HASHTYPES; 3962 MPASS((extra & hashconfig) == 0); 3963 3964 if (extra) { 3965 if_printf(ifp, 3966 "global RSS config (0x%x) cannot be accomodated.\n", 3967 hashconfig); 3968 } 3969 if (extra & RSS_HASHTYPE_RSS_IPV4) 3970 if_printf(ifp, "IPv4 2-tuple hashing forced on.\n"); 3971 if (extra & RSS_HASHTYPE_RSS_TCP_IPV4) 3972 if_printf(ifp, "TCP/IPv4 4-tuple hashing forced on.\n"); 3973 if (extra & RSS_HASHTYPE_RSS_IPV6) 3974 if_printf(ifp, "IPv6 2-tuple hashing forced on.\n"); 3975 if (extra & RSS_HASHTYPE_RSS_TCP_IPV6) 3976 if_printf(ifp, "TCP/IPv6 4-tuple hashing forced on.\n"); 3977 if (extra & RSS_HASHTYPE_RSS_UDP_IPV4) 3978 if_printf(ifp, "UDP/IPv4 4-tuple hashing forced on.\n"); 3979 if (extra & RSS_HASHTYPE_RSS_UDP_IPV6) 3980 if_printf(ifp, "UDP/IPv6 4-tuple hashing forced on.\n"); 3981 #else 3982 hashen = F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN | 3983 F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN | 3984 F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN | 3985 F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN | F_FW_RSS_VI_CONFIG_CMD_UDPEN; 3986 #endif 3987 rc = -t4_config_vi_rss(sc, sc->mbox, vi->viid, hashen, rss[0]); 3988 if (rc != 0) { 3989 if_printf(ifp, "rss hash/defaultq config failed: %d\n", rc); 3990 goto done; 3991 } 3992 3993 vi->rss = rss; 3994 vi->flags |= VI_INIT_DONE; 3995 done: 3996 if (rc != 0) 3997 vi_full_uninit(vi); 3998 3999 return (rc); 4000 } 4001 4002 /* 4003 * Idempotent. 4004 */ 4005 int 4006 vi_full_uninit(struct vi_info *vi) 4007 { 4008 struct port_info *pi = vi->pi; 4009 struct adapter *sc = pi->adapter; 4010 int i; 4011 struct sge_rxq *rxq; 4012 struct sge_txq *txq; 4013 #ifdef TCP_OFFLOAD 4014 struct sge_ofld_rxq *ofld_rxq; 4015 struct sge_wrq *ofld_txq; 4016 #endif 4017 4018 if (vi->flags & VI_INIT_DONE) { 4019 4020 /* Need to quiesce queues. */ 4021 #ifdef DEV_NETMAP 4022 if (vi->flags & VI_NETMAP) 4023 goto skip; 4024 #endif 4025 4026 /* XXX: Only for the first VI? */ 4027 if (IS_MAIN_VI(vi)) 4028 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]); 4029 4030 for_each_txq(vi, i, txq) { 4031 quiesce_txq(sc, txq); 4032 } 4033 4034 #ifdef TCP_OFFLOAD 4035 for_each_ofld_txq(vi, i, ofld_txq) { 4036 quiesce_wrq(sc, ofld_txq); 4037 } 4038 #endif 4039 4040 for_each_rxq(vi, i, rxq) { 4041 quiesce_iq(sc, &rxq->iq); 4042 quiesce_fl(sc, &rxq->fl); 4043 } 4044 4045 #ifdef TCP_OFFLOAD 4046 for_each_ofld_rxq(vi, i, ofld_rxq) { 4047 quiesce_iq(sc, &ofld_rxq->iq); 4048 quiesce_fl(sc, &ofld_rxq->fl); 4049 } 4050 #endif 4051 free(vi->rss, M_CXGBE); 4052 } 4053 #ifdef DEV_NETMAP 4054 skip: 4055 #endif 4056 4057 t4_teardown_vi_queues(vi); 4058 vi->flags &= ~VI_INIT_DONE; 4059 4060 return (0); 4061 } 4062 4063 static void 4064 quiesce_txq(struct adapter *sc, struct sge_txq *txq) 4065 { 4066 struct sge_eq *eq = &txq->eq; 4067 struct sge_qstat *spg = (void *)&eq->desc[eq->sidx]; 4068 4069 (void) sc; /* unused */ 4070 4071 #ifdef INVARIANTS 4072 TXQ_LOCK(txq); 4073 MPASS((eq->flags & EQ_ENABLED) == 0); 4074 TXQ_UNLOCK(txq); 4075 #endif 4076 4077 /* Wait for the mp_ring to empty. */ 4078 while (!mp_ring_is_idle(txq->r)) { 4079 mp_ring_check_drainage(txq->r, 0); 4080 pause("rquiesce", 1); 4081 } 4082 4083 /* Then wait for the hardware to finish. */ 4084 while (spg->cidx != htobe16(eq->pidx)) 4085 pause("equiesce", 1); 4086 4087 /* Finally, wait for the driver to reclaim all descriptors. */ 4088 while (eq->cidx != eq->pidx) 4089 pause("dquiesce", 1); 4090 } 4091 4092 static void 4093 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq) 4094 { 4095 4096 /* XXXTX */ 4097 } 4098 4099 static void 4100 quiesce_iq(struct adapter *sc, struct sge_iq *iq) 4101 { 4102 (void) sc; /* unused */ 4103 4104 /* Synchronize with the interrupt handler */ 4105 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 4106 pause("iqfree", 1); 4107 } 4108 4109 static void 4110 quiesce_fl(struct adapter *sc, struct sge_fl *fl) 4111 { 4112 mtx_lock(&sc->sfl_lock); 4113 FL_LOCK(fl); 4114 fl->flags |= FL_DOOMED; 4115 FL_UNLOCK(fl); 4116 callout_stop(&sc->sfl_callout); 4117 mtx_unlock(&sc->sfl_lock); 4118 4119 KASSERT((fl->flags & FL_STARVING) == 0, 4120 ("%s: still starving", __func__)); 4121 } 4122 4123 static int 4124 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 4125 driver_intr_t *handler, void *arg, char *name) 4126 { 4127 int rc; 4128 4129 irq->rid = rid; 4130 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 4131 RF_SHAREABLE | RF_ACTIVE); 4132 if (irq->res == NULL) { 4133 device_printf(sc->dev, 4134 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 4135 return (ENOMEM); 4136 } 4137 4138 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 4139 NULL, handler, arg, &irq->tag); 4140 if (rc != 0) { 4141 device_printf(sc->dev, 4142 "failed to setup interrupt for rid %d, name %s: %d\n", 4143 rid, name, rc); 4144 } else if (name) 4145 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 4146 4147 return (rc); 4148 } 4149 4150 static int 4151 t4_free_irq(struct adapter *sc, struct irq *irq) 4152 { 4153 if (irq->tag) 4154 bus_teardown_intr(sc->dev, irq->res, irq->tag); 4155 if (irq->res) 4156 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 4157 4158 bzero(irq, sizeof(*irq)); 4159 4160 return (0); 4161 } 4162 4163 static void 4164 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 4165 unsigned int end) 4166 { 4167 uint32_t *p = (uint32_t *)(buf + start); 4168 4169 for ( ; start <= end; start += sizeof(uint32_t)) 4170 *p++ = t4_read_reg(sc, start); 4171 } 4172 4173 static void 4174 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 4175 { 4176 int i, n; 4177 const unsigned int *reg_ranges; 4178 static const unsigned int t4_reg_ranges[] = { 4179 0x1008, 0x1108, 4180 0x1180, 0x11b4, 4181 0x11fc, 0x123c, 4182 0x1300, 0x173c, 4183 0x1800, 0x18fc, 4184 0x3000, 0x30d8, 4185 0x30e0, 0x5924, 4186 0x5960, 0x59d4, 4187 0x5a00, 0x5af8, 4188 0x6000, 0x6098, 4189 0x6100, 0x6150, 4190 0x6200, 0x6208, 4191 0x6240, 0x6248, 4192 0x6280, 0x6338, 4193 0x6370, 0x638c, 4194 0x6400, 0x643c, 4195 0x6500, 0x6524, 4196 0x6a00, 0x6a38, 4197 0x6a60, 0x6a78, 4198 0x6b00, 0x6b84, 4199 0x6bf0, 0x6c84, 4200 0x6cf0, 0x6d84, 4201 0x6df0, 0x6e84, 4202 0x6ef0, 0x6f84, 4203 0x6ff0, 0x7084, 4204 0x70f0, 0x7184, 4205 0x71f0, 0x7284, 4206 0x72f0, 0x7384, 4207 0x73f0, 0x7450, 4208 0x7500, 0x7530, 4209 0x7600, 0x761c, 4210 0x7680, 0x76cc, 4211 0x7700, 0x7798, 4212 0x77c0, 0x77fc, 4213 0x7900, 0x79fc, 4214 0x7b00, 0x7c38, 4215 0x7d00, 0x7efc, 4216 0x8dc0, 0x8e1c, 4217 0x8e30, 0x8e78, 4218 0x8ea0, 0x8f6c, 4219 0x8fc0, 0x9074, 4220 0x90fc, 0x90fc, 4221 0x9400, 0x9458, 4222 0x9600, 0x96bc, 4223 0x9800, 0x9808, 4224 0x9820, 0x983c, 4225 0x9850, 0x9864, 4226 0x9c00, 0x9c6c, 4227 0x9c80, 0x9cec, 4228 0x9d00, 0x9d6c, 4229 0x9d80, 0x9dec, 4230 0x9e00, 0x9e6c, 4231 0x9e80, 0x9eec, 4232 0x9f00, 0x9f6c, 4233 0x9f80, 0x9fec, 4234 0xd004, 0xd03c, 4235 0xdfc0, 0xdfe0, 4236 0xe000, 0xea7c, 4237 0xf000, 0x11110, 4238 0x11118, 0x11190, 4239 0x19040, 0x1906c, 4240 0x19078, 0x19080, 4241 0x1908c, 0x19124, 4242 0x19150, 0x191b0, 4243 0x191d0, 0x191e8, 4244 0x19238, 0x1924c, 4245 0x193f8, 0x19474, 4246 0x19490, 0x194f8, 4247 0x19800, 0x19f30, 4248 0x1a000, 0x1a06c, 4249 0x1a0b0, 0x1a120, 4250 0x1a128, 0x1a138, 4251 0x1a190, 0x1a1c4, 4252 0x1a1fc, 0x1a1fc, 4253 0x1e040, 0x1e04c, 4254 0x1e284, 0x1e28c, 4255 0x1e2c0, 0x1e2c0, 4256 0x1e2e0, 0x1e2e0, 4257 0x1e300, 0x1e384, 4258 0x1e3c0, 0x1e3c8, 4259 0x1e440, 0x1e44c, 4260 0x1e684, 0x1e68c, 4261 0x1e6c0, 0x1e6c0, 4262 0x1e6e0, 0x1e6e0, 4263 0x1e700, 0x1e784, 4264 0x1e7c0, 0x1e7c8, 4265 0x1e840, 0x1e84c, 4266 0x1ea84, 0x1ea8c, 4267 0x1eac0, 0x1eac0, 4268 0x1eae0, 0x1eae0, 4269 0x1eb00, 0x1eb84, 4270 0x1ebc0, 0x1ebc8, 4271 0x1ec40, 0x1ec4c, 4272 0x1ee84, 0x1ee8c, 4273 0x1eec0, 0x1eec0, 4274 0x1eee0, 0x1eee0, 4275 0x1ef00, 0x1ef84, 4276 0x1efc0, 0x1efc8, 4277 0x1f040, 0x1f04c, 4278 0x1f284, 0x1f28c, 4279 0x1f2c0, 0x1f2c0, 4280 0x1f2e0, 0x1f2e0, 4281 0x1f300, 0x1f384, 4282 0x1f3c0, 0x1f3c8, 4283 0x1f440, 0x1f44c, 4284 0x1f684, 0x1f68c, 4285 0x1f6c0, 0x1f6c0, 4286 0x1f6e0, 0x1f6e0, 4287 0x1f700, 0x1f784, 4288 0x1f7c0, 0x1f7c8, 4289 0x1f840, 0x1f84c, 4290 0x1fa84, 0x1fa8c, 4291 0x1fac0, 0x1fac0, 4292 0x1fae0, 0x1fae0, 4293 0x1fb00, 0x1fb84, 4294 0x1fbc0, 0x1fbc8, 4295 0x1fc40, 0x1fc4c, 4296 0x1fe84, 0x1fe8c, 4297 0x1fec0, 0x1fec0, 4298 0x1fee0, 0x1fee0, 4299 0x1ff00, 0x1ff84, 4300 0x1ffc0, 0x1ffc8, 4301 0x20000, 0x2002c, 4302 0x20100, 0x2013c, 4303 0x20190, 0x201c8, 4304 0x20200, 0x20318, 4305 0x20400, 0x20528, 4306 0x20540, 0x20614, 4307 0x21000, 0x21040, 4308 0x2104c, 0x21060, 4309 0x210c0, 0x210ec, 4310 0x21200, 0x21268, 4311 0x21270, 0x21284, 4312 0x212fc, 0x21388, 4313 0x21400, 0x21404, 4314 0x21500, 0x21518, 4315 0x2152c, 0x2153c, 4316 0x21550, 0x21554, 4317 0x21600, 0x21600, 4318 0x21608, 0x21628, 4319 0x21630, 0x2163c, 4320 0x21700, 0x2171c, 4321 0x21780, 0x2178c, 4322 0x21800, 0x21c38, 4323 0x21c80, 0x21d7c, 4324 0x21e00, 0x21e04, 4325 0x22000, 0x2202c, 4326 0x22100, 0x2213c, 4327 0x22190, 0x221c8, 4328 0x22200, 0x22318, 4329 0x22400, 0x22528, 4330 0x22540, 0x22614, 4331 0x23000, 0x23040, 4332 0x2304c, 0x23060, 4333 0x230c0, 0x230ec, 4334 0x23200, 0x23268, 4335 0x23270, 0x23284, 4336 0x232fc, 0x23388, 4337 0x23400, 0x23404, 4338 0x23500, 0x23518, 4339 0x2352c, 0x2353c, 4340 0x23550, 0x23554, 4341 0x23600, 0x23600, 4342 0x23608, 0x23628, 4343 0x23630, 0x2363c, 4344 0x23700, 0x2371c, 4345 0x23780, 0x2378c, 4346 0x23800, 0x23c38, 4347 0x23c80, 0x23d7c, 4348 0x23e00, 0x23e04, 4349 0x24000, 0x2402c, 4350 0x24100, 0x2413c, 4351 0x24190, 0x241c8, 4352 0x24200, 0x24318, 4353 0x24400, 0x24528, 4354 0x24540, 0x24614, 4355 0x25000, 0x25040, 4356 0x2504c, 0x25060, 4357 0x250c0, 0x250ec, 4358 0x25200, 0x25268, 4359 0x25270, 0x25284, 4360 0x252fc, 0x25388, 4361 0x25400, 0x25404, 4362 0x25500, 0x25518, 4363 0x2552c, 0x2553c, 4364 0x25550, 0x25554, 4365 0x25600, 0x25600, 4366 0x25608, 0x25628, 4367 0x25630, 0x2563c, 4368 0x25700, 0x2571c, 4369 0x25780, 0x2578c, 4370 0x25800, 0x25c38, 4371 0x25c80, 0x25d7c, 4372 0x25e00, 0x25e04, 4373 0x26000, 0x2602c, 4374 0x26100, 0x2613c, 4375 0x26190, 0x261c8, 4376 0x26200, 0x26318, 4377 0x26400, 0x26528, 4378 0x26540, 0x26614, 4379 0x27000, 0x27040, 4380 0x2704c, 0x27060, 4381 0x270c0, 0x270ec, 4382 0x27200, 0x27268, 4383 0x27270, 0x27284, 4384 0x272fc, 0x27388, 4385 0x27400, 0x27404, 4386 0x27500, 0x27518, 4387 0x2752c, 0x2753c, 4388 0x27550, 0x27554, 4389 0x27600, 0x27600, 4390 0x27608, 0x27628, 4391 0x27630, 0x2763c, 4392 0x27700, 0x2771c, 4393 0x27780, 0x2778c, 4394 0x27800, 0x27c38, 4395 0x27c80, 0x27d7c, 4396 0x27e00, 0x27e04 4397 }; 4398 static const unsigned int t5_reg_ranges[] = { 4399 0x1008, 0x1148, 4400 0x1180, 0x11b4, 4401 0x11fc, 0x123c, 4402 0x1280, 0x173c, 4403 0x1800, 0x18fc, 4404 0x3000, 0x3028, 4405 0x3060, 0x30d8, 4406 0x30e0, 0x30fc, 4407 0x3140, 0x357c, 4408 0x35a8, 0x35cc, 4409 0x35ec, 0x35ec, 4410 0x3600, 0x5624, 4411 0x56cc, 0x575c, 4412 0x580c, 0x5814, 4413 0x5890, 0x58bc, 4414 0x5940, 0x59dc, 4415 0x59fc, 0x5a18, 4416 0x5a60, 0x5a9c, 4417 0x5b94, 0x5bfc, 4418 0x6000, 0x6040, 4419 0x6058, 0x614c, 4420 0x7700, 0x7798, 4421 0x77c0, 0x78fc, 4422 0x7b00, 0x7c54, 4423 0x7d00, 0x7efc, 4424 0x8dc0, 0x8de0, 4425 0x8df8, 0x8e84, 4426 0x8ea0, 0x8f84, 4427 0x8fc0, 0x90f8, 4428 0x9400, 0x9470, 4429 0x9600, 0x96f4, 4430 0x9800, 0x9808, 4431 0x9820, 0x983c, 4432 0x9850, 0x9864, 4433 0x9c00, 0x9c6c, 4434 0x9c80, 0x9cec, 4435 0x9d00, 0x9d6c, 4436 0x9d80, 0x9dec, 4437 0x9e00, 0x9e6c, 4438 0x9e80, 0x9eec, 4439 0x9f00, 0x9f6c, 4440 0x9f80, 0xa020, 4441 0xd004, 0xd03c, 4442 0xdfc0, 0xdfe0, 4443 0xe000, 0x11088, 4444 0x1109c, 0x11110, 4445 0x11118, 0x1117c, 4446 0x11190, 0x11204, 4447 0x19040, 0x1906c, 4448 0x19078, 0x19080, 4449 0x1908c, 0x19124, 4450 0x19150, 0x191b0, 4451 0x191d0, 0x191e8, 4452 0x19238, 0x19290, 4453 0x193f8, 0x19474, 4454 0x19490, 0x194cc, 4455 0x194f0, 0x194f8, 4456 0x19c00, 0x19c60, 4457 0x19c94, 0x19e10, 4458 0x19e50, 0x19f34, 4459 0x19f40, 0x19f50, 4460 0x19f90, 0x19fe4, 4461 0x1a000, 0x1a06c, 4462 0x1a0b0, 0x1a120, 4463 0x1a128, 0x1a138, 4464 0x1a190, 0x1a1c4, 4465 0x1a1fc, 0x1a1fc, 4466 0x1e008, 0x1e00c, 4467 0x1e040, 0x1e04c, 4468 0x1e284, 0x1e290, 4469 0x1e2c0, 0x1e2c0, 4470 0x1e2e0, 0x1e2e0, 4471 0x1e300, 0x1e384, 4472 0x1e3c0, 0x1e3c8, 4473 0x1e408, 0x1e40c, 4474 0x1e440, 0x1e44c, 4475 0x1e684, 0x1e690, 4476 0x1e6c0, 0x1e6c0, 4477 0x1e6e0, 0x1e6e0, 4478 0x1e700, 0x1e784, 4479 0x1e7c0, 0x1e7c8, 4480 0x1e808, 0x1e80c, 4481 0x1e840, 0x1e84c, 4482 0x1ea84, 0x1ea90, 4483 0x1eac0, 0x1eac0, 4484 0x1eae0, 0x1eae0, 4485 0x1eb00, 0x1eb84, 4486 0x1ebc0, 0x1ebc8, 4487 0x1ec08, 0x1ec0c, 4488 0x1ec40, 0x1ec4c, 4489 0x1ee84, 0x1ee90, 4490 0x1eec0, 0x1eec0, 4491 0x1eee0, 0x1eee0, 4492 0x1ef00, 0x1ef84, 4493 0x1efc0, 0x1efc8, 4494 0x1f008, 0x1f00c, 4495 0x1f040, 0x1f04c, 4496 0x1f284, 0x1f290, 4497 0x1f2c0, 0x1f2c0, 4498 0x1f2e0, 0x1f2e0, 4499 0x1f300, 0x1f384, 4500 0x1f3c0, 0x1f3c8, 4501 0x1f408, 0x1f40c, 4502 0x1f440, 0x1f44c, 4503 0x1f684, 0x1f690, 4504 0x1f6c0, 0x1f6c0, 4505 0x1f6e0, 0x1f6e0, 4506 0x1f700, 0x1f784, 4507 0x1f7c0, 0x1f7c8, 4508 0x1f808, 0x1f80c, 4509 0x1f840, 0x1f84c, 4510 0x1fa84, 0x1fa90, 4511 0x1fac0, 0x1fac0, 4512 0x1fae0, 0x1fae0, 4513 0x1fb00, 0x1fb84, 4514 0x1fbc0, 0x1fbc8, 4515 0x1fc08, 0x1fc0c, 4516 0x1fc40, 0x1fc4c, 4517 0x1fe84, 0x1fe90, 4518 0x1fec0, 0x1fec0, 4519 0x1fee0, 0x1fee0, 4520 0x1ff00, 0x1ff84, 4521 0x1ffc0, 0x1ffc8, 4522 0x30000, 0x30030, 4523 0x30100, 0x30144, 4524 0x30190, 0x301d0, 4525 0x30200, 0x30318, 4526 0x30400, 0x3052c, 4527 0x30540, 0x3061c, 4528 0x30800, 0x30834, 4529 0x308c0, 0x30908, 4530 0x30910, 0x309ac, 4531 0x30a00, 0x30a2c, 4532 0x30a44, 0x30a50, 4533 0x30a74, 0x30c24, 4534 0x30d00, 0x30d00, 4535 0x30d08, 0x30d14, 4536 0x30d1c, 0x30d20, 4537 0x30d3c, 0x30d50, 4538 0x31200, 0x3120c, 4539 0x31220, 0x31220, 4540 0x31240, 0x31240, 4541 0x31600, 0x3160c, 4542 0x31a00, 0x31a1c, 4543 0x31e00, 0x31e20, 4544 0x31e38, 0x31e3c, 4545 0x31e80, 0x31e80, 4546 0x31e88, 0x31ea8, 4547 0x31eb0, 0x31eb4, 4548 0x31ec8, 0x31ed4, 4549 0x31fb8, 0x32004, 4550 0x32200, 0x32200, 4551 0x32208, 0x32240, 4552 0x32248, 0x32280, 4553 0x32288, 0x322c0, 4554 0x322c8, 0x322fc, 4555 0x32600, 0x32630, 4556 0x32a00, 0x32abc, 4557 0x32b00, 0x32b70, 4558 0x33000, 0x33048, 4559 0x33060, 0x3309c, 4560 0x330f0, 0x33148, 4561 0x33160, 0x3319c, 4562 0x331f0, 0x332e4, 4563 0x332f8, 0x333e4, 4564 0x333f8, 0x33448, 4565 0x33460, 0x3349c, 4566 0x334f0, 0x33548, 4567 0x33560, 0x3359c, 4568 0x335f0, 0x336e4, 4569 0x336f8, 0x337e4, 4570 0x337f8, 0x337fc, 4571 0x33814, 0x33814, 4572 0x3382c, 0x3382c, 4573 0x33880, 0x3388c, 4574 0x338e8, 0x338ec, 4575 0x33900, 0x33948, 4576 0x33960, 0x3399c, 4577 0x339f0, 0x33ae4, 4578 0x33af8, 0x33b10, 4579 0x33b28, 0x33b28, 4580 0x33b3c, 0x33b50, 4581 0x33bf0, 0x33c10, 4582 0x33c28, 0x33c28, 4583 0x33c3c, 0x33c50, 4584 0x33cf0, 0x33cfc, 4585 0x34000, 0x34030, 4586 0x34100, 0x34144, 4587 0x34190, 0x341d0, 4588 0x34200, 0x34318, 4589 0x34400, 0x3452c, 4590 0x34540, 0x3461c, 4591 0x34800, 0x34834, 4592 0x348c0, 0x34908, 4593 0x34910, 0x349ac, 4594 0x34a00, 0x34a2c, 4595 0x34a44, 0x34a50, 4596 0x34a74, 0x34c24, 4597 0x34d00, 0x34d00, 4598 0x34d08, 0x34d14, 4599 0x34d1c, 0x34d20, 4600 0x34d3c, 0x34d50, 4601 0x35200, 0x3520c, 4602 0x35220, 0x35220, 4603 0x35240, 0x35240, 4604 0x35600, 0x3560c, 4605 0x35a00, 0x35a1c, 4606 0x35e00, 0x35e20, 4607 0x35e38, 0x35e3c, 4608 0x35e80, 0x35e80, 4609 0x35e88, 0x35ea8, 4610 0x35eb0, 0x35eb4, 4611 0x35ec8, 0x35ed4, 4612 0x35fb8, 0x36004, 4613 0x36200, 0x36200, 4614 0x36208, 0x36240, 4615 0x36248, 0x36280, 4616 0x36288, 0x362c0, 4617 0x362c8, 0x362fc, 4618 0x36600, 0x36630, 4619 0x36a00, 0x36abc, 4620 0x36b00, 0x36b70, 4621 0x37000, 0x37048, 4622 0x37060, 0x3709c, 4623 0x370f0, 0x37148, 4624 0x37160, 0x3719c, 4625 0x371f0, 0x372e4, 4626 0x372f8, 0x373e4, 4627 0x373f8, 0x37448, 4628 0x37460, 0x3749c, 4629 0x374f0, 0x37548, 4630 0x37560, 0x3759c, 4631 0x375f0, 0x376e4, 4632 0x376f8, 0x377e4, 4633 0x377f8, 0x377fc, 4634 0x37814, 0x37814, 4635 0x3782c, 0x3782c, 4636 0x37880, 0x3788c, 4637 0x378e8, 0x378ec, 4638 0x37900, 0x37948, 4639 0x37960, 0x3799c, 4640 0x379f0, 0x37ae4, 4641 0x37af8, 0x37b10, 4642 0x37b28, 0x37b28, 4643 0x37b3c, 0x37b50, 4644 0x37bf0, 0x37c10, 4645 0x37c28, 0x37c28, 4646 0x37c3c, 0x37c50, 4647 0x37cf0, 0x37cfc, 4648 0x38000, 0x38030, 4649 0x38100, 0x38144, 4650 0x38190, 0x381d0, 4651 0x38200, 0x38318, 4652 0x38400, 0x3852c, 4653 0x38540, 0x3861c, 4654 0x38800, 0x38834, 4655 0x388c0, 0x38908, 4656 0x38910, 0x389ac, 4657 0x38a00, 0x38a2c, 4658 0x38a44, 0x38a50, 4659 0x38a74, 0x38c24, 4660 0x38d00, 0x38d00, 4661 0x38d08, 0x38d14, 4662 0x38d1c, 0x38d20, 4663 0x38d3c, 0x38d50, 4664 0x39200, 0x3920c, 4665 0x39220, 0x39220, 4666 0x39240, 0x39240, 4667 0x39600, 0x3960c, 4668 0x39a00, 0x39a1c, 4669 0x39e00, 0x39e20, 4670 0x39e38, 0x39e3c, 4671 0x39e80, 0x39e80, 4672 0x39e88, 0x39ea8, 4673 0x39eb0, 0x39eb4, 4674 0x39ec8, 0x39ed4, 4675 0x39fb8, 0x3a004, 4676 0x3a200, 0x3a200, 4677 0x3a208, 0x3a240, 4678 0x3a248, 0x3a280, 4679 0x3a288, 0x3a2c0, 4680 0x3a2c8, 0x3a2fc, 4681 0x3a600, 0x3a630, 4682 0x3aa00, 0x3aabc, 4683 0x3ab00, 0x3ab70, 4684 0x3b000, 0x3b048, 4685 0x3b060, 0x3b09c, 4686 0x3b0f0, 0x3b148, 4687 0x3b160, 0x3b19c, 4688 0x3b1f0, 0x3b2e4, 4689 0x3b2f8, 0x3b3e4, 4690 0x3b3f8, 0x3b448, 4691 0x3b460, 0x3b49c, 4692 0x3b4f0, 0x3b548, 4693 0x3b560, 0x3b59c, 4694 0x3b5f0, 0x3b6e4, 4695 0x3b6f8, 0x3b7e4, 4696 0x3b7f8, 0x3b7fc, 4697 0x3b814, 0x3b814, 4698 0x3b82c, 0x3b82c, 4699 0x3b880, 0x3b88c, 4700 0x3b8e8, 0x3b8ec, 4701 0x3b900, 0x3b948, 4702 0x3b960, 0x3b99c, 4703 0x3b9f0, 0x3bae4, 4704 0x3baf8, 0x3bb10, 4705 0x3bb28, 0x3bb28, 4706 0x3bb3c, 0x3bb50, 4707 0x3bbf0, 0x3bc10, 4708 0x3bc28, 0x3bc28, 4709 0x3bc3c, 0x3bc50, 4710 0x3bcf0, 0x3bcfc, 4711 0x3c000, 0x3c030, 4712 0x3c100, 0x3c144, 4713 0x3c190, 0x3c1d0, 4714 0x3c200, 0x3c318, 4715 0x3c400, 0x3c52c, 4716 0x3c540, 0x3c61c, 4717 0x3c800, 0x3c834, 4718 0x3c8c0, 0x3c908, 4719 0x3c910, 0x3c9ac, 4720 0x3ca00, 0x3ca2c, 4721 0x3ca44, 0x3ca50, 4722 0x3ca74, 0x3cc24, 4723 0x3cd00, 0x3cd00, 4724 0x3cd08, 0x3cd14, 4725 0x3cd1c, 0x3cd20, 4726 0x3cd3c, 0x3cd50, 4727 0x3d200, 0x3d20c, 4728 0x3d220, 0x3d220, 4729 0x3d240, 0x3d240, 4730 0x3d600, 0x3d60c, 4731 0x3da00, 0x3da1c, 4732 0x3de00, 0x3de20, 4733 0x3de38, 0x3de3c, 4734 0x3de80, 0x3de80, 4735 0x3de88, 0x3dea8, 4736 0x3deb0, 0x3deb4, 4737 0x3dec8, 0x3ded4, 4738 0x3dfb8, 0x3e004, 4739 0x3e200, 0x3e200, 4740 0x3e208, 0x3e240, 4741 0x3e248, 0x3e280, 4742 0x3e288, 0x3e2c0, 4743 0x3e2c8, 0x3e2fc, 4744 0x3e600, 0x3e630, 4745 0x3ea00, 0x3eabc, 4746 0x3eb00, 0x3eb70, 4747 0x3f000, 0x3f048, 4748 0x3f060, 0x3f09c, 4749 0x3f0f0, 0x3f148, 4750 0x3f160, 0x3f19c, 4751 0x3f1f0, 0x3f2e4, 4752 0x3f2f8, 0x3f3e4, 4753 0x3f3f8, 0x3f448, 4754 0x3f460, 0x3f49c, 4755 0x3f4f0, 0x3f548, 4756 0x3f560, 0x3f59c, 4757 0x3f5f0, 0x3f6e4, 4758 0x3f6f8, 0x3f7e4, 4759 0x3f7f8, 0x3f7fc, 4760 0x3f814, 0x3f814, 4761 0x3f82c, 0x3f82c, 4762 0x3f880, 0x3f88c, 4763 0x3f8e8, 0x3f8ec, 4764 0x3f900, 0x3f948, 4765 0x3f960, 0x3f99c, 4766 0x3f9f0, 0x3fae4, 4767 0x3faf8, 0x3fb10, 4768 0x3fb28, 0x3fb28, 4769 0x3fb3c, 0x3fb50, 4770 0x3fbf0, 0x3fc10, 4771 0x3fc28, 0x3fc28, 4772 0x3fc3c, 0x3fc50, 4773 0x3fcf0, 0x3fcfc, 4774 0x40000, 0x4000c, 4775 0x40040, 0x40068, 4776 0x4007c, 0x40144, 4777 0x40180, 0x4018c, 4778 0x40200, 0x40298, 4779 0x402ac, 0x4033c, 4780 0x403f8, 0x403fc, 4781 0x41304, 0x413c4, 4782 0x41400, 0x4141c, 4783 0x41480, 0x414d0, 4784 0x44000, 0x44078, 4785 0x440c0, 0x44278, 4786 0x442c0, 0x44478, 4787 0x444c0, 0x44678, 4788 0x446c0, 0x44878, 4789 0x448c0, 0x449fc, 4790 0x45000, 0x45068, 4791 0x45080, 0x45084, 4792 0x450a0, 0x450b0, 4793 0x45200, 0x45268, 4794 0x45280, 0x45284, 4795 0x452a0, 0x452b0, 4796 0x460c0, 0x460e4, 4797 0x47000, 0x4708c, 4798 0x47200, 0x47250, 4799 0x47400, 0x47420, 4800 0x47600, 0x47618, 4801 0x47800, 0x47814, 4802 0x48000, 0x4800c, 4803 0x48040, 0x48068, 4804 0x4807c, 0x48144, 4805 0x48180, 0x4818c, 4806 0x48200, 0x48298, 4807 0x482ac, 0x4833c, 4808 0x483f8, 0x483fc, 4809 0x49304, 0x493c4, 4810 0x49400, 0x4941c, 4811 0x49480, 0x494d0, 4812 0x4c000, 0x4c078, 4813 0x4c0c0, 0x4c278, 4814 0x4c2c0, 0x4c478, 4815 0x4c4c0, 0x4c678, 4816 0x4c6c0, 0x4c878, 4817 0x4c8c0, 0x4c9fc, 4818 0x4d000, 0x4d068, 4819 0x4d080, 0x4d084, 4820 0x4d0a0, 0x4d0b0, 4821 0x4d200, 0x4d268, 4822 0x4d280, 0x4d284, 4823 0x4d2a0, 0x4d2b0, 4824 0x4e0c0, 0x4e0e4, 4825 0x4f000, 0x4f08c, 4826 0x4f200, 0x4f250, 4827 0x4f400, 0x4f420, 4828 0x4f600, 0x4f618, 4829 0x4f800, 0x4f814, 4830 0x50000, 0x500cc, 4831 0x50400, 0x50400, 4832 0x50800, 0x508cc, 4833 0x50c00, 0x50c00, 4834 0x51000, 0x5101c, 4835 0x51300, 0x51308, 4836 }; 4837 4838 if (is_t4(sc)) { 4839 reg_ranges = &t4_reg_ranges[0]; 4840 n = nitems(t4_reg_ranges); 4841 } else { 4842 reg_ranges = &t5_reg_ranges[0]; 4843 n = nitems(t5_reg_ranges); 4844 } 4845 4846 regs->version = chip_id(sc) | chip_rev(sc) << 10; 4847 for (i = 0; i < n; i += 2) 4848 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 4849 } 4850 4851 #define A_PL_INDIR_CMD 0x1f8 4852 4853 #define S_PL_AUTOINC 31 4854 #define M_PL_AUTOINC 0x1U 4855 #define V_PL_AUTOINC(x) ((x) << S_PL_AUTOINC) 4856 #define G_PL_AUTOINC(x) (((x) >> S_PL_AUTOINC) & M_PL_AUTOINC) 4857 4858 #define S_PL_VFID 20 4859 #define M_PL_VFID 0xffU 4860 #define V_PL_VFID(x) ((x) << S_PL_VFID) 4861 #define G_PL_VFID(x) (((x) >> S_PL_VFID) & M_PL_VFID) 4862 4863 #define S_PL_ADDR 0 4864 #define M_PL_ADDR 0xfffffU 4865 #define V_PL_ADDR(x) ((x) << S_PL_ADDR) 4866 #define G_PL_ADDR(x) (((x) >> S_PL_ADDR) & M_PL_ADDR) 4867 4868 #define A_PL_INDIR_DATA 0x1fc 4869 4870 static uint64_t 4871 read_vf_stat(struct adapter *sc, unsigned int viid, int reg) 4872 { 4873 u32 stats[2]; 4874 4875 mtx_assert(&sc->regwin_lock, MA_OWNED); 4876 t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) | 4877 V_PL_VFID(G_FW_VIID_VIN(viid)) | V_PL_ADDR(VF_MPS_REG(reg))); 4878 stats[0] = t4_read_reg(sc, A_PL_INDIR_DATA); 4879 stats[1] = t4_read_reg(sc, A_PL_INDIR_DATA); 4880 return (((uint64_t)stats[1]) << 32 | stats[0]); 4881 } 4882 4883 static void 4884 t4_get_vi_stats(struct adapter *sc, unsigned int viid, 4885 struct fw_vi_stats_vf *stats) 4886 { 4887 4888 #define GET_STAT(name) \ 4889 read_vf_stat(sc, viid, A_MPS_VF_STAT_##name##_L) 4890 4891 stats->tx_bcast_bytes = GET_STAT(TX_VF_BCAST_BYTES); 4892 stats->tx_bcast_frames = GET_STAT(TX_VF_BCAST_FRAMES); 4893 stats->tx_mcast_bytes = GET_STAT(TX_VF_MCAST_BYTES); 4894 stats->tx_mcast_frames = GET_STAT(TX_VF_MCAST_FRAMES); 4895 stats->tx_ucast_bytes = GET_STAT(TX_VF_UCAST_BYTES); 4896 stats->tx_ucast_frames = GET_STAT(TX_VF_UCAST_FRAMES); 4897 stats->tx_drop_frames = GET_STAT(TX_VF_DROP_FRAMES); 4898 stats->tx_offload_bytes = GET_STAT(TX_VF_OFFLOAD_BYTES); 4899 stats->tx_offload_frames = GET_STAT(TX_VF_OFFLOAD_FRAMES); 4900 stats->rx_bcast_bytes = GET_STAT(RX_VF_BCAST_BYTES); 4901 stats->rx_bcast_frames = GET_STAT(RX_VF_BCAST_FRAMES); 4902 stats->rx_mcast_bytes = GET_STAT(RX_VF_MCAST_BYTES); 4903 stats->rx_mcast_frames = GET_STAT(RX_VF_MCAST_FRAMES); 4904 stats->rx_ucast_bytes = GET_STAT(RX_VF_UCAST_BYTES); 4905 stats->rx_ucast_frames = GET_STAT(RX_VF_UCAST_FRAMES); 4906 stats->rx_err_frames = GET_STAT(RX_VF_ERR_FRAMES); 4907 4908 #undef GET_STAT 4909 } 4910 4911 static void 4912 t4_clr_vi_stats(struct adapter *sc, unsigned int viid) 4913 { 4914 int reg; 4915 4916 t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) | 4917 V_PL_VFID(G_FW_VIID_VIN(viid)) | 4918 V_PL_ADDR(VF_MPS_REG(A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L))); 4919 for (reg = A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L; 4920 reg <= A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H; reg += 4) 4921 t4_write_reg(sc, A_PL_INDIR_DATA, 0); 4922 } 4923 4924 static void 4925 vi_refresh_stats(struct adapter *sc, struct vi_info *vi) 4926 { 4927 struct timeval tv; 4928 const struct timeval interval = {0, 250000}; /* 250ms */ 4929 4930 if (!(vi->flags & VI_INIT_DONE)) 4931 return; 4932 4933 getmicrotime(&tv); 4934 timevalsub(&tv, &interval); 4935 if (timevalcmp(&tv, &vi->last_refreshed, <)) 4936 return; 4937 4938 mtx_lock(&sc->regwin_lock); 4939 t4_get_vi_stats(sc, vi->viid, &vi->stats); 4940 getmicrotime(&vi->last_refreshed); 4941 mtx_unlock(&sc->regwin_lock); 4942 } 4943 4944 static void 4945 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi) 4946 { 4947 int i; 4948 u_int v, tnl_cong_drops; 4949 struct timeval tv; 4950 const struct timeval interval = {0, 250000}; /* 250ms */ 4951 4952 getmicrotime(&tv); 4953 timevalsub(&tv, &interval); 4954 if (timevalcmp(&tv, &pi->last_refreshed, <)) 4955 return; 4956 4957 tnl_cong_drops = 0; 4958 t4_get_port_stats(sc, pi->tx_chan, &pi->stats); 4959 for (i = 0; i < NCHAN; i++) { 4960 if (pi->rx_chan_map & (1 << i)) { 4961 mtx_lock(&sc->regwin_lock); 4962 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 4963 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 4964 mtx_unlock(&sc->regwin_lock); 4965 tnl_cong_drops += v; 4966 } 4967 } 4968 pi->tnl_cong_drops = tnl_cong_drops; 4969 getmicrotime(&pi->last_refreshed); 4970 } 4971 4972 static void 4973 cxgbe_tick(void *arg) 4974 { 4975 struct port_info *pi = arg; 4976 struct adapter *sc = pi->adapter; 4977 4978 PORT_LOCK_ASSERT_OWNED(pi); 4979 cxgbe_refresh_stats(sc, pi); 4980 4981 callout_schedule(&pi->tick, hz); 4982 } 4983 4984 void 4985 vi_tick(void *arg) 4986 { 4987 struct vi_info *vi = arg; 4988 struct adapter *sc = vi->pi->adapter; 4989 4990 vi_refresh_stats(sc, vi); 4991 4992 callout_schedule(&vi->tick, hz); 4993 } 4994 4995 static void 4996 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 4997 { 4998 struct ifnet *vlan; 4999 5000 if (arg != ifp || ifp->if_type != IFT_ETHER) 5001 return; 5002 5003 vlan = VLAN_DEVAT(ifp, vid); 5004 VLAN_SETCOOKIE(vlan, ifp); 5005 } 5006 5007 static int 5008 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 5009 { 5010 5011 #ifdef INVARIANTS 5012 panic("%s: opcode 0x%02x on iq %p with payload %p", 5013 __func__, rss->opcode, iq, m); 5014 #else 5015 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 5016 __func__, rss->opcode, iq, m); 5017 m_freem(m); 5018 #endif 5019 return (EDOOFUS); 5020 } 5021 5022 int 5023 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 5024 { 5025 uintptr_t *loc, new; 5026 5027 if (opcode >= nitems(sc->cpl_handler)) 5028 return (EINVAL); 5029 5030 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 5031 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 5032 atomic_store_rel_ptr(loc, new); 5033 5034 return (0); 5035 } 5036 5037 static int 5038 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 5039 { 5040 5041 #ifdef INVARIANTS 5042 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 5043 #else 5044 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 5045 __func__, iq, ctrl); 5046 #endif 5047 return (EDOOFUS); 5048 } 5049 5050 int 5051 t4_register_an_handler(struct adapter *sc, an_handler_t h) 5052 { 5053 uintptr_t *loc, new; 5054 5055 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 5056 loc = (uintptr_t *) &sc->an_handler; 5057 atomic_store_rel_ptr(loc, new); 5058 5059 return (0); 5060 } 5061 5062 static int 5063 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 5064 { 5065 const struct cpl_fw6_msg *cpl = 5066 __containerof(rpl, struct cpl_fw6_msg, data[0]); 5067 5068 #ifdef INVARIANTS 5069 panic("%s: fw_msg type %d", __func__, cpl->type); 5070 #else 5071 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 5072 #endif 5073 return (EDOOFUS); 5074 } 5075 5076 int 5077 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 5078 { 5079 uintptr_t *loc, new; 5080 5081 if (type >= nitems(sc->fw_msg_handler)) 5082 return (EINVAL); 5083 5084 /* 5085 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 5086 * handler dispatch table. Reject any attempt to install a handler for 5087 * this subtype. 5088 */ 5089 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 5090 return (EINVAL); 5091 5092 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 5093 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 5094 atomic_store_rel_ptr(loc, new); 5095 5096 return (0); 5097 } 5098 5099 static void 5100 t4_sysctls(struct adapter *sc) 5101 { 5102 struct sysctl_ctx_list *ctx; 5103 struct sysctl_oid *oid; 5104 struct sysctl_oid_list *children, *c0; 5105 static char *caps[] = { 5106 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 5107 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL" /* caps[1] niccaps */ 5108 "\6HASHFILTER\7ETHOFLD", 5109 "\20\1TOE", /* caps[2] toecaps */ 5110 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 5111 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 5112 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 5113 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 5114 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 5115 "\4PO_INITIAOR\5PO_TARGET" 5116 }; 5117 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 5118 5119 ctx = device_get_sysctl_ctx(sc->dev); 5120 5121 /* 5122 * dev.t4nex.X. 5123 */ 5124 oid = device_get_sysctl_tree(sc->dev); 5125 c0 = children = SYSCTL_CHILDREN(oid); 5126 5127 sc->sc_do_rxcopy = 1; 5128 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, 5129 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); 5130 5131 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 5132 sc->params.nports, "# of ports"); 5133 5134 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 5135 NULL, chip_rev(sc), "chip hardware revision"); 5136 5137 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 5138 CTLFLAG_RD, sc->fw_version, 0, "firmware version"); 5139 5140 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 5141 CTLFLAG_RD, sc->cfg_file, 0, "configuration file"); 5142 5143 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 5144 sc->cfcsum, "config file checksum"); 5145 5146 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 5147 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 5148 sysctl_bitfield, "A", "available doorbells"); 5149 5150 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 5151 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 5152 sysctl_bitfield, "A", "available link capabilities"); 5153 5154 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 5155 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 5156 sysctl_bitfield, "A", "available NIC capabilities"); 5157 5158 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 5159 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 5160 sysctl_bitfield, "A", "available TCP offload capabilities"); 5161 5162 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 5163 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 5164 sysctl_bitfield, "A", "available RDMA capabilities"); 5165 5166 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 5167 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 5168 sysctl_bitfield, "A", "available iSCSI capabilities"); 5169 5170 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 5171 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 5172 sysctl_bitfield, "A", "available FCoE capabilities"); 5173 5174 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 5175 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 5176 5177 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 5178 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 5179 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 5180 "interrupt holdoff timer values (us)"); 5181 5182 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 5183 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 5184 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 5185 "interrupt holdoff packet counter values"); 5186 5187 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, 5188 NULL, sc->tids.nftids, "number of filters"); 5189 5190 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | 5191 CTLFLAG_RD, sc, 0, sysctl_temperature, "I", 5192 "chip temperature (in Celsius)"); 5193 5194 t4_sge_sysctls(sc, ctx, children); 5195 5196 sc->lro_timeout = 100; 5197 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, 5198 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); 5199 5200 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "debug_flags", CTLFLAG_RW, 5201 &sc->debug_flags, 0, "flags to enable runtime debugging"); 5202 5203 #ifdef SBUF_DRAIN 5204 /* 5205 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 5206 */ 5207 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 5208 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 5209 "logs and miscellaneous information"); 5210 children = SYSCTL_CHILDREN(oid); 5211 5212 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 5213 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5214 sysctl_cctrl, "A", "congestion control"); 5215 5216 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 5217 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5218 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 5219 5220 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 5221 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 5222 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 5223 5224 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 5225 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 5226 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 5227 5228 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 5229 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 5230 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 5231 5232 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 5233 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 5234 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 5235 5236 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 5237 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 5238 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 5239 5240 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 5241 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5242 sysctl_cim_la, "A", "CIM logic analyzer"); 5243 5244 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", 5245 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5246 sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); 5247 5248 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 5249 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 5250 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 5251 5252 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 5253 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 5254 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 5255 5256 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 5257 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 5258 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 5259 5260 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 5261 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 5262 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 5263 5264 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 5265 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 5266 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 5267 5268 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 5269 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 5270 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 5271 5272 if (is_t5(sc)) { 5273 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 5274 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 5275 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 5276 5277 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 5278 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 5279 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 5280 } 5281 5282 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", 5283 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5284 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); 5285 5286 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 5287 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5288 sysctl_cim_qcfg, "A", "CIM queue configuration"); 5289 5290 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 5291 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5292 sysctl_cpl_stats, "A", "CPL statistics"); 5293 5294 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 5295 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5296 sysctl_ddp_stats, "A", "non-TCP DDP statistics"); 5297 5298 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 5299 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5300 sysctl_devlog, "A", "firmware's device log"); 5301 5302 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 5303 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5304 sysctl_fcoe_stats, "A", "FCoE statistics"); 5305 5306 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 5307 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5308 sysctl_hw_sched, "A", "hardware scheduler "); 5309 5310 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 5311 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5312 sysctl_l2t, "A", "hardware L2 table"); 5313 5314 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 5315 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5316 sysctl_lb_stats, "A", "loopback statistics"); 5317 5318 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 5319 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5320 sysctl_meminfo, "A", "memory regions"); 5321 5322 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", 5323 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5324 sysctl_mps_tcam, "A", "MPS TCAM entries"); 5325 5326 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 5327 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5328 sysctl_path_mtus, "A", "path MTUs"); 5329 5330 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 5331 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5332 sysctl_pm_stats, "A", "PM statistics"); 5333 5334 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 5335 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5336 sysctl_rdma_stats, "A", "RDMA statistics"); 5337 5338 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 5339 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5340 sysctl_tcp_stats, "A", "TCP statistics"); 5341 5342 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 5343 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5344 sysctl_tids, "A", "TID information"); 5345 5346 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 5347 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5348 sysctl_tp_err_stats, "A", "TP error statistics"); 5349 5350 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", 5351 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5352 sysctl_tp_la, "A", "TP logic analyzer"); 5353 5354 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 5355 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5356 sysctl_tx_rate, "A", "Tx rate"); 5357 5358 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", 5359 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5360 sysctl_ulprx_la, "A", "ULPRX logic analyzer"); 5361 5362 if (is_t5(sc)) { 5363 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 5364 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5365 sysctl_wcwr_stats, "A", "write combined work requests"); 5366 } 5367 #endif 5368 5369 #ifdef TCP_OFFLOAD 5370 if (is_offload(sc)) { 5371 /* 5372 * dev.t4nex.X.toe. 5373 */ 5374 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 5375 NULL, "TOE parameters"); 5376 children = SYSCTL_CHILDREN(oid); 5377 5378 sc->tt.sndbuf = 256 * 1024; 5379 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 5380 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 5381 5382 sc->tt.ddp = 0; 5383 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 5384 &sc->tt.ddp, 0, "DDP allowed"); 5385 5386 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 5387 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 5388 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 5389 5390 sc->tt.ddp_thres = 5391 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 5392 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 5393 &sc->tt.ddp_thres, 0, "DDP threshold"); 5394 5395 sc->tt.rx_coalesce = 1; 5396 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", 5397 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); 5398 5399 sc->tt.tx_align = 1; 5400 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align", 5401 CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload"); 5402 } 5403 #endif 5404 } 5405 5406 void 5407 vi_sysctls(struct vi_info *vi) 5408 { 5409 struct sysctl_ctx_list *ctx; 5410 struct sysctl_oid *oid; 5411 struct sysctl_oid_list *children; 5412 5413 ctx = device_get_sysctl_ctx(vi->dev); 5414 5415 /* 5416 * dev.[nv](cxgbe|cxl).X. 5417 */ 5418 oid = device_get_sysctl_tree(vi->dev); 5419 children = SYSCTL_CHILDREN(oid); 5420 5421 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "viid", CTLFLAG_RD, NULL, 5422 vi->viid, "VI identifer"); 5423 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 5424 &vi->nrxq, 0, "# of rx queues"); 5425 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 5426 &vi->ntxq, 0, "# of tx queues"); 5427 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 5428 &vi->first_rxq, 0, "index of first rx queue"); 5429 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 5430 &vi->first_txq, 0, "index of first tx queue"); 5431 5432 if (vi->flags & VI_NETMAP) 5433 return; 5434 5435 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT | 5436 CTLFLAG_RW, vi, 0, sysctl_noflowq, "IU", 5437 "Reserve queue 0 for non-flowid packets"); 5438 5439 #ifdef TCP_OFFLOAD 5440 if (vi->nofldrxq != 0) { 5441 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 5442 &vi->nofldrxq, 0, 5443 "# of rx queues for offloaded TCP connections"); 5444 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 5445 &vi->nofldtxq, 0, 5446 "# of tx queues for offloaded TCP connections"); 5447 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 5448 CTLFLAG_RD, &vi->first_ofld_rxq, 0, 5449 "index of first TOE rx queue"); 5450 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 5451 CTLFLAG_RD, &vi->first_ofld_txq, 0, 5452 "index of first TOE tx queue"); 5453 } 5454 #endif 5455 5456 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 5457 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_holdoff_tmr_idx, "I", 5458 "holdoff timer index"); 5459 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 5460 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_holdoff_pktc_idx, "I", 5461 "holdoff packet counter index"); 5462 5463 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 5464 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_qsize_rxq, "I", 5465 "rx queue size"); 5466 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 5467 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_qsize_txq, "I", 5468 "tx queue size"); 5469 } 5470 5471 static void 5472 cxgbe_sysctls(struct port_info *pi) 5473 { 5474 struct sysctl_ctx_list *ctx; 5475 struct sysctl_oid *oid; 5476 struct sysctl_oid_list *children; 5477 struct adapter *sc = pi->adapter; 5478 5479 ctx = device_get_sysctl_ctx(pi->dev); 5480 5481 /* 5482 * dev.cxgbe.X. 5483 */ 5484 oid = device_get_sysctl_tree(pi->dev); 5485 children = SYSCTL_CHILDREN(oid); 5486 5487 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | 5488 CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); 5489 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { 5490 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 5491 CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I", 5492 "PHY temperature (in Celsius)"); 5493 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", 5494 CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I", 5495 "PHY firmware version"); 5496 } 5497 5498 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings", 5499 CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings, 5500 "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)"); 5501 5502 /* 5503 * dev.cxgbe.X.stats. 5504 */ 5505 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 5506 NULL, "port statistics"); 5507 children = SYSCTL_CHILDREN(oid); 5508 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD, 5509 &pi->tx_parse_error, 0, 5510 "# of tx packets with invalid length or # of segments"); 5511 5512 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 5513 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 5514 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ 5515 sysctl_handle_t4_reg64, "QU", desc) 5516 5517 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 5518 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 5519 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 5520 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 5521 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 5522 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 5523 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 5524 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 5525 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 5526 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 5527 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 5528 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 5529 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 5530 "# of tx frames in this range", 5531 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 5532 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 5533 "# of tx frames in this range", 5534 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 5535 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 5536 "# of tx frames in this range", 5537 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 5538 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 5539 "# of tx frames in this range", 5540 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 5541 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 5542 "# of tx frames in this range", 5543 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 5544 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 5545 "# of tx frames in this range", 5546 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 5547 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 5548 "# of tx frames in this range", 5549 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 5550 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 5551 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 5552 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 5553 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 5554 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 5555 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 5556 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 5557 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 5558 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 5559 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 5560 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 5561 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 5562 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 5563 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 5564 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 5565 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 5566 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 5567 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 5568 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 5569 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 5570 5571 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 5572 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 5573 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 5574 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 5575 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 5576 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 5577 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 5578 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 5579 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 5580 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 5581 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 5582 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 5583 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 5584 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 5585 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 5586 "# of frames received with bad FCS", 5587 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 5588 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 5589 "# of frames received with length error", 5590 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 5591 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 5592 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 5593 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 5594 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 5595 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 5596 "# of rx frames in this range", 5597 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 5598 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 5599 "# of rx frames in this range", 5600 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 5601 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 5602 "# of rx frames in this range", 5603 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 5604 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 5605 "# of rx frames in this range", 5606 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 5607 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 5608 "# of rx frames in this range", 5609 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 5610 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 5611 "# of rx frames in this range", 5612 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 5613 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 5614 "# of rx frames in this range", 5615 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 5616 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 5617 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 5618 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 5619 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 5620 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 5621 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 5622 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 5623 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 5624 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 5625 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 5626 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 5627 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 5628 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 5629 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 5630 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 5631 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 5632 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 5633 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 5634 5635 #undef SYSCTL_ADD_T4_REG64 5636 5637 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 5638 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 5639 &pi->stats.name, desc) 5640 5641 /* We get these from port_stats and they may be stale by upto 1s */ 5642 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 5643 "# drops due to buffer-group 0 overflows"); 5644 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 5645 "# drops due to buffer-group 1 overflows"); 5646 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 5647 "# drops due to buffer-group 2 overflows"); 5648 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 5649 "# drops due to buffer-group 3 overflows"); 5650 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 5651 "# of buffer-group 0 truncated packets"); 5652 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 5653 "# of buffer-group 1 truncated packets"); 5654 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 5655 "# of buffer-group 2 truncated packets"); 5656 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 5657 "# of buffer-group 3 truncated packets"); 5658 5659 #undef SYSCTL_ADD_T4_PORTSTAT 5660 } 5661 5662 static int 5663 sysctl_int_array(SYSCTL_HANDLER_ARGS) 5664 { 5665 int rc, *i, space = 0; 5666 struct sbuf sb; 5667 5668 sbuf_new_for_sysctl(&sb, NULL, 64, req); 5669 for (i = arg1; arg2; arg2 -= sizeof(int), i++) { 5670 if (space) 5671 sbuf_printf(&sb, " "); 5672 sbuf_printf(&sb, "%d", *i); 5673 space = 1; 5674 } 5675 rc = sbuf_finish(&sb); 5676 sbuf_delete(&sb); 5677 return (rc); 5678 } 5679 5680 static int 5681 sysctl_bitfield(SYSCTL_HANDLER_ARGS) 5682 { 5683 int rc; 5684 struct sbuf *sb; 5685 5686 rc = sysctl_wire_old_buffer(req, 0); 5687 if (rc != 0) 5688 return(rc); 5689 5690 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5691 if (sb == NULL) 5692 return (ENOMEM); 5693 5694 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 5695 rc = sbuf_finish(sb); 5696 sbuf_delete(sb); 5697 5698 return (rc); 5699 } 5700 5701 static int 5702 sysctl_btphy(SYSCTL_HANDLER_ARGS) 5703 { 5704 struct port_info *pi = arg1; 5705 int op = arg2; 5706 struct adapter *sc = pi->adapter; 5707 u_int v; 5708 int rc; 5709 5710 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, "t4btt"); 5711 if (rc) 5712 return (rc); 5713 /* XXX: magic numbers */ 5714 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, 5715 &v); 5716 end_synchronized_op(sc, 0); 5717 if (rc) 5718 return (rc); 5719 if (op == 0) 5720 v /= 256; 5721 5722 rc = sysctl_handle_int(oidp, &v, 0, req); 5723 return (rc); 5724 } 5725 5726 static int 5727 sysctl_noflowq(SYSCTL_HANDLER_ARGS) 5728 { 5729 struct vi_info *vi = arg1; 5730 int rc, val; 5731 5732 val = vi->rsrv_noflowq; 5733 rc = sysctl_handle_int(oidp, &val, 0, req); 5734 if (rc != 0 || req->newptr == NULL) 5735 return (rc); 5736 5737 if ((val >= 1) && (vi->ntxq > 1)) 5738 vi->rsrv_noflowq = 1; 5739 else 5740 vi->rsrv_noflowq = 0; 5741 5742 return (rc); 5743 } 5744 5745 static int 5746 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 5747 { 5748 struct vi_info *vi = arg1; 5749 struct adapter *sc = vi->pi->adapter; 5750 int idx, rc, i; 5751 struct sge_rxq *rxq; 5752 #ifdef TCP_OFFLOAD 5753 struct sge_ofld_rxq *ofld_rxq; 5754 #endif 5755 uint8_t v; 5756 5757 idx = vi->tmr_idx; 5758 5759 rc = sysctl_handle_int(oidp, &idx, 0, req); 5760 if (rc != 0 || req->newptr == NULL) 5761 return (rc); 5762 5763 if (idx < 0 || idx >= SGE_NTIMERS) 5764 return (EINVAL); 5765 5766 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5767 "t4tmr"); 5768 if (rc) 5769 return (rc); 5770 5771 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(vi->pktc_idx != -1); 5772 for_each_rxq(vi, i, rxq) { 5773 #ifdef atomic_store_rel_8 5774 atomic_store_rel_8(&rxq->iq.intr_params, v); 5775 #else 5776 rxq->iq.intr_params = v; 5777 #endif 5778 } 5779 #ifdef TCP_OFFLOAD 5780 for_each_ofld_rxq(vi, i, ofld_rxq) { 5781 #ifdef atomic_store_rel_8 5782 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); 5783 #else 5784 ofld_rxq->iq.intr_params = v; 5785 #endif 5786 } 5787 #endif 5788 vi->tmr_idx = idx; 5789 5790 end_synchronized_op(sc, LOCK_HELD); 5791 return (0); 5792 } 5793 5794 static int 5795 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 5796 { 5797 struct vi_info *vi = arg1; 5798 struct adapter *sc = vi->pi->adapter; 5799 int idx, rc; 5800 5801 idx = vi->pktc_idx; 5802 5803 rc = sysctl_handle_int(oidp, &idx, 0, req); 5804 if (rc != 0 || req->newptr == NULL) 5805 return (rc); 5806 5807 if (idx < -1 || idx >= SGE_NCOUNTERS) 5808 return (EINVAL); 5809 5810 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5811 "t4pktc"); 5812 if (rc) 5813 return (rc); 5814 5815 if (vi->flags & VI_INIT_DONE) 5816 rc = EBUSY; /* cannot be changed once the queues are created */ 5817 else 5818 vi->pktc_idx = idx; 5819 5820 end_synchronized_op(sc, LOCK_HELD); 5821 return (rc); 5822 } 5823 5824 static int 5825 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 5826 { 5827 struct vi_info *vi = arg1; 5828 struct adapter *sc = vi->pi->adapter; 5829 int qsize, rc; 5830 5831 qsize = vi->qsize_rxq; 5832 5833 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5834 if (rc != 0 || req->newptr == NULL) 5835 return (rc); 5836 5837 if (qsize < 128 || (qsize & 7)) 5838 return (EINVAL); 5839 5840 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5841 "t4rxqs"); 5842 if (rc) 5843 return (rc); 5844 5845 if (vi->flags & VI_INIT_DONE) 5846 rc = EBUSY; /* cannot be changed once the queues are created */ 5847 else 5848 vi->qsize_rxq = qsize; 5849 5850 end_synchronized_op(sc, LOCK_HELD); 5851 return (rc); 5852 } 5853 5854 static int 5855 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 5856 { 5857 struct vi_info *vi = arg1; 5858 struct adapter *sc = vi->pi->adapter; 5859 int qsize, rc; 5860 5861 qsize = vi->qsize_txq; 5862 5863 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5864 if (rc != 0 || req->newptr == NULL) 5865 return (rc); 5866 5867 if (qsize < 128 || qsize > 65536) 5868 return (EINVAL); 5869 5870 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5871 "t4txqs"); 5872 if (rc) 5873 return (rc); 5874 5875 if (vi->flags & VI_INIT_DONE) 5876 rc = EBUSY; /* cannot be changed once the queues are created */ 5877 else 5878 vi->qsize_txq = qsize; 5879 5880 end_synchronized_op(sc, LOCK_HELD); 5881 return (rc); 5882 } 5883 5884 static int 5885 sysctl_pause_settings(SYSCTL_HANDLER_ARGS) 5886 { 5887 struct port_info *pi = arg1; 5888 struct adapter *sc = pi->adapter; 5889 struct link_config *lc = &pi->link_cfg; 5890 int rc; 5891 5892 if (req->newptr == NULL) { 5893 struct sbuf *sb; 5894 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX"; 5895 5896 rc = sysctl_wire_old_buffer(req, 0); 5897 if (rc != 0) 5898 return(rc); 5899 5900 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5901 if (sb == NULL) 5902 return (ENOMEM); 5903 5904 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits); 5905 rc = sbuf_finish(sb); 5906 sbuf_delete(sb); 5907 } else { 5908 char s[2]; 5909 int n; 5910 5911 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX)); 5912 s[1] = 0; 5913 5914 rc = sysctl_handle_string(oidp, s, sizeof(s), req); 5915 if (rc != 0) 5916 return(rc); 5917 5918 if (s[1] != 0) 5919 return (EINVAL); 5920 if (s[0] < '0' || s[0] > '9') 5921 return (EINVAL); /* not a number */ 5922 n = s[0] - '0'; 5923 if (n & ~(PAUSE_TX | PAUSE_RX)) 5924 return (EINVAL); /* some other bit is set too */ 5925 5926 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, 5927 "t4PAUSE"); 5928 if (rc) 5929 return (rc); 5930 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) { 5931 int link_ok = lc->link_ok; 5932 5933 lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX); 5934 lc->requested_fc |= n; 5935 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc); 5936 lc->link_ok = link_ok; /* restore */ 5937 } 5938 end_synchronized_op(sc, 0); 5939 } 5940 5941 return (rc); 5942 } 5943 5944 static int 5945 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 5946 { 5947 struct adapter *sc = arg1; 5948 int reg = arg2; 5949 uint64_t val; 5950 5951 val = t4_read_reg64(sc, reg); 5952 5953 return (sysctl_handle_64(oidp, &val, 0, req)); 5954 } 5955 5956 static int 5957 sysctl_temperature(SYSCTL_HANDLER_ARGS) 5958 { 5959 struct adapter *sc = arg1; 5960 int rc, t; 5961 uint32_t param, val; 5962 5963 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); 5964 if (rc) 5965 return (rc); 5966 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 5967 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 5968 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); 5969 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 5970 end_synchronized_op(sc, 0); 5971 if (rc) 5972 return (rc); 5973 5974 /* unknown is returned as 0 but we display -1 in that case */ 5975 t = val == 0 ? -1 : val; 5976 5977 rc = sysctl_handle_int(oidp, &t, 0, req); 5978 return (rc); 5979 } 5980 5981 #ifdef SBUF_DRAIN 5982 static int 5983 sysctl_cctrl(SYSCTL_HANDLER_ARGS) 5984 { 5985 struct adapter *sc = arg1; 5986 struct sbuf *sb; 5987 int rc, i; 5988 uint16_t incr[NMTUS][NCCTRL_WIN]; 5989 static const char *dec_fac[] = { 5990 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 5991 "0.9375" 5992 }; 5993 5994 rc = sysctl_wire_old_buffer(req, 0); 5995 if (rc != 0) 5996 return (rc); 5997 5998 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5999 if (sb == NULL) 6000 return (ENOMEM); 6001 6002 t4_read_cong_tbl(sc, incr); 6003 6004 for (i = 0; i < NCCTRL_WIN; ++i) { 6005 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 6006 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 6007 incr[5][i], incr[6][i], incr[7][i]); 6008 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 6009 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 6010 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 6011 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 6012 } 6013 6014 rc = sbuf_finish(sb); 6015 sbuf_delete(sb); 6016 6017 return (rc); 6018 } 6019 6020 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 6021 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 6022 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 6023 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 6024 }; 6025 6026 static int 6027 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 6028 { 6029 struct adapter *sc = arg1; 6030 struct sbuf *sb; 6031 int rc, i, n, qid = arg2; 6032 uint32_t *buf, *p; 6033 char *qtype; 6034 u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5; 6035 6036 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 6037 ("%s: bad qid %d\n", __func__, qid)); 6038 6039 if (qid < CIM_NUM_IBQ) { 6040 /* inbound queue */ 6041 qtype = "IBQ"; 6042 n = 4 * CIM_IBQ_SIZE; 6043 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 6044 rc = t4_read_cim_ibq(sc, qid, buf, n); 6045 } else { 6046 /* outbound queue */ 6047 qtype = "OBQ"; 6048 qid -= CIM_NUM_IBQ; 6049 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 6050 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 6051 rc = t4_read_cim_obq(sc, qid, buf, n); 6052 } 6053 6054 if (rc < 0) { 6055 rc = -rc; 6056 goto done; 6057 } 6058 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 6059 6060 rc = sysctl_wire_old_buffer(req, 0); 6061 if (rc != 0) 6062 goto done; 6063 6064 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 6065 if (sb == NULL) { 6066 rc = ENOMEM; 6067 goto done; 6068 } 6069 6070 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 6071 for (i = 0, p = buf; i < n; i += 16, p += 4) 6072 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 6073 p[2], p[3]); 6074 6075 rc = sbuf_finish(sb); 6076 sbuf_delete(sb); 6077 done: 6078 free(buf, M_CXGBE); 6079 return (rc); 6080 } 6081 6082 static int 6083 sysctl_cim_la(SYSCTL_HANDLER_ARGS) 6084 { 6085 struct adapter *sc = arg1; 6086 u_int cfg; 6087 struct sbuf *sb; 6088 uint32_t *buf, *p; 6089 int rc; 6090 6091 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 6092 if (rc != 0) 6093 return (rc); 6094 6095 rc = sysctl_wire_old_buffer(req, 0); 6096 if (rc != 0) 6097 return (rc); 6098 6099 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6100 if (sb == NULL) 6101 return (ENOMEM); 6102 6103 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 6104 M_ZERO | M_WAITOK); 6105 6106 rc = -t4_cim_read_la(sc, buf, NULL); 6107 if (rc != 0) 6108 goto done; 6109 6110 sbuf_printf(sb, "Status Data PC%s", 6111 cfg & F_UPDBGLACAPTPCONLY ? "" : 6112 " LS0Stat LS0Addr LS0Data"); 6113 6114 KASSERT((sc->params.cim_la_size & 7) == 0, 6115 ("%s: p will walk off the end of buf", __func__)); 6116 6117 for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) { 6118 if (cfg & F_UPDBGLACAPTPCONLY) { 6119 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 6120 p[6], p[7]); 6121 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 6122 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 6123 p[4] & 0xff, p[5] >> 8); 6124 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 6125 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 6126 p[1] & 0xf, p[2] >> 4); 6127 } else { 6128 sbuf_printf(sb, 6129 "\n %02x %x%07x %x%07x %08x %08x " 6130 "%08x%08x%08x%08x", 6131 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 6132 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 6133 p[6], p[7]); 6134 } 6135 } 6136 6137 rc = sbuf_finish(sb); 6138 sbuf_delete(sb); 6139 done: 6140 free(buf, M_CXGBE); 6141 return (rc); 6142 } 6143 6144 static int 6145 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) 6146 { 6147 struct adapter *sc = arg1; 6148 u_int i; 6149 struct sbuf *sb; 6150 uint32_t *buf, *p; 6151 int rc; 6152 6153 rc = sysctl_wire_old_buffer(req, 0); 6154 if (rc != 0) 6155 return (rc); 6156 6157 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6158 if (sb == NULL) 6159 return (ENOMEM); 6160 6161 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, 6162 M_ZERO | M_WAITOK); 6163 6164 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); 6165 p = buf; 6166 6167 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 6168 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], 6169 p[1], p[0]); 6170 } 6171 6172 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); 6173 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 6174 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", 6175 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, 6176 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, 6177 (p[1] >> 2) | ((p[2] & 3) << 30), 6178 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, 6179 p[0] & 1); 6180 } 6181 6182 rc = sbuf_finish(sb); 6183 sbuf_delete(sb); 6184 free(buf, M_CXGBE); 6185 return (rc); 6186 } 6187 6188 static int 6189 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) 6190 { 6191 struct adapter *sc = arg1; 6192 u_int i; 6193 struct sbuf *sb; 6194 uint32_t *buf, *p; 6195 int rc; 6196 6197 rc = sysctl_wire_old_buffer(req, 0); 6198 if (rc != 0) 6199 return (rc); 6200 6201 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6202 if (sb == NULL) 6203 return (ENOMEM); 6204 6205 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, 6206 M_ZERO | M_WAITOK); 6207 6208 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); 6209 p = buf; 6210 6211 sbuf_printf(sb, "Cntl ID DataBE Addr Data"); 6212 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 6213 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", 6214 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, 6215 p[4], p[3], p[2], p[1], p[0]); 6216 } 6217 6218 sbuf_printf(sb, "\n\nCntl ID Data"); 6219 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 6220 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", 6221 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); 6222 } 6223 6224 rc = sbuf_finish(sb); 6225 sbuf_delete(sb); 6226 free(buf, M_CXGBE); 6227 return (rc); 6228 } 6229 6230 static int 6231 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 6232 { 6233 struct adapter *sc = arg1; 6234 struct sbuf *sb; 6235 int rc, i; 6236 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 6237 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 6238 uint16_t thres[CIM_NUM_IBQ]; 6239 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 6240 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 6241 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 6242 6243 if (is_t4(sc)) { 6244 cim_num_obq = CIM_NUM_OBQ; 6245 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 6246 obq_rdaddr = A_UP_OBQ_0_REALADDR; 6247 } else { 6248 cim_num_obq = CIM_NUM_OBQ_T5; 6249 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 6250 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 6251 } 6252 nq = CIM_NUM_IBQ + cim_num_obq; 6253 6254 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 6255 if (rc == 0) 6256 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 6257 if (rc != 0) 6258 return (rc); 6259 6260 t4_read_cimq_cfg(sc, base, size, thres); 6261 6262 rc = sysctl_wire_old_buffer(req, 0); 6263 if (rc != 0) 6264 return (rc); 6265 6266 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 6267 if (sb == NULL) 6268 return (ENOMEM); 6269 6270 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 6271 6272 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 6273 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 6274 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 6275 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 6276 G_QUEREMFLITS(p[2]) * 16); 6277 for ( ; i < nq; i++, p += 4, wr += 2) 6278 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 6279 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 6280 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 6281 G_QUEREMFLITS(p[2]) * 16); 6282 6283 rc = sbuf_finish(sb); 6284 sbuf_delete(sb); 6285 6286 return (rc); 6287 } 6288 6289 static int 6290 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 6291 { 6292 struct adapter *sc = arg1; 6293 struct sbuf *sb; 6294 int rc; 6295 struct tp_cpl_stats stats; 6296 6297 rc = sysctl_wire_old_buffer(req, 0); 6298 if (rc != 0) 6299 return (rc); 6300 6301 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6302 if (sb == NULL) 6303 return (ENOMEM); 6304 6305 t4_tp_get_cpl_stats(sc, &stats); 6306 6307 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6308 "channel 3\n"); 6309 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 6310 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 6311 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 6312 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 6313 6314 rc = sbuf_finish(sb); 6315 sbuf_delete(sb); 6316 6317 return (rc); 6318 } 6319 6320 static int 6321 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 6322 { 6323 struct adapter *sc = arg1; 6324 struct sbuf *sb; 6325 int rc; 6326 struct tp_usm_stats stats; 6327 6328 rc = sysctl_wire_old_buffer(req, 0); 6329 if (rc != 0) 6330 return(rc); 6331 6332 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6333 if (sb == NULL) 6334 return (ENOMEM); 6335 6336 t4_get_usm_stats(sc, &stats); 6337 6338 sbuf_printf(sb, "Frames: %u\n", stats.frames); 6339 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 6340 sbuf_printf(sb, "Drops: %u", stats.drops); 6341 6342 rc = sbuf_finish(sb); 6343 sbuf_delete(sb); 6344 6345 return (rc); 6346 } 6347 6348 const char *devlog_level_strings[] = { 6349 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 6350 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 6351 [FW_DEVLOG_LEVEL_ERR] = "ERR", 6352 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 6353 [FW_DEVLOG_LEVEL_INFO] = "INFO", 6354 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 6355 }; 6356 6357 const char *devlog_facility_strings[] = { 6358 [FW_DEVLOG_FACILITY_CORE] = "CORE", 6359 [FW_DEVLOG_FACILITY_CF] = "CF", 6360 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 6361 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 6362 [FW_DEVLOG_FACILITY_RES] = "RES", 6363 [FW_DEVLOG_FACILITY_HW] = "HW", 6364 [FW_DEVLOG_FACILITY_FLR] = "FLR", 6365 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 6366 [FW_DEVLOG_FACILITY_PHY] = "PHY", 6367 [FW_DEVLOG_FACILITY_MAC] = "MAC", 6368 [FW_DEVLOG_FACILITY_PORT] = "PORT", 6369 [FW_DEVLOG_FACILITY_VI] = "VI", 6370 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 6371 [FW_DEVLOG_FACILITY_ACL] = "ACL", 6372 [FW_DEVLOG_FACILITY_TM] = "TM", 6373 [FW_DEVLOG_FACILITY_QFC] = "QFC", 6374 [FW_DEVLOG_FACILITY_DCB] = "DCB", 6375 [FW_DEVLOG_FACILITY_ETH] = "ETH", 6376 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 6377 [FW_DEVLOG_FACILITY_RI] = "RI", 6378 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 6379 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 6380 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 6381 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 6382 }; 6383 6384 static int 6385 sysctl_devlog(SYSCTL_HANDLER_ARGS) 6386 { 6387 struct adapter *sc = arg1; 6388 struct devlog_params *dparams = &sc->params.devlog; 6389 struct fw_devlog_e *buf, *e; 6390 int i, j, rc, nentries, first = 0, m; 6391 struct sbuf *sb; 6392 uint64_t ftstamp = UINT64_MAX; 6393 6394 if (dparams->start == 0) { 6395 dparams->memtype = FW_MEMTYPE_EDC0; 6396 dparams->start = 0x84000; 6397 dparams->size = 32768; 6398 } 6399 6400 nentries = dparams->size / sizeof(struct fw_devlog_e); 6401 6402 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 6403 if (buf == NULL) 6404 return (ENOMEM); 6405 6406 m = fwmtype_to_hwmtype(dparams->memtype); 6407 rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf); 6408 if (rc != 0) 6409 goto done; 6410 6411 for (i = 0; i < nentries; i++) { 6412 e = &buf[i]; 6413 6414 if (e->timestamp == 0) 6415 break; /* end */ 6416 6417 e->timestamp = be64toh(e->timestamp); 6418 e->seqno = be32toh(e->seqno); 6419 for (j = 0; j < 8; j++) 6420 e->params[j] = be32toh(e->params[j]); 6421 6422 if (e->timestamp < ftstamp) { 6423 ftstamp = e->timestamp; 6424 first = i; 6425 } 6426 } 6427 6428 if (buf[first].timestamp == 0) 6429 goto done; /* nothing in the log */ 6430 6431 rc = sysctl_wire_old_buffer(req, 0); 6432 if (rc != 0) 6433 goto done; 6434 6435 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6436 if (sb == NULL) { 6437 rc = ENOMEM; 6438 goto done; 6439 } 6440 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 6441 "Seq#", "Tstamp", "Level", "Facility", "Message"); 6442 6443 i = first; 6444 do { 6445 e = &buf[i]; 6446 if (e->timestamp == 0) 6447 break; /* end */ 6448 6449 sbuf_printf(sb, "%10d %15ju %8s %8s ", 6450 e->seqno, e->timestamp, 6451 (e->level < nitems(devlog_level_strings) ? 6452 devlog_level_strings[e->level] : "UNKNOWN"), 6453 (e->facility < nitems(devlog_facility_strings) ? 6454 devlog_facility_strings[e->facility] : "UNKNOWN")); 6455 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 6456 e->params[2], e->params[3], e->params[4], 6457 e->params[5], e->params[6], e->params[7]); 6458 6459 if (++i == nentries) 6460 i = 0; 6461 } while (i != first); 6462 6463 rc = sbuf_finish(sb); 6464 sbuf_delete(sb); 6465 done: 6466 free(buf, M_CXGBE); 6467 return (rc); 6468 } 6469 6470 static int 6471 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 6472 { 6473 struct adapter *sc = arg1; 6474 struct sbuf *sb; 6475 int rc; 6476 struct tp_fcoe_stats stats[4]; 6477 6478 rc = sysctl_wire_old_buffer(req, 0); 6479 if (rc != 0) 6480 return (rc); 6481 6482 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6483 if (sb == NULL) 6484 return (ENOMEM); 6485 6486 t4_get_fcoe_stats(sc, 0, &stats[0]); 6487 t4_get_fcoe_stats(sc, 1, &stats[1]); 6488 t4_get_fcoe_stats(sc, 2, &stats[2]); 6489 t4_get_fcoe_stats(sc, 3, &stats[3]); 6490 6491 sbuf_printf(sb, " channel 0 channel 1 " 6492 "channel 2 channel 3\n"); 6493 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 6494 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 6495 stats[3].octetsDDP); 6496 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 6497 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 6498 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 6499 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 6500 stats[3].framesDrop); 6501 6502 rc = sbuf_finish(sb); 6503 sbuf_delete(sb); 6504 6505 return (rc); 6506 } 6507 6508 static int 6509 sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 6510 { 6511 struct adapter *sc = arg1; 6512 struct sbuf *sb; 6513 int rc, i; 6514 unsigned int map, kbps, ipg, mode; 6515 unsigned int pace_tab[NTX_SCHED]; 6516 6517 rc = sysctl_wire_old_buffer(req, 0); 6518 if (rc != 0) 6519 return (rc); 6520 6521 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6522 if (sb == NULL) 6523 return (ENOMEM); 6524 6525 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 6526 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 6527 t4_read_pace_tbl(sc, pace_tab); 6528 6529 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 6530 "Class IPG (0.1 ns) Flow IPG (us)"); 6531 6532 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 6533 t4_get_tx_sched(sc, i, &kbps, &ipg); 6534 sbuf_printf(sb, "\n %u %-5s %u ", i, 6535 (mode & (1 << i)) ? "flow" : "class", map & 3); 6536 if (kbps) 6537 sbuf_printf(sb, "%9u ", kbps); 6538 else 6539 sbuf_printf(sb, " disabled "); 6540 6541 if (ipg) 6542 sbuf_printf(sb, "%13u ", ipg); 6543 else 6544 sbuf_printf(sb, " disabled "); 6545 6546 if (pace_tab[i]) 6547 sbuf_printf(sb, "%10u", pace_tab[i]); 6548 else 6549 sbuf_printf(sb, " disabled"); 6550 } 6551 6552 rc = sbuf_finish(sb); 6553 sbuf_delete(sb); 6554 6555 return (rc); 6556 } 6557 6558 static int 6559 sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 6560 { 6561 struct adapter *sc = arg1; 6562 struct sbuf *sb; 6563 int rc, i, j; 6564 uint64_t *p0, *p1; 6565 struct lb_port_stats s[2]; 6566 static const char *stat_name[] = { 6567 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 6568 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 6569 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 6570 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 6571 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 6572 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 6573 "BG2FramesTrunc:", "BG3FramesTrunc:" 6574 }; 6575 6576 rc = sysctl_wire_old_buffer(req, 0); 6577 if (rc != 0) 6578 return (rc); 6579 6580 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6581 if (sb == NULL) 6582 return (ENOMEM); 6583 6584 memset(s, 0, sizeof(s)); 6585 6586 for (i = 0; i < 4; i += 2) { 6587 t4_get_lb_stats(sc, i, &s[0]); 6588 t4_get_lb_stats(sc, i + 1, &s[1]); 6589 6590 p0 = &s[0].octets; 6591 p1 = &s[1].octets; 6592 sbuf_printf(sb, "%s Loopback %u" 6593 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 6594 6595 for (j = 0; j < nitems(stat_name); j++) 6596 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 6597 *p0++, *p1++); 6598 } 6599 6600 rc = sbuf_finish(sb); 6601 sbuf_delete(sb); 6602 6603 return (rc); 6604 } 6605 6606 static int 6607 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) 6608 { 6609 int rc = 0; 6610 struct port_info *pi = arg1; 6611 struct sbuf *sb; 6612 static const char *linkdnreasons[] = { 6613 "non-specific", "remote fault", "autoneg failed", "reserved3", 6614 "PHY overheated", "unknown", "rx los", "reserved7" 6615 }; 6616 6617 rc = sysctl_wire_old_buffer(req, 0); 6618 if (rc != 0) 6619 return(rc); 6620 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); 6621 if (sb == NULL) 6622 return (ENOMEM); 6623 6624 if (pi->linkdnrc < 0) 6625 sbuf_printf(sb, "n/a"); 6626 else if (pi->linkdnrc < nitems(linkdnreasons)) 6627 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]); 6628 else 6629 sbuf_printf(sb, "%d", pi->linkdnrc); 6630 6631 rc = sbuf_finish(sb); 6632 sbuf_delete(sb); 6633 6634 return (rc); 6635 } 6636 6637 struct mem_desc { 6638 unsigned int base; 6639 unsigned int limit; 6640 unsigned int idx; 6641 }; 6642 6643 static int 6644 mem_desc_cmp(const void *a, const void *b) 6645 { 6646 return ((const struct mem_desc *)a)->base - 6647 ((const struct mem_desc *)b)->base; 6648 } 6649 6650 static void 6651 mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 6652 unsigned int to) 6653 { 6654 unsigned int size; 6655 6656 size = to - from + 1; 6657 if (size == 0) 6658 return; 6659 6660 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 6661 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 6662 } 6663 6664 static int 6665 sysctl_meminfo(SYSCTL_HANDLER_ARGS) 6666 { 6667 struct adapter *sc = arg1; 6668 struct sbuf *sb; 6669 int rc, i, n; 6670 uint32_t lo, hi, used, alloc; 6671 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 6672 static const char *region[] = { 6673 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 6674 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 6675 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 6676 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 6677 "RQUDP region:", "PBL region:", "TXPBL region:", 6678 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 6679 "On-chip queues:" 6680 }; 6681 struct mem_desc avail[4]; 6682 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 6683 struct mem_desc *md = mem; 6684 6685 rc = sysctl_wire_old_buffer(req, 0); 6686 if (rc != 0) 6687 return (rc); 6688 6689 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6690 if (sb == NULL) 6691 return (ENOMEM); 6692 6693 for (i = 0; i < nitems(mem); i++) { 6694 mem[i].limit = 0; 6695 mem[i].idx = i; 6696 } 6697 6698 /* Find and sort the populated memory ranges */ 6699 i = 0; 6700 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 6701 if (lo & F_EDRAM0_ENABLE) { 6702 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 6703 avail[i].base = G_EDRAM0_BASE(hi) << 20; 6704 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 6705 avail[i].idx = 0; 6706 i++; 6707 } 6708 if (lo & F_EDRAM1_ENABLE) { 6709 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 6710 avail[i].base = G_EDRAM1_BASE(hi) << 20; 6711 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 6712 avail[i].idx = 1; 6713 i++; 6714 } 6715 if (lo & F_EXT_MEM_ENABLE) { 6716 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 6717 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 6718 avail[i].limit = avail[i].base + 6719 (G_EXT_MEM_SIZE(hi) << 20); 6720 avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */ 6721 i++; 6722 } 6723 if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) { 6724 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 6725 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 6726 avail[i].limit = avail[i].base + 6727 (G_EXT_MEM1_SIZE(hi) << 20); 6728 avail[i].idx = 4; 6729 i++; 6730 } 6731 if (!i) /* no memory available */ 6732 return 0; 6733 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 6734 6735 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 6736 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 6737 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 6738 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 6739 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 6740 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 6741 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 6742 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 6743 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 6744 6745 /* the next few have explicit upper bounds */ 6746 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 6747 md->limit = md->base - 1 + 6748 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 6749 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 6750 md++; 6751 6752 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 6753 md->limit = md->base - 1 + 6754 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 6755 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 6756 md++; 6757 6758 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6759 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 6760 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 6761 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 6762 } else { 6763 md->base = 0; 6764 md->idx = nitems(region); /* hide it */ 6765 } 6766 md++; 6767 6768 #define ulp_region(reg) \ 6769 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 6770 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 6771 6772 ulp_region(RX_ISCSI); 6773 ulp_region(RX_TDDP); 6774 ulp_region(TX_TPT); 6775 ulp_region(RX_STAG); 6776 ulp_region(RX_RQ); 6777 ulp_region(RX_RQUDP); 6778 ulp_region(RX_PBL); 6779 ulp_region(TX_PBL); 6780 #undef ulp_region 6781 6782 md->base = 0; 6783 md->idx = nitems(region); 6784 if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) { 6785 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); 6786 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc, 6787 A_SGE_DBVFIFO_SIZE))) << 2) - 1; 6788 } 6789 md++; 6790 6791 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 6792 md->limit = md->base + sc->tids.ntids - 1; 6793 md++; 6794 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 6795 md->limit = md->base + sc->tids.ntids - 1; 6796 md++; 6797 6798 md->base = sc->vres.ocq.start; 6799 if (sc->vres.ocq.size) 6800 md->limit = md->base + sc->vres.ocq.size - 1; 6801 else 6802 md->idx = nitems(region); /* hide it */ 6803 md++; 6804 6805 /* add any address-space holes, there can be up to 3 */ 6806 for (n = 0; n < i - 1; n++) 6807 if (avail[n].limit < avail[n + 1].base) 6808 (md++)->base = avail[n].limit; 6809 if (avail[n].limit) 6810 (md++)->base = avail[n].limit; 6811 6812 n = md - mem; 6813 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 6814 6815 for (lo = 0; lo < i; lo++) 6816 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 6817 avail[lo].limit - 1); 6818 6819 sbuf_printf(sb, "\n"); 6820 for (i = 0; i < n; i++) { 6821 if (mem[i].idx >= nitems(region)) 6822 continue; /* skip holes */ 6823 if (!mem[i].limit) 6824 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 6825 mem_region_show(sb, region[mem[i].idx], mem[i].base, 6826 mem[i].limit); 6827 } 6828 6829 sbuf_printf(sb, "\n"); 6830 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 6831 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 6832 mem_region_show(sb, "uP RAM:", lo, hi); 6833 6834 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 6835 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 6836 mem_region_show(sb, "uP Extmem2:", lo, hi); 6837 6838 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 6839 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 6840 G_PMRXMAXPAGE(lo), 6841 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 6842 (lo & F_PMRXNUMCHN) ? 2 : 1); 6843 6844 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 6845 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 6846 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 6847 G_PMTXMAXPAGE(lo), 6848 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 6849 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 6850 sbuf_printf(sb, "%u p-structs\n", 6851 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 6852 6853 for (i = 0; i < 4; i++) { 6854 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 6855 if (is_t4(sc)) { 6856 used = G_USED(lo); 6857 alloc = G_ALLOC(lo); 6858 } else { 6859 used = G_T5_USED(lo); 6860 alloc = G_T5_ALLOC(lo); 6861 } 6862 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 6863 i, used, alloc); 6864 } 6865 for (i = 0; i < 4; i++) { 6866 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 6867 if (is_t4(sc)) { 6868 used = G_USED(lo); 6869 alloc = G_ALLOC(lo); 6870 } else { 6871 used = G_T5_USED(lo); 6872 alloc = G_T5_ALLOC(lo); 6873 } 6874 sbuf_printf(sb, 6875 "\nLoopback %d using %u pages out of %u allocated", 6876 i, used, alloc); 6877 } 6878 6879 rc = sbuf_finish(sb); 6880 sbuf_delete(sb); 6881 6882 return (rc); 6883 } 6884 6885 static inline void 6886 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) 6887 { 6888 *mask = x | y; 6889 y = htobe64(y); 6890 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); 6891 } 6892 6893 static int 6894 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) 6895 { 6896 struct adapter *sc = arg1; 6897 struct sbuf *sb; 6898 int rc, i, n; 6899 6900 rc = sysctl_wire_old_buffer(req, 0); 6901 if (rc != 0) 6902 return (rc); 6903 6904 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6905 if (sb == NULL) 6906 return (ENOMEM); 6907 6908 sbuf_printf(sb, 6909 "Idx Ethernet address Mask Vld Ports PF" 6910 " VF Replication P0 P1 P2 P3 ML"); 6911 n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES : 6912 NUM_MPS_T5_CLS_SRAM_L_INSTANCES; 6913 for (i = 0; i < n; i++) { 6914 uint64_t tcamx, tcamy, mask; 6915 uint32_t cls_lo, cls_hi; 6916 uint8_t addr[ETHER_ADDR_LEN]; 6917 6918 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); 6919 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); 6920 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 6921 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 6922 6923 if (tcamx & tcamy) 6924 continue; 6925 6926 tcamxy2valmask(tcamx, tcamy, addr, &mask); 6927 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" 6928 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], 6929 addr[3], addr[4], addr[5], (uintmax_t)mask, 6930 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', 6931 G_PORTMAP(cls_hi), G_PF(cls_lo), 6932 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); 6933 6934 if (cls_lo & F_REPLICATE) { 6935 struct fw_ldst_cmd ldst_cmd; 6936 6937 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 6938 ldst_cmd.op_to_addrspace = 6939 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 6940 F_FW_CMD_REQUEST | F_FW_CMD_READ | 6941 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 6942 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 6943 ldst_cmd.u.mps.rplc.fid_idx = 6944 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 6945 V_FW_LDST_CMD_IDX(i)); 6946 6947 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 6948 "t4mps"); 6949 if (rc) 6950 break; 6951 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 6952 sizeof(ldst_cmd), &ldst_cmd); 6953 end_synchronized_op(sc, 0); 6954 6955 if (rc != 0) { 6956 sbuf_printf(sb, 6957 " ------------ error %3u ------------", rc); 6958 rc = 0; 6959 } else { 6960 sbuf_printf(sb, " %08x %08x %08x %08x", 6961 be32toh(ldst_cmd.u.mps.rplc.rplc127_96), 6962 be32toh(ldst_cmd.u.mps.rplc.rplc95_64), 6963 be32toh(ldst_cmd.u.mps.rplc.rplc63_32), 6964 be32toh(ldst_cmd.u.mps.rplc.rplc31_0)); 6965 } 6966 } else 6967 sbuf_printf(sb, "%36s", ""); 6968 6969 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), 6970 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), 6971 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); 6972 } 6973 6974 if (rc) 6975 (void) sbuf_finish(sb); 6976 else 6977 rc = sbuf_finish(sb); 6978 sbuf_delete(sb); 6979 6980 return (rc); 6981 } 6982 6983 static int 6984 sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 6985 { 6986 struct adapter *sc = arg1; 6987 struct sbuf *sb; 6988 int rc; 6989 uint16_t mtus[NMTUS]; 6990 6991 rc = sysctl_wire_old_buffer(req, 0); 6992 if (rc != 0) 6993 return (rc); 6994 6995 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6996 if (sb == NULL) 6997 return (ENOMEM); 6998 6999 t4_read_mtu_tbl(sc, mtus, NULL); 7000 7001 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 7002 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 7003 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 7004 mtus[14], mtus[15]); 7005 7006 rc = sbuf_finish(sb); 7007 sbuf_delete(sb); 7008 7009 return (rc); 7010 } 7011 7012 static int 7013 sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 7014 { 7015 struct adapter *sc = arg1; 7016 struct sbuf *sb; 7017 int rc, i; 7018 uint32_t cnt[PM_NSTATS]; 7019 uint64_t cyc[PM_NSTATS]; 7020 static const char *rx_stats[] = { 7021 "Read:", "Write bypass:", "Write mem:", "Flush:" 7022 }; 7023 static const char *tx_stats[] = { 7024 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:" 7025 }; 7026 7027 rc = sysctl_wire_old_buffer(req, 0); 7028 if (rc != 0) 7029 return (rc); 7030 7031 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7032 if (sb == NULL) 7033 return (ENOMEM); 7034 7035 t4_pmtx_get_stats(sc, cnt, cyc); 7036 sbuf_printf(sb, " Tx pcmds Tx bytes"); 7037 for (i = 0; i < ARRAY_SIZE(tx_stats); i++) 7038 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i], 7039 cyc[i]); 7040 7041 t4_pmrx_get_stats(sc, cnt, cyc); 7042 sbuf_printf(sb, "\n Rx pcmds Rx bytes"); 7043 for (i = 0; i < ARRAY_SIZE(rx_stats); i++) 7044 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i], 7045 cyc[i]); 7046 7047 rc = sbuf_finish(sb); 7048 sbuf_delete(sb); 7049 7050 return (rc); 7051 } 7052 7053 static int 7054 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 7055 { 7056 struct adapter *sc = arg1; 7057 struct sbuf *sb; 7058 int rc; 7059 struct tp_rdma_stats stats; 7060 7061 rc = sysctl_wire_old_buffer(req, 0); 7062 if (rc != 0) 7063 return (rc); 7064 7065 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7066 if (sb == NULL) 7067 return (ENOMEM); 7068 7069 t4_tp_get_rdma_stats(sc, &stats); 7070 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 7071 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 7072 7073 rc = sbuf_finish(sb); 7074 sbuf_delete(sb); 7075 7076 return (rc); 7077 } 7078 7079 static int 7080 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 7081 { 7082 struct adapter *sc = arg1; 7083 struct sbuf *sb; 7084 int rc; 7085 struct tp_tcp_stats v4, v6; 7086 7087 rc = sysctl_wire_old_buffer(req, 0); 7088 if (rc != 0) 7089 return (rc); 7090 7091 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7092 if (sb == NULL) 7093 return (ENOMEM); 7094 7095 t4_tp_get_tcp_stats(sc, &v4, &v6); 7096 sbuf_printf(sb, 7097 " IP IPv6\n"); 7098 sbuf_printf(sb, "OutRsts: %20u %20u\n", 7099 v4.tcpOutRsts, v6.tcpOutRsts); 7100 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 7101 v4.tcpInSegs, v6.tcpInSegs); 7102 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 7103 v4.tcpOutSegs, v6.tcpOutSegs); 7104 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 7105 v4.tcpRetransSegs, v6.tcpRetransSegs); 7106 7107 rc = sbuf_finish(sb); 7108 sbuf_delete(sb); 7109 7110 return (rc); 7111 } 7112 7113 static int 7114 sysctl_tids(SYSCTL_HANDLER_ARGS) 7115 { 7116 struct adapter *sc = arg1; 7117 struct sbuf *sb; 7118 int rc; 7119 struct tid_info *t = &sc->tids; 7120 7121 rc = sysctl_wire_old_buffer(req, 0); 7122 if (rc != 0) 7123 return (rc); 7124 7125 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7126 if (sb == NULL) 7127 return (ENOMEM); 7128 7129 if (t->natids) { 7130 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 7131 t->atids_in_use); 7132 } 7133 7134 if (t->ntids) { 7135 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 7136 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 7137 7138 if (b) { 7139 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 7140 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 7141 t->ntids - 1); 7142 } else { 7143 sbuf_printf(sb, "TID range: %u-%u", 7144 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 7145 t->ntids - 1); 7146 } 7147 } else 7148 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 7149 sbuf_printf(sb, ", in use: %u\n", 7150 atomic_load_acq_int(&t->tids_in_use)); 7151 } 7152 7153 if (t->nstids) { 7154 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 7155 t->stid_base + t->nstids - 1, t->stids_in_use); 7156 } 7157 7158 if (t->nftids) { 7159 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 7160 t->ftid_base + t->nftids - 1); 7161 } 7162 7163 if (t->netids) { 7164 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base, 7165 t->etid_base + t->netids - 1); 7166 } 7167 7168 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 7169 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 7170 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 7171 7172 rc = sbuf_finish(sb); 7173 sbuf_delete(sb); 7174 7175 return (rc); 7176 } 7177 7178 static int 7179 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 7180 { 7181 struct adapter *sc = arg1; 7182 struct sbuf *sb; 7183 int rc; 7184 struct tp_err_stats stats; 7185 7186 rc = sysctl_wire_old_buffer(req, 0); 7187 if (rc != 0) 7188 return (rc); 7189 7190 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7191 if (sb == NULL) 7192 return (ENOMEM); 7193 7194 t4_tp_get_err_stats(sc, &stats); 7195 7196 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 7197 "channel 3\n"); 7198 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 7199 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 7200 stats.macInErrs[3]); 7201 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 7202 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 7203 stats.hdrInErrs[3]); 7204 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 7205 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 7206 stats.tcpInErrs[3]); 7207 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 7208 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 7209 stats.tcp6InErrs[3]); 7210 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 7211 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 7212 stats.tnlCongDrops[3]); 7213 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 7214 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 7215 stats.tnlTxDrops[3]); 7216 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 7217 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 7218 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 7219 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 7220 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 7221 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 7222 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 7223 stats.ofldNoNeigh, stats.ofldCongDefer); 7224 7225 rc = sbuf_finish(sb); 7226 sbuf_delete(sb); 7227 7228 return (rc); 7229 } 7230 7231 struct field_desc { 7232 const char *name; 7233 u_int start; 7234 u_int width; 7235 }; 7236 7237 static void 7238 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) 7239 { 7240 char buf[32]; 7241 int line_size = 0; 7242 7243 while (f->name) { 7244 uint64_t mask = (1ULL << f->width) - 1; 7245 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, 7246 ((uintmax_t)v >> f->start) & mask); 7247 7248 if (line_size + len >= 79) { 7249 line_size = 8; 7250 sbuf_printf(sb, "\n "); 7251 } 7252 sbuf_printf(sb, "%s ", buf); 7253 line_size += len + 1; 7254 f++; 7255 } 7256 sbuf_printf(sb, "\n"); 7257 } 7258 7259 static struct field_desc tp_la0[] = { 7260 { "RcfOpCodeOut", 60, 4 }, 7261 { "State", 56, 4 }, 7262 { "WcfState", 52, 4 }, 7263 { "RcfOpcSrcOut", 50, 2 }, 7264 { "CRxError", 49, 1 }, 7265 { "ERxError", 48, 1 }, 7266 { "SanityFailed", 47, 1 }, 7267 { "SpuriousMsg", 46, 1 }, 7268 { "FlushInputMsg", 45, 1 }, 7269 { "FlushInputCpl", 44, 1 }, 7270 { "RssUpBit", 43, 1 }, 7271 { "RssFilterHit", 42, 1 }, 7272 { "Tid", 32, 10 }, 7273 { "InitTcb", 31, 1 }, 7274 { "LineNumber", 24, 7 }, 7275 { "Emsg", 23, 1 }, 7276 { "EdataOut", 22, 1 }, 7277 { "Cmsg", 21, 1 }, 7278 { "CdataOut", 20, 1 }, 7279 { "EreadPdu", 19, 1 }, 7280 { "CreadPdu", 18, 1 }, 7281 { "TunnelPkt", 17, 1 }, 7282 { "RcfPeerFin", 16, 1 }, 7283 { "RcfReasonOut", 12, 4 }, 7284 { "TxCchannel", 10, 2 }, 7285 { "RcfTxChannel", 8, 2 }, 7286 { "RxEchannel", 6, 2 }, 7287 { "RcfRxChannel", 5, 1 }, 7288 { "RcfDataOutSrdy", 4, 1 }, 7289 { "RxDvld", 3, 1 }, 7290 { "RxOoDvld", 2, 1 }, 7291 { "RxCongestion", 1, 1 }, 7292 { "TxCongestion", 0, 1 }, 7293 { NULL } 7294 }; 7295 7296 static struct field_desc tp_la1[] = { 7297 { "CplCmdIn", 56, 8 }, 7298 { "CplCmdOut", 48, 8 }, 7299 { "ESynOut", 47, 1 }, 7300 { "EAckOut", 46, 1 }, 7301 { "EFinOut", 45, 1 }, 7302 { "ERstOut", 44, 1 }, 7303 { "SynIn", 43, 1 }, 7304 { "AckIn", 42, 1 }, 7305 { "FinIn", 41, 1 }, 7306 { "RstIn", 40, 1 }, 7307 { "DataIn", 39, 1 }, 7308 { "DataInVld", 38, 1 }, 7309 { "PadIn", 37, 1 }, 7310 { "RxBufEmpty", 36, 1 }, 7311 { "RxDdp", 35, 1 }, 7312 { "RxFbCongestion", 34, 1 }, 7313 { "TxFbCongestion", 33, 1 }, 7314 { "TxPktSumSrdy", 32, 1 }, 7315 { "RcfUlpType", 28, 4 }, 7316 { "Eread", 27, 1 }, 7317 { "Ebypass", 26, 1 }, 7318 { "Esave", 25, 1 }, 7319 { "Static0", 24, 1 }, 7320 { "Cread", 23, 1 }, 7321 { "Cbypass", 22, 1 }, 7322 { "Csave", 21, 1 }, 7323 { "CPktOut", 20, 1 }, 7324 { "RxPagePoolFull", 18, 2 }, 7325 { "RxLpbkPkt", 17, 1 }, 7326 { "TxLpbkPkt", 16, 1 }, 7327 { "RxVfValid", 15, 1 }, 7328 { "SynLearned", 14, 1 }, 7329 { "SetDelEntry", 13, 1 }, 7330 { "SetInvEntry", 12, 1 }, 7331 { "CpcmdDvld", 11, 1 }, 7332 { "CpcmdSave", 10, 1 }, 7333 { "RxPstructsFull", 8, 2 }, 7334 { "EpcmdDvld", 7, 1 }, 7335 { "EpcmdFlush", 6, 1 }, 7336 { "EpcmdTrimPrefix", 5, 1 }, 7337 { "EpcmdTrimPostfix", 4, 1 }, 7338 { "ERssIp4Pkt", 3, 1 }, 7339 { "ERssIp6Pkt", 2, 1 }, 7340 { "ERssTcpUdpPkt", 1, 1 }, 7341 { "ERssFceFipPkt", 0, 1 }, 7342 { NULL } 7343 }; 7344 7345 static struct field_desc tp_la2[] = { 7346 { "CplCmdIn", 56, 8 }, 7347 { "MpsVfVld", 55, 1 }, 7348 { "MpsPf", 52, 3 }, 7349 { "MpsVf", 44, 8 }, 7350 { "SynIn", 43, 1 }, 7351 { "AckIn", 42, 1 }, 7352 { "FinIn", 41, 1 }, 7353 { "RstIn", 40, 1 }, 7354 { "DataIn", 39, 1 }, 7355 { "DataInVld", 38, 1 }, 7356 { "PadIn", 37, 1 }, 7357 { "RxBufEmpty", 36, 1 }, 7358 { "RxDdp", 35, 1 }, 7359 { "RxFbCongestion", 34, 1 }, 7360 { "TxFbCongestion", 33, 1 }, 7361 { "TxPktSumSrdy", 32, 1 }, 7362 { "RcfUlpType", 28, 4 }, 7363 { "Eread", 27, 1 }, 7364 { "Ebypass", 26, 1 }, 7365 { "Esave", 25, 1 }, 7366 { "Static0", 24, 1 }, 7367 { "Cread", 23, 1 }, 7368 { "Cbypass", 22, 1 }, 7369 { "Csave", 21, 1 }, 7370 { "CPktOut", 20, 1 }, 7371 { "RxPagePoolFull", 18, 2 }, 7372 { "RxLpbkPkt", 17, 1 }, 7373 { "TxLpbkPkt", 16, 1 }, 7374 { "RxVfValid", 15, 1 }, 7375 { "SynLearned", 14, 1 }, 7376 { "SetDelEntry", 13, 1 }, 7377 { "SetInvEntry", 12, 1 }, 7378 { "CpcmdDvld", 11, 1 }, 7379 { "CpcmdSave", 10, 1 }, 7380 { "RxPstructsFull", 8, 2 }, 7381 { "EpcmdDvld", 7, 1 }, 7382 { "EpcmdFlush", 6, 1 }, 7383 { "EpcmdTrimPrefix", 5, 1 }, 7384 { "EpcmdTrimPostfix", 4, 1 }, 7385 { "ERssIp4Pkt", 3, 1 }, 7386 { "ERssIp6Pkt", 2, 1 }, 7387 { "ERssTcpUdpPkt", 1, 1 }, 7388 { "ERssFceFipPkt", 0, 1 }, 7389 { NULL } 7390 }; 7391 7392 static void 7393 tp_la_show(struct sbuf *sb, uint64_t *p, int idx) 7394 { 7395 7396 field_desc_show(sb, *p, tp_la0); 7397 } 7398 7399 static void 7400 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) 7401 { 7402 7403 if (idx) 7404 sbuf_printf(sb, "\n"); 7405 field_desc_show(sb, p[0], tp_la0); 7406 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 7407 field_desc_show(sb, p[1], tp_la0); 7408 } 7409 7410 static void 7411 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) 7412 { 7413 7414 if (idx) 7415 sbuf_printf(sb, "\n"); 7416 field_desc_show(sb, p[0], tp_la0); 7417 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 7418 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); 7419 } 7420 7421 static int 7422 sysctl_tp_la(SYSCTL_HANDLER_ARGS) 7423 { 7424 struct adapter *sc = arg1; 7425 struct sbuf *sb; 7426 uint64_t *buf, *p; 7427 int rc; 7428 u_int i, inc; 7429 void (*show_func)(struct sbuf *, uint64_t *, int); 7430 7431 rc = sysctl_wire_old_buffer(req, 0); 7432 if (rc != 0) 7433 return (rc); 7434 7435 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7436 if (sb == NULL) 7437 return (ENOMEM); 7438 7439 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); 7440 7441 t4_tp_read_la(sc, buf, NULL); 7442 p = buf; 7443 7444 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { 7445 case 2: 7446 inc = 2; 7447 show_func = tp_la_show2; 7448 break; 7449 case 3: 7450 inc = 2; 7451 show_func = tp_la_show3; 7452 break; 7453 default: 7454 inc = 1; 7455 show_func = tp_la_show; 7456 } 7457 7458 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) 7459 (*show_func)(sb, p, i); 7460 7461 rc = sbuf_finish(sb); 7462 sbuf_delete(sb); 7463 free(buf, M_CXGBE); 7464 return (rc); 7465 } 7466 7467 static int 7468 sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 7469 { 7470 struct adapter *sc = arg1; 7471 struct sbuf *sb; 7472 int rc; 7473 u64 nrate[NCHAN], orate[NCHAN]; 7474 7475 rc = sysctl_wire_old_buffer(req, 0); 7476 if (rc != 0) 7477 return (rc); 7478 7479 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7480 if (sb == NULL) 7481 return (ENOMEM); 7482 7483 t4_get_chan_txrate(sc, nrate, orate); 7484 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 7485 "channel 3\n"); 7486 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 7487 nrate[0], nrate[1], nrate[2], nrate[3]); 7488 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 7489 orate[0], orate[1], orate[2], orate[3]); 7490 7491 rc = sbuf_finish(sb); 7492 sbuf_delete(sb); 7493 7494 return (rc); 7495 } 7496 7497 static int 7498 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) 7499 { 7500 struct adapter *sc = arg1; 7501 struct sbuf *sb; 7502 uint32_t *buf, *p; 7503 int rc, i; 7504 7505 rc = sysctl_wire_old_buffer(req, 0); 7506 if (rc != 0) 7507 return (rc); 7508 7509 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7510 if (sb == NULL) 7511 return (ENOMEM); 7512 7513 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, 7514 M_ZERO | M_WAITOK); 7515 7516 t4_ulprx_read_la(sc, buf); 7517 p = buf; 7518 7519 sbuf_printf(sb, " Pcmd Type Message" 7520 " Data"); 7521 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { 7522 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", 7523 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); 7524 } 7525 7526 rc = sbuf_finish(sb); 7527 sbuf_delete(sb); 7528 free(buf, M_CXGBE); 7529 return (rc); 7530 } 7531 7532 static int 7533 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 7534 { 7535 struct adapter *sc = arg1; 7536 struct sbuf *sb; 7537 int rc, v; 7538 7539 rc = sysctl_wire_old_buffer(req, 0); 7540 if (rc != 0) 7541 return (rc); 7542 7543 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7544 if (sb == NULL) 7545 return (ENOMEM); 7546 7547 v = t4_read_reg(sc, A_SGE_STAT_CFG); 7548 if (G_STATSOURCE_T5(v) == 7) { 7549 if (G_STATMODE(v) == 0) { 7550 sbuf_printf(sb, "total %d, incomplete %d", 7551 t4_read_reg(sc, A_SGE_STAT_TOTAL), 7552 t4_read_reg(sc, A_SGE_STAT_MATCH)); 7553 } else if (G_STATMODE(v) == 1) { 7554 sbuf_printf(sb, "total %d, data overflow %d", 7555 t4_read_reg(sc, A_SGE_STAT_TOTAL), 7556 t4_read_reg(sc, A_SGE_STAT_MATCH)); 7557 } 7558 } 7559 rc = sbuf_finish(sb); 7560 sbuf_delete(sb); 7561 7562 return (rc); 7563 } 7564 #endif 7565 7566 static uint32_t 7567 fconf_to_mode(uint32_t fconf) 7568 { 7569 uint32_t mode; 7570 7571 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 7572 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 7573 7574 if (fconf & F_FRAGMENTATION) 7575 mode |= T4_FILTER_IP_FRAGMENT; 7576 7577 if (fconf & F_MPSHITTYPE) 7578 mode |= T4_FILTER_MPS_HIT_TYPE; 7579 7580 if (fconf & F_MACMATCH) 7581 mode |= T4_FILTER_MAC_IDX; 7582 7583 if (fconf & F_ETHERTYPE) 7584 mode |= T4_FILTER_ETH_TYPE; 7585 7586 if (fconf & F_PROTOCOL) 7587 mode |= T4_FILTER_IP_PROTO; 7588 7589 if (fconf & F_TOS) 7590 mode |= T4_FILTER_IP_TOS; 7591 7592 if (fconf & F_VLAN) 7593 mode |= T4_FILTER_VLAN; 7594 7595 if (fconf & F_VNIC_ID) 7596 mode |= T4_FILTER_VNIC; 7597 7598 if (fconf & F_PORT) 7599 mode |= T4_FILTER_PORT; 7600 7601 if (fconf & F_FCOE) 7602 mode |= T4_FILTER_FCoE; 7603 7604 return (mode); 7605 } 7606 7607 static uint32_t 7608 mode_to_fconf(uint32_t mode) 7609 { 7610 uint32_t fconf = 0; 7611 7612 if (mode & T4_FILTER_IP_FRAGMENT) 7613 fconf |= F_FRAGMENTATION; 7614 7615 if (mode & T4_FILTER_MPS_HIT_TYPE) 7616 fconf |= F_MPSHITTYPE; 7617 7618 if (mode & T4_FILTER_MAC_IDX) 7619 fconf |= F_MACMATCH; 7620 7621 if (mode & T4_FILTER_ETH_TYPE) 7622 fconf |= F_ETHERTYPE; 7623 7624 if (mode & T4_FILTER_IP_PROTO) 7625 fconf |= F_PROTOCOL; 7626 7627 if (mode & T4_FILTER_IP_TOS) 7628 fconf |= F_TOS; 7629 7630 if (mode & T4_FILTER_VLAN) 7631 fconf |= F_VLAN; 7632 7633 if (mode & T4_FILTER_VNIC) 7634 fconf |= F_VNIC_ID; 7635 7636 if (mode & T4_FILTER_PORT) 7637 fconf |= F_PORT; 7638 7639 if (mode & T4_FILTER_FCoE) 7640 fconf |= F_FCOE; 7641 7642 return (fconf); 7643 } 7644 7645 static uint32_t 7646 fspec_to_fconf(struct t4_filter_specification *fs) 7647 { 7648 uint32_t fconf = 0; 7649 7650 if (fs->val.frag || fs->mask.frag) 7651 fconf |= F_FRAGMENTATION; 7652 7653 if (fs->val.matchtype || fs->mask.matchtype) 7654 fconf |= F_MPSHITTYPE; 7655 7656 if (fs->val.macidx || fs->mask.macidx) 7657 fconf |= F_MACMATCH; 7658 7659 if (fs->val.ethtype || fs->mask.ethtype) 7660 fconf |= F_ETHERTYPE; 7661 7662 if (fs->val.proto || fs->mask.proto) 7663 fconf |= F_PROTOCOL; 7664 7665 if (fs->val.tos || fs->mask.tos) 7666 fconf |= F_TOS; 7667 7668 if (fs->val.vlan_vld || fs->mask.vlan_vld) 7669 fconf |= F_VLAN; 7670 7671 if (fs->val.vnic_vld || fs->mask.vnic_vld) 7672 fconf |= F_VNIC_ID; 7673 7674 if (fs->val.iport || fs->mask.iport) 7675 fconf |= F_PORT; 7676 7677 if (fs->val.fcoe || fs->mask.fcoe) 7678 fconf |= F_FCOE; 7679 7680 return (fconf); 7681 } 7682 7683 static int 7684 get_filter_mode(struct adapter *sc, uint32_t *mode) 7685 { 7686 int rc; 7687 uint32_t fconf; 7688 7689 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7690 "t4getfm"); 7691 if (rc) 7692 return (rc); 7693 7694 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 7695 A_TP_VLAN_PRI_MAP); 7696 7697 if (sc->params.tp.vlan_pri_map != fconf) { 7698 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 7699 device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map, 7700 fconf); 7701 } 7702 7703 *mode = fconf_to_mode(fconf); 7704 7705 end_synchronized_op(sc, LOCK_HELD); 7706 return (0); 7707 } 7708 7709 static int 7710 set_filter_mode(struct adapter *sc, uint32_t mode) 7711 { 7712 uint32_t fconf; 7713 int rc; 7714 7715 fconf = mode_to_fconf(mode); 7716 7717 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7718 "t4setfm"); 7719 if (rc) 7720 return (rc); 7721 7722 if (sc->tids.ftids_in_use > 0) { 7723 rc = EBUSY; 7724 goto done; 7725 } 7726 7727 #ifdef TCP_OFFLOAD 7728 if (uld_active(sc, ULD_TOM)) { 7729 rc = EBUSY; 7730 goto done; 7731 } 7732 #endif 7733 7734 rc = -t4_set_filter_mode(sc, fconf); 7735 done: 7736 end_synchronized_op(sc, LOCK_HELD); 7737 return (rc); 7738 } 7739 7740 static inline uint64_t 7741 get_filter_hits(struct adapter *sc, uint32_t fid) 7742 { 7743 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 7744 uint64_t hits; 7745 7746 memwin_info(sc, 0, &mw_base, NULL); 7747 off = position_memwin(sc, 0, 7748 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 7749 if (is_t4(sc)) { 7750 hits = t4_read_reg64(sc, mw_base + off + 16); 7751 hits = be64toh(hits); 7752 } else { 7753 hits = t4_read_reg(sc, mw_base + off + 24); 7754 hits = be32toh(hits); 7755 } 7756 7757 return (hits); 7758 } 7759 7760 static int 7761 get_filter(struct adapter *sc, struct t4_filter *t) 7762 { 7763 int i, rc, nfilters = sc->tids.nftids; 7764 struct filter_entry *f; 7765 7766 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7767 "t4getf"); 7768 if (rc) 7769 return (rc); 7770 7771 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 7772 t->idx >= nfilters) { 7773 t->idx = 0xffffffff; 7774 goto done; 7775 } 7776 7777 f = &sc->tids.ftid_tab[t->idx]; 7778 for (i = t->idx; i < nfilters; i++, f++) { 7779 if (f->valid) { 7780 t->idx = i; 7781 t->l2tidx = f->l2t ? f->l2t->idx : 0; 7782 t->smtidx = f->smtidx; 7783 if (f->fs.hitcnts) 7784 t->hits = get_filter_hits(sc, t->idx); 7785 else 7786 t->hits = UINT64_MAX; 7787 t->fs = f->fs; 7788 7789 goto done; 7790 } 7791 } 7792 7793 t->idx = 0xffffffff; 7794 done: 7795 end_synchronized_op(sc, LOCK_HELD); 7796 return (0); 7797 } 7798 7799 static int 7800 set_filter(struct adapter *sc, struct t4_filter *t) 7801 { 7802 unsigned int nfilters, nports; 7803 struct filter_entry *f; 7804 int i, rc; 7805 7806 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 7807 if (rc) 7808 return (rc); 7809 7810 nfilters = sc->tids.nftids; 7811 nports = sc->params.nports; 7812 7813 if (nfilters == 0) { 7814 rc = ENOTSUP; 7815 goto done; 7816 } 7817 7818 if (!(sc->flags & FULL_INIT_DONE)) { 7819 rc = EAGAIN; 7820 goto done; 7821 } 7822 7823 if (t->idx >= nfilters) { 7824 rc = EINVAL; 7825 goto done; 7826 } 7827 7828 /* Validate against the global filter mode */ 7829 if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) != 7830 sc->params.tp.vlan_pri_map) { 7831 rc = E2BIG; 7832 goto done; 7833 } 7834 7835 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 7836 rc = EINVAL; 7837 goto done; 7838 } 7839 7840 if (t->fs.val.iport >= nports) { 7841 rc = EINVAL; 7842 goto done; 7843 } 7844 7845 /* Can't specify an iq if not steering to it */ 7846 if (!t->fs.dirsteer && t->fs.iq) { 7847 rc = EINVAL; 7848 goto done; 7849 } 7850 7851 /* IPv6 filter idx must be 4 aligned */ 7852 if (t->fs.type == 1 && 7853 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 7854 rc = EINVAL; 7855 goto done; 7856 } 7857 7858 if (sc->tids.ftid_tab == NULL) { 7859 KASSERT(sc->tids.ftids_in_use == 0, 7860 ("%s: no memory allocated but filters_in_use > 0", 7861 __func__)); 7862 7863 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 7864 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 7865 if (sc->tids.ftid_tab == NULL) { 7866 rc = ENOMEM; 7867 goto done; 7868 } 7869 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 7870 } 7871 7872 for (i = 0; i < 4; i++) { 7873 f = &sc->tids.ftid_tab[t->idx + i]; 7874 7875 if (f->pending || f->valid) { 7876 rc = EBUSY; 7877 goto done; 7878 } 7879 if (f->locked) { 7880 rc = EPERM; 7881 goto done; 7882 } 7883 7884 if (t->fs.type == 0) 7885 break; 7886 } 7887 7888 f = &sc->tids.ftid_tab[t->idx]; 7889 f->fs = t->fs; 7890 7891 rc = set_filter_wr(sc, t->idx); 7892 done: 7893 end_synchronized_op(sc, 0); 7894 7895 if (rc == 0) { 7896 mtx_lock(&sc->tids.ftid_lock); 7897 for (;;) { 7898 if (f->pending == 0) { 7899 rc = f->valid ? 0 : EIO; 7900 break; 7901 } 7902 7903 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7904 PCATCH, "t4setfw", 0)) { 7905 rc = EINPROGRESS; 7906 break; 7907 } 7908 } 7909 mtx_unlock(&sc->tids.ftid_lock); 7910 } 7911 return (rc); 7912 } 7913 7914 static int 7915 del_filter(struct adapter *sc, struct t4_filter *t) 7916 { 7917 unsigned int nfilters; 7918 struct filter_entry *f; 7919 int rc; 7920 7921 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 7922 if (rc) 7923 return (rc); 7924 7925 nfilters = sc->tids.nftids; 7926 7927 if (nfilters == 0) { 7928 rc = ENOTSUP; 7929 goto done; 7930 } 7931 7932 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 7933 t->idx >= nfilters) { 7934 rc = EINVAL; 7935 goto done; 7936 } 7937 7938 if (!(sc->flags & FULL_INIT_DONE)) { 7939 rc = EAGAIN; 7940 goto done; 7941 } 7942 7943 f = &sc->tids.ftid_tab[t->idx]; 7944 7945 if (f->pending) { 7946 rc = EBUSY; 7947 goto done; 7948 } 7949 if (f->locked) { 7950 rc = EPERM; 7951 goto done; 7952 } 7953 7954 if (f->valid) { 7955 t->fs = f->fs; /* extra info for the caller */ 7956 rc = del_filter_wr(sc, t->idx); 7957 } 7958 7959 done: 7960 end_synchronized_op(sc, 0); 7961 7962 if (rc == 0) { 7963 mtx_lock(&sc->tids.ftid_lock); 7964 for (;;) { 7965 if (f->pending == 0) { 7966 rc = f->valid ? EIO : 0; 7967 break; 7968 } 7969 7970 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7971 PCATCH, "t4delfw", 0)) { 7972 rc = EINPROGRESS; 7973 break; 7974 } 7975 } 7976 mtx_unlock(&sc->tids.ftid_lock); 7977 } 7978 7979 return (rc); 7980 } 7981 7982 static void 7983 clear_filter(struct filter_entry *f) 7984 { 7985 if (f->l2t) 7986 t4_l2t_release(f->l2t); 7987 7988 bzero(f, sizeof (*f)); 7989 } 7990 7991 static int 7992 set_filter_wr(struct adapter *sc, int fidx) 7993 { 7994 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7995 struct fw_filter_wr *fwr; 7996 unsigned int ftid; 7997 struct wrq_cookie cookie; 7998 7999 ASSERT_SYNCHRONIZED_OP(sc); 8000 8001 if (f->fs.newdmac || f->fs.newvlan) { 8002 /* This filter needs an L2T entry; allocate one. */ 8003 f->l2t = t4_l2t_alloc_switching(sc->l2t); 8004 if (f->l2t == NULL) 8005 return (EAGAIN); 8006 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 8007 f->fs.dmac)) { 8008 t4_l2t_release(f->l2t); 8009 f->l2t = NULL; 8010 return (ENOMEM); 8011 } 8012 } 8013 8014 ftid = sc->tids.ftid_base + fidx; 8015 8016 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 8017 if (fwr == NULL) 8018 return (ENOMEM); 8019 bzero(fwr, sizeof(*fwr)); 8020 8021 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 8022 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 8023 fwr->tid_to_iq = 8024 htobe32(V_FW_FILTER_WR_TID(ftid) | 8025 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 8026 V_FW_FILTER_WR_NOREPLY(0) | 8027 V_FW_FILTER_WR_IQ(f->fs.iq)); 8028 fwr->del_filter_to_l2tix = 8029 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 8030 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 8031 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 8032 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 8033 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 8034 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 8035 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 8036 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 8037 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 8038 f->fs.newvlan == VLAN_REWRITE) | 8039 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 8040 f->fs.newvlan == VLAN_REWRITE) | 8041 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 8042 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 8043 V_FW_FILTER_WR_PRIO(f->fs.prio) | 8044 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 8045 fwr->ethtype = htobe16(f->fs.val.ethtype); 8046 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 8047 fwr->frag_to_ovlan_vldm = 8048 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 8049 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 8050 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 8051 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 8052 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 8053 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 8054 fwr->smac_sel = 0; 8055 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 8056 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 8057 fwr->maci_to_matchtypem = 8058 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 8059 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 8060 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 8061 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 8062 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 8063 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 8064 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 8065 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 8066 fwr->ptcl = f->fs.val.proto; 8067 fwr->ptclm = f->fs.mask.proto; 8068 fwr->ttyp = f->fs.val.tos; 8069 fwr->ttypm = f->fs.mask.tos; 8070 fwr->ivlan = htobe16(f->fs.val.vlan); 8071 fwr->ivlanm = htobe16(f->fs.mask.vlan); 8072 fwr->ovlan = htobe16(f->fs.val.vnic); 8073 fwr->ovlanm = htobe16(f->fs.mask.vnic); 8074 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 8075 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 8076 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 8077 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 8078 fwr->lp = htobe16(f->fs.val.dport); 8079 fwr->lpm = htobe16(f->fs.mask.dport); 8080 fwr->fp = htobe16(f->fs.val.sport); 8081 fwr->fpm = htobe16(f->fs.mask.sport); 8082 if (f->fs.newsmac) 8083 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 8084 8085 f->pending = 1; 8086 sc->tids.ftids_in_use++; 8087 8088 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 8089 return (0); 8090 } 8091 8092 static int 8093 del_filter_wr(struct adapter *sc, int fidx) 8094 { 8095 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 8096 struct fw_filter_wr *fwr; 8097 unsigned int ftid; 8098 struct wrq_cookie cookie; 8099 8100 ftid = sc->tids.ftid_base + fidx; 8101 8102 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 8103 if (fwr == NULL) 8104 return (ENOMEM); 8105 bzero(fwr, sizeof (*fwr)); 8106 8107 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 8108 8109 f->pending = 1; 8110 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 8111 return (0); 8112 } 8113 8114 int 8115 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 8116 { 8117 struct adapter *sc = iq->adapter; 8118 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 8119 unsigned int idx = GET_TID(rpl); 8120 unsigned int rc; 8121 struct filter_entry *f; 8122 8123 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 8124 rss->opcode)); 8125 8126 if (is_ftid(sc, idx)) { 8127 8128 idx -= sc->tids.ftid_base; 8129 f = &sc->tids.ftid_tab[idx]; 8130 rc = G_COOKIE(rpl->cookie); 8131 8132 mtx_lock(&sc->tids.ftid_lock); 8133 if (rc == FW_FILTER_WR_FLT_ADDED) { 8134 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 8135 __func__, idx)); 8136 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 8137 f->pending = 0; /* asynchronous setup completed */ 8138 f->valid = 1; 8139 } else { 8140 if (rc != FW_FILTER_WR_FLT_DELETED) { 8141 /* Add or delete failed, display an error */ 8142 log(LOG_ERR, 8143 "filter %u setup failed with error %u\n", 8144 idx, rc); 8145 } 8146 8147 clear_filter(f); 8148 sc->tids.ftids_in_use--; 8149 } 8150 wakeup(&sc->tids.ftid_tab); 8151 mtx_unlock(&sc->tids.ftid_lock); 8152 } 8153 8154 return (0); 8155 } 8156 8157 static int 8158 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 8159 { 8160 int rc; 8161 8162 if (cntxt->cid > M_CTXTQID) 8163 return (EINVAL); 8164 8165 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 8166 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 8167 return (EINVAL); 8168 8169 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 8170 if (rc) 8171 return (rc); 8172 8173 if (sc->flags & FW_OK) { 8174 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 8175 &cntxt->data[0]); 8176 if (rc == 0) 8177 goto done; 8178 } 8179 8180 /* 8181 * Read via firmware failed or wasn't even attempted. Read directly via 8182 * the backdoor. 8183 */ 8184 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 8185 done: 8186 end_synchronized_op(sc, 0); 8187 return (rc); 8188 } 8189 8190 static int 8191 load_fw(struct adapter *sc, struct t4_data *fw) 8192 { 8193 int rc; 8194 uint8_t *fw_data; 8195 8196 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 8197 if (rc) 8198 return (rc); 8199 8200 if (sc->flags & FULL_INIT_DONE) { 8201 rc = EBUSY; 8202 goto done; 8203 } 8204 8205 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 8206 if (fw_data == NULL) { 8207 rc = ENOMEM; 8208 goto done; 8209 } 8210 8211 rc = copyin(fw->data, fw_data, fw->len); 8212 if (rc == 0) 8213 rc = -t4_load_fw(sc, fw_data, fw->len); 8214 8215 free(fw_data, M_CXGBE); 8216 done: 8217 end_synchronized_op(sc, 0); 8218 return (rc); 8219 } 8220 8221 static int 8222 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 8223 { 8224 uint32_t addr, off, remaining, i, n; 8225 uint32_t *buf, *b; 8226 uint32_t mw_base, mw_aperture; 8227 int rc; 8228 uint8_t *dst; 8229 8230 rc = validate_mem_range(sc, mr->addr, mr->len); 8231 if (rc != 0) 8232 return (rc); 8233 8234 memwin_info(sc, win, &mw_base, &mw_aperture); 8235 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 8236 addr = mr->addr; 8237 remaining = mr->len; 8238 dst = (void *)mr->data; 8239 8240 while (remaining) { 8241 off = position_memwin(sc, win, addr); 8242 8243 /* number of bytes that we'll copy in the inner loop */ 8244 n = min(remaining, mw_aperture - off); 8245 for (i = 0; i < n; i += 4) 8246 *b++ = t4_read_reg(sc, mw_base + off + i); 8247 8248 rc = copyout(buf, dst, n); 8249 if (rc != 0) 8250 break; 8251 8252 b = buf; 8253 dst += n; 8254 remaining -= n; 8255 addr += n; 8256 } 8257 8258 free(buf, M_CXGBE); 8259 return (rc); 8260 } 8261 8262 static int 8263 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 8264 { 8265 int rc; 8266 8267 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 8268 return (EINVAL); 8269 8270 if (i2cd->len > sizeof(i2cd->data)) 8271 return (EFBIG); 8272 8273 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 8274 if (rc) 8275 return (rc); 8276 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 8277 i2cd->offset, i2cd->len, &i2cd->data[0]); 8278 end_synchronized_op(sc, 0); 8279 8280 return (rc); 8281 } 8282 8283 static int 8284 in_range(int val, int lo, int hi) 8285 { 8286 8287 return (val < 0 || (val <= hi && val >= lo)); 8288 } 8289 8290 static int 8291 set_sched_class(struct adapter *sc, struct t4_sched_params *p) 8292 { 8293 int fw_subcmd, fw_type, rc; 8294 8295 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc"); 8296 if (rc) 8297 return (rc); 8298 8299 if (!(sc->flags & FULL_INIT_DONE)) { 8300 rc = EAGAIN; 8301 goto done; 8302 } 8303 8304 /* 8305 * Translate the cxgbetool parameters into T4 firmware parameters. (The 8306 * sub-command and type are in common locations.) 8307 */ 8308 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG) 8309 fw_subcmd = FW_SCHED_SC_CONFIG; 8310 else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS) 8311 fw_subcmd = FW_SCHED_SC_PARAMS; 8312 else { 8313 rc = EINVAL; 8314 goto done; 8315 } 8316 if (p->type == SCHED_CLASS_TYPE_PACKET) 8317 fw_type = FW_SCHED_TYPE_PKTSCHED; 8318 else { 8319 rc = EINVAL; 8320 goto done; 8321 } 8322 8323 if (fw_subcmd == FW_SCHED_SC_CONFIG) { 8324 /* Vet our parameters ..*/ 8325 if (p->u.config.minmax < 0) { 8326 rc = EINVAL; 8327 goto done; 8328 } 8329 8330 /* And pass the request to the firmware ...*/ 8331 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1); 8332 goto done; 8333 } 8334 8335 if (fw_subcmd == FW_SCHED_SC_PARAMS) { 8336 int fw_level; 8337 int fw_mode; 8338 int fw_rateunit; 8339 int fw_ratemode; 8340 8341 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL) 8342 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL; 8343 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) 8344 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR; 8345 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) 8346 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL; 8347 else { 8348 rc = EINVAL; 8349 goto done; 8350 } 8351 8352 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS) 8353 fw_mode = FW_SCHED_PARAMS_MODE_CLASS; 8354 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW) 8355 fw_mode = FW_SCHED_PARAMS_MODE_FLOW; 8356 else { 8357 rc = EINVAL; 8358 goto done; 8359 } 8360 8361 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS) 8362 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 8363 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS) 8364 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE; 8365 else { 8366 rc = EINVAL; 8367 goto done; 8368 } 8369 8370 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL) 8371 fw_ratemode = FW_SCHED_PARAMS_RATE_REL; 8372 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS) 8373 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS; 8374 else { 8375 rc = EINVAL; 8376 goto done; 8377 } 8378 8379 /* Vet our parameters ... */ 8380 if (!in_range(p->u.params.channel, 0, 3) || 8381 !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) || 8382 !in_range(p->u.params.minrate, 0, 10000000) || 8383 !in_range(p->u.params.maxrate, 0, 10000000) || 8384 !in_range(p->u.params.weight, 0, 100)) { 8385 rc = ERANGE; 8386 goto done; 8387 } 8388 8389 /* 8390 * Translate any unset parameters into the firmware's 8391 * nomenclature and/or fail the call if the parameters 8392 * are required ... 8393 */ 8394 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 || 8395 p->u.params.channel < 0 || p->u.params.cl < 0) { 8396 rc = EINVAL; 8397 goto done; 8398 } 8399 if (p->u.params.minrate < 0) 8400 p->u.params.minrate = 0; 8401 if (p->u.params.maxrate < 0) { 8402 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 8403 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 8404 rc = EINVAL; 8405 goto done; 8406 } else 8407 p->u.params.maxrate = 0; 8408 } 8409 if (p->u.params.weight < 0) { 8410 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) { 8411 rc = EINVAL; 8412 goto done; 8413 } else 8414 p->u.params.weight = 0; 8415 } 8416 if (p->u.params.pktsize < 0) { 8417 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 8418 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 8419 rc = EINVAL; 8420 goto done; 8421 } else 8422 p->u.params.pktsize = 0; 8423 } 8424 8425 /* See what the firmware thinks of the request ... */ 8426 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode, 8427 fw_rateunit, fw_ratemode, p->u.params.channel, 8428 p->u.params.cl, p->u.params.minrate, p->u.params.maxrate, 8429 p->u.params.weight, p->u.params.pktsize, 1); 8430 goto done; 8431 } 8432 8433 rc = EINVAL; 8434 done: 8435 end_synchronized_op(sc, 0); 8436 return (rc); 8437 } 8438 8439 static int 8440 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p) 8441 { 8442 struct port_info *pi = NULL; 8443 struct vi_info *vi; 8444 struct sge_txq *txq; 8445 uint32_t fw_mnem, fw_queue, fw_class; 8446 int i, rc; 8447 8448 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq"); 8449 if (rc) 8450 return (rc); 8451 8452 if (!(sc->flags & FULL_INIT_DONE)) { 8453 rc = EAGAIN; 8454 goto done; 8455 } 8456 8457 if (p->port >= sc->params.nports) { 8458 rc = EINVAL; 8459 goto done; 8460 } 8461 8462 /* XXX: Only supported for the main VI. */ 8463 pi = sc->port[p->port]; 8464 vi = &pi->vi[0]; 8465 if (!in_range(p->queue, 0, vi->ntxq - 1) || !in_range(p->cl, 0, 7)) { 8466 rc = EINVAL; 8467 goto done; 8468 } 8469 8470 /* 8471 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX 8472 * Scheduling Class in this case). 8473 */ 8474 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 8475 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH)); 8476 fw_class = p->cl < 0 ? 0xffffffff : p->cl; 8477 8478 /* 8479 * If op.queue is non-negative, then we're only changing the scheduling 8480 * on a single specified TX queue. 8481 */ 8482 if (p->queue >= 0) { 8483 txq = &sc->sge.txq[vi->first_txq + p->queue]; 8484 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 8485 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 8486 &fw_class); 8487 goto done; 8488 } 8489 8490 /* 8491 * Change the scheduling on all the TX queues for the 8492 * interface. 8493 */ 8494 for_each_txq(vi, i, txq) { 8495 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 8496 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 8497 &fw_class); 8498 if (rc) 8499 goto done; 8500 } 8501 8502 rc = 0; 8503 done: 8504 end_synchronized_op(sc, 0); 8505 return (rc); 8506 } 8507 8508 int 8509 t4_os_find_pci_capability(struct adapter *sc, int cap) 8510 { 8511 int i; 8512 8513 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 8514 } 8515 8516 int 8517 t4_os_pci_save_state(struct adapter *sc) 8518 { 8519 device_t dev; 8520 struct pci_devinfo *dinfo; 8521 8522 dev = sc->dev; 8523 dinfo = device_get_ivars(dev); 8524 8525 pci_cfg_save(dev, dinfo, 0); 8526 return (0); 8527 } 8528 8529 int 8530 t4_os_pci_restore_state(struct adapter *sc) 8531 { 8532 device_t dev; 8533 struct pci_devinfo *dinfo; 8534 8535 dev = sc->dev; 8536 dinfo = device_get_ivars(dev); 8537 8538 pci_cfg_restore(dev, dinfo); 8539 return (0); 8540 } 8541 8542 void 8543 t4_os_portmod_changed(const struct adapter *sc, int idx) 8544 { 8545 struct port_info *pi = sc->port[idx]; 8546 struct vi_info *vi; 8547 struct ifnet *ifp; 8548 int v; 8549 static const char *mod_str[] = { 8550 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 8551 }; 8552 8553 for_each_vi(pi, v, vi) { 8554 build_medialist(pi, &vi->media); 8555 } 8556 8557 ifp = pi->vi[0].ifp; 8558 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 8559 if_printf(ifp, "transceiver unplugged.\n"); 8560 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 8561 if_printf(ifp, "unknown transceiver inserted.\n"); 8562 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 8563 if_printf(ifp, "unsupported transceiver inserted.\n"); 8564 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 8565 if_printf(ifp, "%s transceiver inserted.\n", 8566 mod_str[pi->mod_type]); 8567 } else { 8568 if_printf(ifp, "transceiver (type %d) inserted.\n", 8569 pi->mod_type); 8570 } 8571 } 8572 8573 void 8574 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason) 8575 { 8576 struct port_info *pi = sc->port[idx]; 8577 struct vi_info *vi; 8578 struct ifnet *ifp; 8579 int v; 8580 8581 if (link_stat) 8582 pi->linkdnrc = -1; 8583 else { 8584 if (reason >= 0) 8585 pi->linkdnrc = reason; 8586 } 8587 for_each_vi(pi, v, vi) { 8588 ifp = vi->ifp; 8589 if (ifp == NULL) 8590 continue; 8591 8592 if (link_stat) { 8593 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 8594 if_link_state_change(ifp, LINK_STATE_UP); 8595 } else { 8596 if_link_state_change(ifp, LINK_STATE_DOWN); 8597 } 8598 } 8599 } 8600 8601 void 8602 t4_iterate(void (*func)(struct adapter *, void *), void *arg) 8603 { 8604 struct adapter *sc; 8605 8606 sx_slock(&t4_list_lock); 8607 SLIST_FOREACH(sc, &t4_list, link) { 8608 /* 8609 * func should not make any assumptions about what state sc is 8610 * in - the only guarantee is that sc->sc_lock is a valid lock. 8611 */ 8612 func(sc, arg); 8613 } 8614 sx_sunlock(&t4_list_lock); 8615 } 8616 8617 static int 8618 t4_open(struct cdev *dev, int flags, int type, struct thread *td) 8619 { 8620 return (0); 8621 } 8622 8623 static int 8624 t4_close(struct cdev *dev, int flags, int type, struct thread *td) 8625 { 8626 return (0); 8627 } 8628 8629 static int 8630 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 8631 struct thread *td) 8632 { 8633 int rc; 8634 struct adapter *sc = dev->si_drv1; 8635 8636 rc = priv_check(td, PRIV_DRIVER); 8637 if (rc != 0) 8638 return (rc); 8639 8640 switch (cmd) { 8641 case CHELSIO_T4_GETREG: { 8642 struct t4_reg *edata = (struct t4_reg *)data; 8643 8644 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8645 return (EFAULT); 8646 8647 if (edata->size == 4) 8648 edata->val = t4_read_reg(sc, edata->addr); 8649 else if (edata->size == 8) 8650 edata->val = t4_read_reg64(sc, edata->addr); 8651 else 8652 return (EINVAL); 8653 8654 break; 8655 } 8656 case CHELSIO_T4_SETREG: { 8657 struct t4_reg *edata = (struct t4_reg *)data; 8658 8659 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8660 return (EFAULT); 8661 8662 if (edata->size == 4) { 8663 if (edata->val & 0xffffffff00000000) 8664 return (EINVAL); 8665 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 8666 } else if (edata->size == 8) 8667 t4_write_reg64(sc, edata->addr, edata->val); 8668 else 8669 return (EINVAL); 8670 break; 8671 } 8672 case CHELSIO_T4_REGDUMP: { 8673 struct t4_regdump *regs = (struct t4_regdump *)data; 8674 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 8675 uint8_t *buf; 8676 8677 if (regs->len < reglen) { 8678 regs->len = reglen; /* hint to the caller */ 8679 return (ENOBUFS); 8680 } 8681 8682 regs->len = reglen; 8683 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 8684 t4_get_regs(sc, regs, buf); 8685 rc = copyout(buf, regs->data, reglen); 8686 free(buf, M_CXGBE); 8687 break; 8688 } 8689 case CHELSIO_T4_GET_FILTER_MODE: 8690 rc = get_filter_mode(sc, (uint32_t *)data); 8691 break; 8692 case CHELSIO_T4_SET_FILTER_MODE: 8693 rc = set_filter_mode(sc, *(uint32_t *)data); 8694 break; 8695 case CHELSIO_T4_GET_FILTER: 8696 rc = get_filter(sc, (struct t4_filter *)data); 8697 break; 8698 case CHELSIO_T4_SET_FILTER: 8699 rc = set_filter(sc, (struct t4_filter *)data); 8700 break; 8701 case CHELSIO_T4_DEL_FILTER: 8702 rc = del_filter(sc, (struct t4_filter *)data); 8703 break; 8704 case CHELSIO_T4_GET_SGE_CONTEXT: 8705 rc = get_sge_context(sc, (struct t4_sge_context *)data); 8706 break; 8707 case CHELSIO_T4_LOAD_FW: 8708 rc = load_fw(sc, (struct t4_data *)data); 8709 break; 8710 case CHELSIO_T4_GET_MEM: 8711 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 8712 break; 8713 case CHELSIO_T4_GET_I2C: 8714 rc = read_i2c(sc, (struct t4_i2c_data *)data); 8715 break; 8716 case CHELSIO_T4_CLEAR_STATS: { 8717 int i, v; 8718 u_int port_id = *(uint32_t *)data; 8719 struct port_info *pi; 8720 struct vi_info *vi; 8721 8722 if (port_id >= sc->params.nports) 8723 return (EINVAL); 8724 pi = sc->port[port_id]; 8725 8726 /* MAC stats */ 8727 t4_clr_port_stats(sc, pi->tx_chan); 8728 pi->tx_parse_error = 0; 8729 mtx_lock(&sc->regwin_lock); 8730 for_each_vi(pi, v, vi) { 8731 if (vi->flags & VI_INIT_DONE) 8732 t4_clr_vi_stats(sc, vi->viid); 8733 } 8734 mtx_unlock(&sc->regwin_lock); 8735 8736 /* 8737 * Since this command accepts a port, clear stats for 8738 * all VIs on this port. 8739 */ 8740 for_each_vi(pi, v, vi) { 8741 if (vi->flags & VI_INIT_DONE) { 8742 struct sge_rxq *rxq; 8743 struct sge_txq *txq; 8744 struct sge_wrq *wrq; 8745 8746 if (vi->flags & VI_NETMAP) 8747 continue; 8748 8749 for_each_rxq(vi, i, rxq) { 8750 #if defined(INET) || defined(INET6) 8751 rxq->lro.lro_queued = 0; 8752 rxq->lro.lro_flushed = 0; 8753 #endif 8754 rxq->rxcsum = 0; 8755 rxq->vlan_extraction = 0; 8756 } 8757 8758 for_each_txq(vi, i, txq) { 8759 txq->txcsum = 0; 8760 txq->tso_wrs = 0; 8761 txq->vlan_insertion = 0; 8762 txq->imm_wrs = 0; 8763 txq->sgl_wrs = 0; 8764 txq->txpkt_wrs = 0; 8765 txq->txpkts0_wrs = 0; 8766 txq->txpkts1_wrs = 0; 8767 txq->txpkts0_pkts = 0; 8768 txq->txpkts1_pkts = 0; 8769 mp_ring_reset_stats(txq->r); 8770 } 8771 8772 #ifdef TCP_OFFLOAD 8773 /* nothing to clear for each ofld_rxq */ 8774 8775 for_each_ofld_txq(vi, i, wrq) { 8776 wrq->tx_wrs_direct = 0; 8777 wrq->tx_wrs_copied = 0; 8778 } 8779 #endif 8780 8781 if (IS_MAIN_VI(vi)) { 8782 wrq = &sc->sge.ctrlq[pi->port_id]; 8783 wrq->tx_wrs_direct = 0; 8784 wrq->tx_wrs_copied = 0; 8785 } 8786 } 8787 } 8788 break; 8789 } 8790 case CHELSIO_T4_SCHED_CLASS: 8791 rc = set_sched_class(sc, (struct t4_sched_params *)data); 8792 break; 8793 case CHELSIO_T4_SCHED_QUEUE: 8794 rc = set_sched_queue(sc, (struct t4_sched_queue *)data); 8795 break; 8796 case CHELSIO_T4_GET_TRACER: 8797 rc = t4_get_tracer(sc, (struct t4_tracer *)data); 8798 break; 8799 case CHELSIO_T4_SET_TRACER: 8800 rc = t4_set_tracer(sc, (struct t4_tracer *)data); 8801 break; 8802 default: 8803 rc = EINVAL; 8804 } 8805 8806 return (rc); 8807 } 8808 8809 #ifdef TCP_OFFLOAD 8810 void 8811 t4_iscsi_init(struct adapter *sc, u_int tag_mask, const u_int *pgsz_order) 8812 { 8813 8814 t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask); 8815 t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) | 8816 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) | 8817 V_HPZ3(pgsz_order[3])); 8818 } 8819 8820 static int 8821 toe_capability(struct vi_info *vi, int enable) 8822 { 8823 int rc; 8824 struct port_info *pi = vi->pi; 8825 struct adapter *sc = pi->adapter; 8826 8827 ASSERT_SYNCHRONIZED_OP(sc); 8828 8829 if (!is_offload(sc)) 8830 return (ENODEV); 8831 8832 if (enable) { 8833 if ((vi->ifp->if_capenable & IFCAP_TOE) != 0) { 8834 /* TOE is already enabled. */ 8835 return (0); 8836 } 8837 8838 /* 8839 * We need the port's queues around so that we're able to send 8840 * and receive CPLs to/from the TOE even if the ifnet for this 8841 * port has never been UP'd administratively. 8842 */ 8843 if (!(vi->flags & VI_INIT_DONE)) { 8844 rc = cxgbe_init_synchronized(vi); 8845 if (rc) 8846 return (rc); 8847 } 8848 if (!(pi->vi[0].flags & VI_INIT_DONE)) { 8849 rc = cxgbe_init_synchronized(&pi->vi[0]); 8850 if (rc) 8851 return (rc); 8852 } 8853 8854 if (isset(&sc->offload_map, pi->port_id)) { 8855 /* TOE is enabled on another VI of this port. */ 8856 pi->uld_vis++; 8857 return (0); 8858 } 8859 8860 if (!uld_active(sc, ULD_TOM)) { 8861 rc = t4_activate_uld(sc, ULD_TOM); 8862 if (rc == EAGAIN) { 8863 log(LOG_WARNING, 8864 "You must kldload t4_tom.ko before trying " 8865 "to enable TOE on a cxgbe interface.\n"); 8866 } 8867 if (rc != 0) 8868 return (rc); 8869 KASSERT(sc->tom_softc != NULL, 8870 ("%s: TOM activated but softc NULL", __func__)); 8871 KASSERT(uld_active(sc, ULD_TOM), 8872 ("%s: TOM activated but flag not set", __func__)); 8873 } 8874 8875 /* Activate iWARP and iSCSI too, if the modules are loaded. */ 8876 if (!uld_active(sc, ULD_IWARP)) 8877 (void) t4_activate_uld(sc, ULD_IWARP); 8878 if (!uld_active(sc, ULD_ISCSI)) 8879 (void) t4_activate_uld(sc, ULD_ISCSI); 8880 8881 pi->uld_vis++; 8882 setbit(&sc->offload_map, pi->port_id); 8883 } else { 8884 pi->uld_vis--; 8885 8886 if (!isset(&sc->offload_map, pi->port_id) || pi->uld_vis > 0) 8887 return (0); 8888 8889 KASSERT(uld_active(sc, ULD_TOM), 8890 ("%s: TOM never initialized?", __func__)); 8891 clrbit(&sc->offload_map, pi->port_id); 8892 } 8893 8894 return (0); 8895 } 8896 8897 /* 8898 * Add an upper layer driver to the global list. 8899 */ 8900 int 8901 t4_register_uld(struct uld_info *ui) 8902 { 8903 int rc = 0; 8904 struct uld_info *u; 8905 8906 sx_xlock(&t4_uld_list_lock); 8907 SLIST_FOREACH(u, &t4_uld_list, link) { 8908 if (u->uld_id == ui->uld_id) { 8909 rc = EEXIST; 8910 goto done; 8911 } 8912 } 8913 8914 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 8915 ui->refcount = 0; 8916 done: 8917 sx_xunlock(&t4_uld_list_lock); 8918 return (rc); 8919 } 8920 8921 int 8922 t4_unregister_uld(struct uld_info *ui) 8923 { 8924 int rc = EINVAL; 8925 struct uld_info *u; 8926 8927 sx_xlock(&t4_uld_list_lock); 8928 8929 SLIST_FOREACH(u, &t4_uld_list, link) { 8930 if (u == ui) { 8931 if (ui->refcount > 0) { 8932 rc = EBUSY; 8933 goto done; 8934 } 8935 8936 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 8937 rc = 0; 8938 goto done; 8939 } 8940 } 8941 done: 8942 sx_xunlock(&t4_uld_list_lock); 8943 return (rc); 8944 } 8945 8946 int 8947 t4_activate_uld(struct adapter *sc, int id) 8948 { 8949 int rc; 8950 struct uld_info *ui; 8951 8952 ASSERT_SYNCHRONIZED_OP(sc); 8953 8954 if (id < 0 || id > ULD_MAX) 8955 return (EINVAL); 8956 rc = EAGAIN; /* kldoad the module with this ULD and try again. */ 8957 8958 sx_slock(&t4_uld_list_lock); 8959 8960 SLIST_FOREACH(ui, &t4_uld_list, link) { 8961 if (ui->uld_id == id) { 8962 if (!(sc->flags & FULL_INIT_DONE)) { 8963 rc = adapter_full_init(sc); 8964 if (rc != 0) 8965 break; 8966 } 8967 8968 rc = ui->activate(sc); 8969 if (rc == 0) { 8970 setbit(&sc->active_ulds, id); 8971 ui->refcount++; 8972 } 8973 break; 8974 } 8975 } 8976 8977 sx_sunlock(&t4_uld_list_lock); 8978 8979 return (rc); 8980 } 8981 8982 int 8983 t4_deactivate_uld(struct adapter *sc, int id) 8984 { 8985 int rc; 8986 struct uld_info *ui; 8987 8988 ASSERT_SYNCHRONIZED_OP(sc); 8989 8990 if (id < 0 || id > ULD_MAX) 8991 return (EINVAL); 8992 rc = ENXIO; 8993 8994 sx_slock(&t4_uld_list_lock); 8995 8996 SLIST_FOREACH(ui, &t4_uld_list, link) { 8997 if (ui->uld_id == id) { 8998 rc = ui->deactivate(sc); 8999 if (rc == 0) { 9000 clrbit(&sc->active_ulds, id); 9001 ui->refcount--; 9002 } 9003 break; 9004 } 9005 } 9006 9007 sx_sunlock(&t4_uld_list_lock); 9008 9009 return (rc); 9010 } 9011 9012 int 9013 uld_active(struct adapter *sc, int uld_id) 9014 { 9015 9016 MPASS(uld_id >= 0 && uld_id <= ULD_MAX); 9017 9018 return (isset(&sc->active_ulds, uld_id)); 9019 } 9020 #endif 9021 9022 /* 9023 * Come up with reasonable defaults for some of the tunables, provided they're 9024 * not set by the user (in which case we'll use the values as is). 9025 */ 9026 static void 9027 tweak_tunables(void) 9028 { 9029 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 9030 9031 if (t4_ntxq10g < 1) { 9032 #ifdef RSS 9033 t4_ntxq10g = rss_getnumbuckets(); 9034 #else 9035 t4_ntxq10g = min(nc, NTXQ_10G); 9036 #endif 9037 } 9038 9039 if (t4_ntxq1g < 1) { 9040 #ifdef RSS 9041 /* XXX: way too many for 1GbE? */ 9042 t4_ntxq1g = rss_getnumbuckets(); 9043 #else 9044 t4_ntxq1g = min(nc, NTXQ_1G); 9045 #endif 9046 } 9047 9048 if (t4_nrxq10g < 1) { 9049 #ifdef RSS 9050 t4_nrxq10g = rss_getnumbuckets(); 9051 #else 9052 t4_nrxq10g = min(nc, NRXQ_10G); 9053 #endif 9054 } 9055 9056 if (t4_nrxq1g < 1) { 9057 #ifdef RSS 9058 /* XXX: way too many for 1GbE? */ 9059 t4_nrxq1g = rss_getnumbuckets(); 9060 #else 9061 t4_nrxq1g = min(nc, NRXQ_1G); 9062 #endif 9063 } 9064 9065 #ifdef TCP_OFFLOAD 9066 if (t4_nofldtxq10g < 1) 9067 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 9068 9069 if (t4_nofldtxq1g < 1) 9070 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 9071 9072 if (t4_nofldrxq10g < 1) 9073 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 9074 9075 if (t4_nofldrxq1g < 1) 9076 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 9077 9078 if (t4_toecaps_allowed == -1) 9079 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 9080 #else 9081 if (t4_toecaps_allowed == -1) 9082 t4_toecaps_allowed = 0; 9083 #endif 9084 9085 #ifdef DEV_NETMAP 9086 if (t4_nnmtxq10g < 1) 9087 t4_nnmtxq10g = min(nc, NNMTXQ_10G); 9088 9089 if (t4_nnmtxq1g < 1) 9090 t4_nnmtxq1g = min(nc, NNMTXQ_1G); 9091 9092 if (t4_nnmrxq10g < 1) 9093 t4_nnmrxq10g = min(nc, NNMRXQ_10G); 9094 9095 if (t4_nnmrxq1g < 1) 9096 t4_nnmrxq1g = min(nc, NNMRXQ_1G); 9097 #endif 9098 9099 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 9100 t4_tmr_idx_10g = TMR_IDX_10G; 9101 9102 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 9103 t4_pktc_idx_10g = PKTC_IDX_10G; 9104 9105 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 9106 t4_tmr_idx_1g = TMR_IDX_1G; 9107 9108 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 9109 t4_pktc_idx_1g = PKTC_IDX_1G; 9110 9111 if (t4_qsize_txq < 128) 9112 t4_qsize_txq = 128; 9113 9114 if (t4_qsize_rxq < 128) 9115 t4_qsize_rxq = 128; 9116 while (t4_qsize_rxq & 7) 9117 t4_qsize_rxq++; 9118 9119 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 9120 } 9121 9122 static struct sx mlu; /* mod load unload */ 9123 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload"); 9124 9125 static int 9126 mod_event(module_t mod, int cmd, void *arg) 9127 { 9128 int rc = 0; 9129 static int loaded = 0; 9130 9131 switch (cmd) { 9132 case MOD_LOAD: 9133 sx_xlock(&mlu); 9134 if (loaded++ == 0) { 9135 t4_sge_modload(); 9136 sx_init(&t4_list_lock, "T4/T5 adapters"); 9137 SLIST_INIT(&t4_list); 9138 #ifdef TCP_OFFLOAD 9139 sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); 9140 SLIST_INIT(&t4_uld_list); 9141 #endif 9142 t4_tracer_modload(); 9143 tweak_tunables(); 9144 } 9145 sx_xunlock(&mlu); 9146 break; 9147 9148 case MOD_UNLOAD: 9149 sx_xlock(&mlu); 9150 if (--loaded == 0) { 9151 int tries; 9152 9153 sx_slock(&t4_list_lock); 9154 if (!SLIST_EMPTY(&t4_list)) { 9155 rc = EBUSY; 9156 sx_sunlock(&t4_list_lock); 9157 goto done_unload; 9158 } 9159 #ifdef TCP_OFFLOAD 9160 sx_slock(&t4_uld_list_lock); 9161 if (!SLIST_EMPTY(&t4_uld_list)) { 9162 rc = EBUSY; 9163 sx_sunlock(&t4_uld_list_lock); 9164 sx_sunlock(&t4_list_lock); 9165 goto done_unload; 9166 } 9167 #endif 9168 tries = 0; 9169 while (tries++ < 5 && t4_sge_extfree_refs() != 0) { 9170 uprintf("%ju clusters with custom free routine " 9171 "still is use.\n", t4_sge_extfree_refs()); 9172 pause("t4unload", 2 * hz); 9173 } 9174 #ifdef TCP_OFFLOAD 9175 sx_sunlock(&t4_uld_list_lock); 9176 #endif 9177 sx_sunlock(&t4_list_lock); 9178 9179 if (t4_sge_extfree_refs() == 0) { 9180 t4_tracer_modunload(); 9181 #ifdef TCP_OFFLOAD 9182 sx_destroy(&t4_uld_list_lock); 9183 #endif 9184 sx_destroy(&t4_list_lock); 9185 t4_sge_modunload(); 9186 loaded = 0; 9187 } else { 9188 rc = EBUSY; 9189 loaded++; /* undo earlier decrement */ 9190 } 9191 } 9192 done_unload: 9193 sx_xunlock(&mlu); 9194 break; 9195 } 9196 9197 return (rc); 9198 } 9199 9200 static devclass_t t4_devclass, t5_devclass; 9201 static devclass_t cxgbe_devclass, cxl_devclass; 9202 static devclass_t vcxgbe_devclass, vcxl_devclass; 9203 9204 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 9205 MODULE_VERSION(t4nex, 1); 9206 MODULE_DEPEND(t4nex, firmware, 1, 1, 1); 9207 #ifdef DEV_NETMAP 9208 MODULE_DEPEND(t4nex, netmap, 1, 1, 1); 9209 #endif /* DEV_NETMAP */ 9210 9211 9212 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 9213 MODULE_VERSION(t5nex, 1); 9214 MODULE_DEPEND(t5nex, firmware, 1, 1, 1); 9215 #ifdef DEV_NETMAP 9216 MODULE_DEPEND(t5nex, netmap, 1, 1, 1); 9217 #endif /* DEV_NETMAP */ 9218 9219 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 9220 MODULE_VERSION(cxgbe, 1); 9221 9222 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 9223 MODULE_VERSION(cxl, 1); 9224 9225 DRIVER_MODULE(vcxgbe, cxgbe, vcxgbe_driver, vcxgbe_devclass, 0, 0); 9226 MODULE_VERSION(vcxgbe, 1); 9227 9228 DRIVER_MODULE(vcxl, cxl, vcxl_driver, vcxl_devclass, 0, 0); 9229 MODULE_VERSION(vcxl, 1); 9230