1 /*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_inet.h" 32 #include "opt_inet6.h" 33 #include "opt_rss.h" 34 35 #include <sys/param.h> 36 #include <sys/conf.h> 37 #include <sys/priv.h> 38 #include <sys/kernel.h> 39 #include <sys/bus.h> 40 #include <sys/module.h> 41 #include <sys/malloc.h> 42 #include <sys/queue.h> 43 #include <sys/taskqueue.h> 44 #include <sys/pciio.h> 45 #include <dev/pci/pcireg.h> 46 #include <dev/pci/pcivar.h> 47 #include <dev/pci/pci_private.h> 48 #include <sys/firmware.h> 49 #include <sys/sbuf.h> 50 #include <sys/smp.h> 51 #include <sys/socket.h> 52 #include <sys/sockio.h> 53 #include <sys/sysctl.h> 54 #include <net/ethernet.h> 55 #include <net/if.h> 56 #include <net/if_types.h> 57 #include <net/if_dl.h> 58 #include <net/if_vlan_var.h> 59 #ifdef RSS 60 #include <net/rss_config.h> 61 #endif 62 #if defined(__i386__) || defined(__amd64__) 63 #include <vm/vm.h> 64 #include <vm/pmap.h> 65 #endif 66 67 #include "common/common.h" 68 #include "common/t4_msg.h" 69 #include "common/t4_regs.h" 70 #include "common/t4_regs_values.h" 71 #include "t4_ioctl.h" 72 #include "t4_l2t.h" 73 #include "t4_mp_ring.h" 74 75 /* T4 bus driver interface */ 76 static int t4_probe(device_t); 77 static int t4_attach(device_t); 78 static int t4_detach(device_t); 79 static device_method_t t4_methods[] = { 80 DEVMETHOD(device_probe, t4_probe), 81 DEVMETHOD(device_attach, t4_attach), 82 DEVMETHOD(device_detach, t4_detach), 83 84 DEVMETHOD_END 85 }; 86 static driver_t t4_driver = { 87 "t4nex", 88 t4_methods, 89 sizeof(struct adapter) 90 }; 91 92 93 /* T4 port (cxgbe) interface */ 94 static int cxgbe_probe(device_t); 95 static int cxgbe_attach(device_t); 96 static int cxgbe_detach(device_t); 97 static device_method_t cxgbe_methods[] = { 98 DEVMETHOD(device_probe, cxgbe_probe), 99 DEVMETHOD(device_attach, cxgbe_attach), 100 DEVMETHOD(device_detach, cxgbe_detach), 101 { 0, 0 } 102 }; 103 static driver_t cxgbe_driver = { 104 "cxgbe", 105 cxgbe_methods, 106 sizeof(struct port_info) 107 }; 108 109 /* T4 VI (vcxgbe) interface */ 110 static int vcxgbe_probe(device_t); 111 static int vcxgbe_attach(device_t); 112 static int vcxgbe_detach(device_t); 113 static device_method_t vcxgbe_methods[] = { 114 DEVMETHOD(device_probe, vcxgbe_probe), 115 DEVMETHOD(device_attach, vcxgbe_attach), 116 DEVMETHOD(device_detach, vcxgbe_detach), 117 { 0, 0 } 118 }; 119 static driver_t vcxgbe_driver = { 120 "vcxgbe", 121 vcxgbe_methods, 122 sizeof(struct vi_info) 123 }; 124 125 static d_ioctl_t t4_ioctl; 126 static d_open_t t4_open; 127 static d_close_t t4_close; 128 129 static struct cdevsw t4_cdevsw = { 130 .d_version = D_VERSION, 131 .d_flags = 0, 132 .d_open = t4_open, 133 .d_close = t4_close, 134 .d_ioctl = t4_ioctl, 135 .d_name = "t4nex", 136 }; 137 138 /* T5 bus driver interface */ 139 static int t5_probe(device_t); 140 static device_method_t t5_methods[] = { 141 DEVMETHOD(device_probe, t5_probe), 142 DEVMETHOD(device_attach, t4_attach), 143 DEVMETHOD(device_detach, t4_detach), 144 145 DEVMETHOD_END 146 }; 147 static driver_t t5_driver = { 148 "t5nex", 149 t5_methods, 150 sizeof(struct adapter) 151 }; 152 153 154 /* T5 port (cxl) interface */ 155 static driver_t cxl_driver = { 156 "cxl", 157 cxgbe_methods, 158 sizeof(struct port_info) 159 }; 160 161 /* T5 VI (vcxl) interface */ 162 static driver_t vcxl_driver = { 163 "vcxl", 164 vcxgbe_methods, 165 sizeof(struct vi_info) 166 }; 167 168 static struct cdevsw t5_cdevsw = { 169 .d_version = D_VERSION, 170 .d_flags = 0, 171 .d_open = t4_open, 172 .d_close = t4_close, 173 .d_ioctl = t4_ioctl, 174 .d_name = "t5nex", 175 }; 176 177 /* ifnet + media interface */ 178 static void cxgbe_init(void *); 179 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 180 static int cxgbe_transmit(struct ifnet *, struct mbuf *); 181 static void cxgbe_qflush(struct ifnet *); 182 static int cxgbe_media_change(struct ifnet *); 183 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 184 185 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 186 187 /* 188 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 189 * then ADAPTER_LOCK, then t4_uld_list_lock. 190 */ 191 static struct sx t4_list_lock; 192 SLIST_HEAD(, adapter) t4_list; 193 #ifdef TCP_OFFLOAD 194 static struct sx t4_uld_list_lock; 195 SLIST_HEAD(, uld_info) t4_uld_list; 196 #endif 197 198 /* 199 * Tunables. See tweak_tunables() too. 200 * 201 * Each tunable is set to a default value here if it's known at compile-time. 202 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 203 * provide a reasonable default when the driver is loaded. 204 * 205 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 206 * T5 are under hw.cxl. 207 */ 208 209 /* 210 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 211 */ 212 #define NTXQ_10G 16 213 static int t4_ntxq10g = -1; 214 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 215 216 #define NRXQ_10G 8 217 static int t4_nrxq10g = -1; 218 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 219 220 #define NTXQ_1G 4 221 static int t4_ntxq1g = -1; 222 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 223 224 #define NRXQ_1G 2 225 static int t4_nrxq1g = -1; 226 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 227 228 static int t4_rsrv_noflowq = 0; 229 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq); 230 231 #ifdef TCP_OFFLOAD 232 #define NOFLDTXQ_10G 8 233 static int t4_nofldtxq10g = -1; 234 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 235 236 #define NOFLDRXQ_10G 2 237 static int t4_nofldrxq10g = -1; 238 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 239 240 #define NOFLDTXQ_1G 2 241 static int t4_nofldtxq1g = -1; 242 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 243 244 #define NOFLDRXQ_1G 1 245 static int t4_nofldrxq1g = -1; 246 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 247 #endif 248 249 #ifdef DEV_NETMAP 250 #define NNMTXQ_10G 2 251 static int t4_nnmtxq10g = -1; 252 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g); 253 254 #define NNMRXQ_10G 2 255 static int t4_nnmrxq10g = -1; 256 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g); 257 258 #define NNMTXQ_1G 1 259 static int t4_nnmtxq1g = -1; 260 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g); 261 262 #define NNMRXQ_1G 1 263 static int t4_nnmrxq1g = -1; 264 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g); 265 #endif 266 267 /* 268 * Holdoff parameters for 10G and 1G ports. 269 */ 270 #define TMR_IDX_10G 1 271 static int t4_tmr_idx_10g = TMR_IDX_10G; 272 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 273 274 #define PKTC_IDX_10G (-1) 275 static int t4_pktc_idx_10g = PKTC_IDX_10G; 276 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 277 278 #define TMR_IDX_1G 1 279 static int t4_tmr_idx_1g = TMR_IDX_1G; 280 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 281 282 #define PKTC_IDX_1G (-1) 283 static int t4_pktc_idx_1g = PKTC_IDX_1G; 284 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 285 286 /* 287 * Size (# of entries) of each tx and rx queue. 288 */ 289 static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 290 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 291 292 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 293 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 294 295 /* 296 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 297 */ 298 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 299 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 300 301 /* 302 * Configuration file. 303 */ 304 #define DEFAULT_CF "default" 305 #define FLASH_CF "flash" 306 #define UWIRE_CF "uwire" 307 #define FPGA_CF "fpga" 308 static char t4_cfg_file[32] = DEFAULT_CF; 309 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 310 311 /* 312 * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively). 313 * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them. 314 * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water 315 * mark or when signalled to do so, 0 to never emit PAUSE. 316 */ 317 static int t4_pause_settings = PAUSE_TX | PAUSE_RX; 318 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings); 319 320 /* 321 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 322 * encouraged respectively). 323 */ 324 static unsigned int t4_fw_install = 1; 325 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 326 327 /* 328 * ASIC features that will be used. Disable the ones you don't want so that the 329 * chip resources aren't wasted on features that will not be used. 330 */ 331 static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 332 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 333 334 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 335 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 336 337 static int t4_toecaps_allowed = -1; 338 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 339 340 static int t4_rdmacaps_allowed = 0; 341 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 342 343 static int t4_iscsicaps_allowed = 0; 344 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 345 346 static int t4_fcoecaps_allowed = 0; 347 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 348 349 static int t5_write_combine = 0; 350 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 351 352 static int t4_num_vis = 1; 353 TUNABLE_INT("hw.cxgbe.num_vis", &t4_num_vis); 354 355 /* Functions used by extra VIs to obtain unique MAC addresses for each VI. */ 356 static int vi_mac_funcs[] = { 357 FW_VI_FUNC_OFLD, 358 FW_VI_FUNC_IWARP, 359 FW_VI_FUNC_OPENISCSI, 360 FW_VI_FUNC_OPENFCOE, 361 FW_VI_FUNC_FOISCSI, 362 FW_VI_FUNC_FOFCOE, 363 }; 364 365 struct intrs_and_queues { 366 uint16_t intr_type; /* INTx, MSI, or MSI-X */ 367 uint16_t nirq; /* Total # of vectors */ 368 uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */ 369 uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */ 370 uint16_t ntxq10g; /* # of NIC txq's for each 10G port */ 371 uint16_t nrxq10g; /* # of NIC rxq's for each 10G port */ 372 uint16_t ntxq1g; /* # of NIC txq's for each 1G port */ 373 uint16_t nrxq1g; /* # of NIC rxq's for each 1G port */ 374 uint16_t rsrv_noflowq; /* Flag whether to reserve queue 0 */ 375 #ifdef TCP_OFFLOAD 376 uint16_t nofldtxq10g; /* # of TOE txq's for each 10G port */ 377 uint16_t nofldrxq10g; /* # of TOE rxq's for each 10G port */ 378 uint16_t nofldtxq1g; /* # of TOE txq's for each 1G port */ 379 uint16_t nofldrxq1g; /* # of TOE rxq's for each 1G port */ 380 #endif 381 #ifdef DEV_NETMAP 382 uint16_t nnmtxq10g; /* # of netmap txq's for each 10G port */ 383 uint16_t nnmrxq10g; /* # of netmap rxq's for each 10G port */ 384 uint16_t nnmtxq1g; /* # of netmap txq's for each 1G port */ 385 uint16_t nnmrxq1g; /* # of netmap rxq's for each 1G port */ 386 #endif 387 }; 388 389 struct filter_entry { 390 uint32_t valid:1; /* filter allocated and valid */ 391 uint32_t locked:1; /* filter is administratively locked */ 392 uint32_t pending:1; /* filter action is pending firmware reply */ 393 uint32_t smtidx:8; /* Source MAC Table index for smac */ 394 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 395 396 struct t4_filter_specification fs; 397 }; 398 399 static int map_bars_0_and_4(struct adapter *); 400 static int map_bar_2(struct adapter *); 401 static void setup_memwin(struct adapter *); 402 static int validate_mem_range(struct adapter *, uint32_t, int); 403 static int fwmtype_to_hwmtype(int); 404 static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 405 uint32_t *); 406 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 407 static uint32_t position_memwin(struct adapter *, int, uint32_t); 408 static int cfg_itype_and_nqueues(struct adapter *, int, int, int, 409 struct intrs_and_queues *); 410 static int prep_firmware(struct adapter *); 411 static int partition_resources(struct adapter *, const struct firmware *, 412 const char *); 413 static int get_params__pre_init(struct adapter *); 414 static int get_params__post_init(struct adapter *); 415 static int set_params__post_init(struct adapter *); 416 static void t4_set_desc(struct adapter *); 417 static void build_medialist(struct port_info *, struct ifmedia *); 418 static int cxgbe_init_synchronized(struct vi_info *); 419 static int cxgbe_uninit_synchronized(struct vi_info *); 420 static int setup_intr_handlers(struct adapter *); 421 static void quiesce_txq(struct adapter *, struct sge_txq *); 422 static void quiesce_wrq(struct adapter *, struct sge_wrq *); 423 static void quiesce_iq(struct adapter *, struct sge_iq *); 424 static void quiesce_fl(struct adapter *, struct sge_fl *); 425 static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 426 driver_intr_t *, void *, char *); 427 static int t4_free_irq(struct adapter *, struct irq *); 428 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 429 unsigned int); 430 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 431 static void vi_refresh_stats(struct adapter *, struct vi_info *); 432 static void cxgbe_refresh_stats(struct adapter *, struct port_info *); 433 static void cxgbe_tick(void *); 434 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 435 static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 436 struct mbuf *); 437 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 438 static int fw_msg_not_handled(struct adapter *, const __be64 *); 439 static void t4_sysctls(struct adapter *); 440 static void cxgbe_sysctls(struct port_info *); 441 static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 442 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 443 static int sysctl_btphy(SYSCTL_HANDLER_ARGS); 444 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); 445 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 446 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 447 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 448 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 449 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS); 450 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 451 static int sysctl_temperature(SYSCTL_HANDLER_ARGS); 452 #ifdef SBUF_DRAIN 453 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 454 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 455 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 456 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); 457 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); 458 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 459 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 460 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 461 static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 462 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 463 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 464 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 465 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); 466 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 467 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); 468 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 469 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 470 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 471 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 472 static int sysctl_tids(SYSCTL_HANDLER_ARGS); 473 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 474 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 475 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 476 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 477 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 478 #endif 479 static uint32_t fconf_to_mode(uint32_t); 480 static uint32_t mode_to_fconf(uint32_t); 481 static uint32_t fspec_to_fconf(struct t4_filter_specification *); 482 static int get_filter_mode(struct adapter *, uint32_t *); 483 static int set_filter_mode(struct adapter *, uint32_t); 484 static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 485 static int get_filter(struct adapter *, struct t4_filter *); 486 static int set_filter(struct adapter *, struct t4_filter *); 487 static int del_filter(struct adapter *, struct t4_filter *); 488 static void clear_filter(struct filter_entry *); 489 static int set_filter_wr(struct adapter *, int); 490 static int del_filter_wr(struct adapter *, int); 491 static int get_sge_context(struct adapter *, struct t4_sge_context *); 492 static int load_fw(struct adapter *, struct t4_data *); 493 static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 494 static int read_i2c(struct adapter *, struct t4_i2c_data *); 495 static int set_sched_class(struct adapter *, struct t4_sched_params *); 496 static int set_sched_queue(struct adapter *, struct t4_sched_queue *); 497 #ifdef TCP_OFFLOAD 498 static int toe_capability(struct vi_info *, int); 499 #endif 500 static int mod_event(module_t, int, void *); 501 502 struct { 503 uint16_t device; 504 char *desc; 505 } t4_pciids[] = { 506 {0xa000, "Chelsio Terminator 4 FPGA"}, 507 {0x4400, "Chelsio T440-dbg"}, 508 {0x4401, "Chelsio T420-CR"}, 509 {0x4402, "Chelsio T422-CR"}, 510 {0x4403, "Chelsio T440-CR"}, 511 {0x4404, "Chelsio T420-BCH"}, 512 {0x4405, "Chelsio T440-BCH"}, 513 {0x4406, "Chelsio T440-CH"}, 514 {0x4407, "Chelsio T420-SO"}, 515 {0x4408, "Chelsio T420-CX"}, 516 {0x4409, "Chelsio T420-BT"}, 517 {0x440a, "Chelsio T404-BT"}, 518 {0x440e, "Chelsio T440-LP-CR"}, 519 }, t5_pciids[] = { 520 {0xb000, "Chelsio Terminator 5 FPGA"}, 521 {0x5400, "Chelsio T580-dbg"}, 522 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ 523 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ 524 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ 525 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ 526 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ 527 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ 528 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ 529 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ 530 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 531 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ 532 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ 533 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ 534 {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */ 535 #ifdef notyet 536 {0x5404, "Chelsio T520-BCH"}, 537 {0x5405, "Chelsio T540-BCH"}, 538 {0x5406, "Chelsio T540-CH"}, 539 {0x5408, "Chelsio T520-CX"}, 540 {0x540b, "Chelsio B520-SR"}, 541 {0x540c, "Chelsio B504-BT"}, 542 {0x540f, "Chelsio Amsterdam"}, 543 {0x5413, "Chelsio T580-CHR"}, 544 #endif 545 }; 546 547 #ifdef TCP_OFFLOAD 548 /* 549 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 550 * exactly the same for both rxq and ofld_rxq. 551 */ 552 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 553 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 554 #endif 555 556 /* No easy way to include t4_msg.h before adapter.h so we check this way */ 557 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 558 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 559 560 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); 561 562 static int 563 t4_probe(device_t dev) 564 { 565 int i; 566 uint16_t v = pci_get_vendor(dev); 567 uint16_t d = pci_get_device(dev); 568 uint8_t f = pci_get_function(dev); 569 570 if (v != PCI_VENDOR_ID_CHELSIO) 571 return (ENXIO); 572 573 /* Attach only to PF0 of the FPGA */ 574 if (d == 0xa000 && f != 0) 575 return (ENXIO); 576 577 for (i = 0; i < nitems(t4_pciids); i++) { 578 if (d == t4_pciids[i].device) { 579 device_set_desc(dev, t4_pciids[i].desc); 580 return (BUS_PROBE_DEFAULT); 581 } 582 } 583 584 return (ENXIO); 585 } 586 587 static int 588 t5_probe(device_t dev) 589 { 590 int i; 591 uint16_t v = pci_get_vendor(dev); 592 uint16_t d = pci_get_device(dev); 593 uint8_t f = pci_get_function(dev); 594 595 if (v != PCI_VENDOR_ID_CHELSIO) 596 return (ENXIO); 597 598 /* Attach only to PF0 of the FPGA */ 599 if (d == 0xb000 && f != 0) 600 return (ENXIO); 601 602 for (i = 0; i < nitems(t5_pciids); i++) { 603 if (d == t5_pciids[i].device) { 604 device_set_desc(dev, t5_pciids[i].desc); 605 return (BUS_PROBE_DEFAULT); 606 } 607 } 608 609 return (ENXIO); 610 } 611 612 static void 613 t5_attribute_workaround(device_t dev) 614 { 615 device_t root_port; 616 uint32_t v; 617 618 /* 619 * The T5 chips do not properly echo the No Snoop and Relaxed 620 * Ordering attributes when replying to a TLP from a Root 621 * Port. As a workaround, find the parent Root Port and 622 * disable No Snoop and Relaxed Ordering. Note that this 623 * affects all devices under this root port. 624 */ 625 root_port = pci_find_pcie_root_port(dev); 626 if (root_port == NULL) { 627 device_printf(dev, "Unable to find parent root port\n"); 628 return; 629 } 630 631 v = pcie_adjust_config(root_port, PCIER_DEVICE_CTL, 632 PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE, 0, 2); 633 if ((v & (PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE)) != 634 0) 635 device_printf(dev, "Disabled No Snoop/Relaxed Ordering on %s\n", 636 device_get_nameunit(root_port)); 637 } 638 639 static int 640 t4_attach(device_t dev) 641 { 642 struct adapter *sc; 643 int rc = 0, i, j, n10g, n1g, rqidx, tqidx; 644 struct intrs_and_queues iaq; 645 struct sge *s; 646 #ifdef TCP_OFFLOAD 647 int ofld_rqidx, ofld_tqidx; 648 #endif 649 #ifdef DEV_NETMAP 650 int nm_rqidx, nm_tqidx; 651 #endif 652 int num_vis; 653 654 sc = device_get_softc(dev); 655 sc->dev = dev; 656 TUNABLE_INT_FETCH("hw.cxgbe.debug_flags", &sc->debug_flags); 657 658 if ((pci_get_device(dev) & 0xff00) == 0x5400) 659 t5_attribute_workaround(dev); 660 pci_enable_busmaster(dev); 661 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 662 uint32_t v; 663 664 pci_set_max_read_req(dev, 4096); 665 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 666 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 667 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 668 669 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5); 670 } 671 672 sc->traceq = -1; 673 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); 674 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", 675 device_get_nameunit(dev)); 676 677 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 678 device_get_nameunit(dev)); 679 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 680 sx_xlock(&t4_list_lock); 681 SLIST_INSERT_HEAD(&t4_list, sc, link); 682 sx_xunlock(&t4_list_lock); 683 684 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 685 TAILQ_INIT(&sc->sfl); 686 callout_init_mtx(&sc->sfl_callout, &sc->sfl_lock, 0); 687 688 mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF); 689 690 rc = map_bars_0_and_4(sc); 691 if (rc != 0) 692 goto done; /* error message displayed already */ 693 694 /* 695 * This is the real PF# to which we're attaching. Works from within PCI 696 * passthrough environments too, where pci_get_function() could return a 697 * different PF# depending on the passthrough configuration. We need to 698 * use the real PF# in all our communication with the firmware. 699 */ 700 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 701 sc->mbox = sc->pf; 702 703 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 704 sc->an_handler = an_not_handled; 705 for (i = 0; i < nitems(sc->cpl_handler); i++) 706 sc->cpl_handler[i] = cpl_not_handled; 707 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 708 sc->fw_msg_handler[i] = fw_msg_not_handled; 709 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 710 t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt); 711 t4_register_cpl_handler(sc, CPL_T5_TRACE_PKT, t5_trace_pkt); 712 t4_init_sge_cpl_handlers(sc); 713 714 /* Prepare the adapter for operation */ 715 rc = -t4_prep_adapter(sc); 716 if (rc != 0) { 717 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 718 goto done; 719 } 720 721 /* 722 * Do this really early, with the memory windows set up even before the 723 * character device. The userland tool's register i/o and mem read 724 * will work even in "recovery mode". 725 */ 726 setup_memwin(sc); 727 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 728 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 729 device_get_nameunit(dev)); 730 if (sc->cdev == NULL) 731 device_printf(dev, "failed to create nexus char device.\n"); 732 else 733 sc->cdev->si_drv1 = sc; 734 735 /* Go no further if recovery mode has been requested. */ 736 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 737 device_printf(dev, "recovery mode.\n"); 738 goto done; 739 } 740 741 #if defined(__i386__) 742 if ((cpu_feature & CPUID_CX8) == 0) { 743 device_printf(dev, "64 bit atomics not available.\n"); 744 rc = ENOTSUP; 745 goto done; 746 } 747 #endif 748 749 /* Prepare the firmware for operation */ 750 rc = prep_firmware(sc); 751 if (rc != 0) 752 goto done; /* error message displayed already */ 753 754 rc = get_params__post_init(sc); 755 if (rc != 0) 756 goto done; /* error message displayed already */ 757 758 rc = set_params__post_init(sc); 759 if (rc != 0) 760 goto done; /* error message displayed already */ 761 762 rc = map_bar_2(sc); 763 if (rc != 0) 764 goto done; /* error message displayed already */ 765 766 rc = t4_create_dma_tag(sc); 767 if (rc != 0) 768 goto done; /* error message displayed already */ 769 770 /* 771 * Number of VIs to create per-port. The first VI is the 772 * "main" regular VI for the port. The second VI is used for 773 * netmap if present, and any remaining VIs are used for 774 * additional virtual interfaces. 775 * 776 * Limit the number of VIs per port to the number of available 777 * MAC addresses per port. 778 */ 779 if (t4_num_vis >= 1) 780 num_vis = t4_num_vis; 781 else 782 num_vis = 1; 783 #ifdef DEV_NETMAP 784 num_vis++; 785 #endif 786 if (num_vis > nitems(vi_mac_funcs)) { 787 num_vis = nitems(vi_mac_funcs); 788 device_printf(dev, "Number of VIs limited to %d\n", num_vis); 789 } 790 791 /* 792 * First pass over all the ports - allocate VIs and initialize some 793 * basic parameters like mac address, port type, etc. We also figure 794 * out whether a port is 10G or 1G and use that information when 795 * calculating how many interrupts to attempt to allocate. 796 */ 797 n10g = n1g = 0; 798 for_each_port(sc, i) { 799 struct port_info *pi; 800 struct vi_info *vi; 801 802 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 803 sc->port[i] = pi; 804 805 /* These must be set before t4_port_init */ 806 pi->adapter = sc; 807 pi->port_id = i; 808 pi->nvi = num_vis; 809 pi->vi = malloc(sizeof(struct vi_info) * num_vis, M_CXGBE, 810 M_ZERO | M_WAITOK); 811 812 /* 813 * Allocate the "main" VI and initialize parameters 814 * like mac addr. 815 */ 816 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 817 if (rc != 0) { 818 device_printf(dev, "unable to initialize port %d: %d\n", 819 i, rc); 820 free(pi->vi, M_CXGBE); 821 free(pi, M_CXGBE); 822 sc->port[i] = NULL; 823 goto done; 824 } 825 826 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX); 827 pi->link_cfg.requested_fc |= t4_pause_settings; 828 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX); 829 pi->link_cfg.fc |= t4_pause_settings; 830 831 rc = -t4_link_l1cfg(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 832 if (rc != 0) { 833 device_printf(dev, "port %d l1cfg failed: %d\n", i, rc); 834 free(pi->vi, M_CXGBE); 835 free(pi, M_CXGBE); 836 sc->port[i] = NULL; 837 goto done; 838 } 839 840 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 841 device_get_nameunit(dev), i); 842 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 843 sc->chan_map[pi->tx_chan] = i; 844 845 if (is_10G_port(pi) || is_40G_port(pi)) { 846 n10g++; 847 for_each_vi(pi, j, vi) { 848 vi->tmr_idx = t4_tmr_idx_10g; 849 vi->pktc_idx = t4_pktc_idx_10g; 850 } 851 } else { 852 n1g++; 853 for_each_vi(pi, j, vi) { 854 vi->tmr_idx = t4_tmr_idx_1g; 855 vi->pktc_idx = t4_pktc_idx_1g; 856 } 857 } 858 859 pi->linkdnrc = -1; 860 861 for_each_vi(pi, j, vi) { 862 vi->qsize_rxq = t4_qsize_rxq; 863 vi->qsize_txq = t4_qsize_txq; 864 vi->pi = pi; 865 } 866 867 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 868 if (pi->dev == NULL) { 869 device_printf(dev, 870 "failed to add device for port %d.\n", i); 871 rc = ENXIO; 872 goto done; 873 } 874 pi->vi[0].dev = pi->dev; 875 device_set_softc(pi->dev, pi); 876 } 877 878 /* 879 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 880 */ 881 #ifdef DEV_NETMAP 882 num_vis--; 883 #endif 884 rc = cfg_itype_and_nqueues(sc, n10g, n1g, num_vis, &iaq); 885 if (rc != 0) 886 goto done; /* error message displayed already */ 887 888 sc->intr_type = iaq.intr_type; 889 sc->intr_count = iaq.nirq; 890 891 s = &sc->sge; 892 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 893 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 894 if (num_vis > 1) { 895 s->nrxq += (n10g + n1g) * (num_vis - 1); 896 s->ntxq += (n10g + n1g) * (num_vis - 1); 897 } 898 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 899 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 900 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 901 #ifdef TCP_OFFLOAD 902 if (is_offload(sc)) { 903 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 904 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 905 if (num_vis > 1) { 906 s->nofldrxq += (n10g + n1g) * (num_vis - 1); 907 s->nofldtxq += (n10g + n1g) * (num_vis - 1); 908 } 909 s->neq += s->nofldtxq + s->nofldrxq; 910 s->niq += s->nofldrxq; 911 912 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 913 M_CXGBE, M_ZERO | M_WAITOK); 914 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 915 M_CXGBE, M_ZERO | M_WAITOK); 916 } 917 #endif 918 #ifdef DEV_NETMAP 919 s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g; 920 s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g; 921 s->neq += s->nnmtxq + s->nnmrxq; 922 s->niq += s->nnmrxq; 923 924 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq), 925 M_CXGBE, M_ZERO | M_WAITOK); 926 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq), 927 M_CXGBE, M_ZERO | M_WAITOK); 928 #endif 929 930 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 931 M_ZERO | M_WAITOK); 932 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 933 M_ZERO | M_WAITOK); 934 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 935 M_ZERO | M_WAITOK); 936 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 937 M_ZERO | M_WAITOK); 938 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 939 M_ZERO | M_WAITOK); 940 941 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 942 M_ZERO | M_WAITOK); 943 944 t4_init_l2t(sc, M_WAITOK); 945 946 /* 947 * Second pass over the ports. This time we know the number of rx and 948 * tx queues that each port should get. 949 */ 950 rqidx = tqidx = 0; 951 #ifdef TCP_OFFLOAD 952 ofld_rqidx = ofld_tqidx = 0; 953 #endif 954 #ifdef DEV_NETMAP 955 nm_rqidx = nm_tqidx = 0; 956 #endif 957 for_each_port(sc, i) { 958 struct port_info *pi = sc->port[i]; 959 struct vi_info *vi; 960 961 if (pi == NULL) 962 continue; 963 964 for_each_vi(pi, j, vi) { 965 #ifdef DEV_NETMAP 966 if (j == 1) { 967 vi->flags |= VI_NETMAP | INTR_RXQ; 968 vi->first_rxq = nm_rqidx; 969 vi->first_txq = nm_tqidx; 970 if (is_10G_port(pi) || is_40G_port(pi)) { 971 vi->nrxq = iaq.nnmrxq10g; 972 vi->ntxq = iaq.nnmtxq10g; 973 } else { 974 vi->nrxq = iaq.nnmrxq1g; 975 vi->ntxq = iaq.nnmtxq1g; 976 } 977 nm_rqidx += vi->nrxq; 978 nm_tqidx += vi->ntxq; 979 continue; 980 } 981 #endif 982 983 vi->first_rxq = rqidx; 984 vi->first_txq = tqidx; 985 if (is_10G_port(pi) || is_40G_port(pi)) { 986 vi->flags |= iaq.intr_flags_10g & INTR_RXQ; 987 vi->nrxq = j == 0 ? iaq.nrxq10g : 1; 988 vi->ntxq = j == 0 ? iaq.ntxq10g : 1; 989 } else { 990 vi->flags |= iaq.intr_flags_1g & INTR_RXQ; 991 vi->nrxq = j == 0 ? iaq.nrxq1g : 1; 992 vi->ntxq = j == 0 ? iaq.ntxq1g : 1; 993 } 994 995 if (vi->ntxq > 1) 996 vi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0; 997 else 998 vi->rsrv_noflowq = 0; 999 1000 rqidx += vi->nrxq; 1001 tqidx += vi->ntxq; 1002 1003 #ifdef TCP_OFFLOAD 1004 if (!is_offload(sc)) 1005 continue; 1006 vi->first_ofld_rxq = ofld_rqidx; 1007 vi->first_ofld_txq = ofld_tqidx; 1008 if (is_10G_port(pi) || is_40G_port(pi)) { 1009 vi->flags |= iaq.intr_flags_10g & INTR_OFLD_RXQ; 1010 vi->nofldrxq = j == 0 ? iaq.nofldrxq10g : 1; 1011 vi->nofldtxq = j == 0 ? iaq.nofldtxq10g : 1; 1012 } else { 1013 vi->flags |= iaq.intr_flags_1g & INTR_OFLD_RXQ; 1014 vi->nofldrxq = j == 0 ? iaq.nofldrxq1g : 1; 1015 vi->nofldtxq = j == 0 ? iaq.nofldtxq1g : 1; 1016 } 1017 ofld_rqidx += vi->nofldrxq; 1018 ofld_tqidx += vi->nofldtxq; 1019 #endif 1020 } 1021 } 1022 1023 rc = setup_intr_handlers(sc); 1024 if (rc != 0) { 1025 device_printf(dev, 1026 "failed to setup interrupt handlers: %d\n", rc); 1027 goto done; 1028 } 1029 1030 rc = bus_generic_attach(dev); 1031 if (rc != 0) { 1032 device_printf(dev, 1033 "failed to attach all child ports: %d\n", rc); 1034 goto done; 1035 } 1036 1037 device_printf(dev, 1038 "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 1039 sc->params.pci.speed, sc->params.pci.width, sc->params.nports, 1040 sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" : 1041 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 1042 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 1043 1044 t4_set_desc(sc); 1045 1046 done: 1047 if (rc != 0 && sc->cdev) { 1048 /* cdev was created and so cxgbetool works; recover that way. */ 1049 device_printf(dev, 1050 "error during attach, adapter is now in recovery mode.\n"); 1051 rc = 0; 1052 } 1053 1054 if (rc != 0) 1055 t4_detach(dev); 1056 else 1057 t4_sysctls(sc); 1058 1059 return (rc); 1060 } 1061 1062 /* 1063 * Idempotent 1064 */ 1065 static int 1066 t4_detach(device_t dev) 1067 { 1068 struct adapter *sc; 1069 struct port_info *pi; 1070 int i, rc; 1071 1072 sc = device_get_softc(dev); 1073 1074 if (sc->flags & FULL_INIT_DONE) 1075 t4_intr_disable(sc); 1076 1077 if (sc->cdev) { 1078 destroy_dev(sc->cdev); 1079 sc->cdev = NULL; 1080 } 1081 1082 rc = bus_generic_detach(dev); 1083 if (rc) { 1084 device_printf(dev, 1085 "failed to detach child devices: %d\n", rc); 1086 return (rc); 1087 } 1088 1089 for (i = 0; i < sc->intr_count; i++) 1090 t4_free_irq(sc, &sc->irq[i]); 1091 1092 for (i = 0; i < MAX_NPORTS; i++) { 1093 pi = sc->port[i]; 1094 if (pi) { 1095 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->vi[0].viid); 1096 if (pi->dev) 1097 device_delete_child(dev, pi->dev); 1098 1099 mtx_destroy(&pi->pi_lock); 1100 free(pi->vi, M_CXGBE); 1101 free(pi, M_CXGBE); 1102 } 1103 } 1104 1105 if (sc->flags & FULL_INIT_DONE) 1106 adapter_full_uninit(sc); 1107 1108 if (sc->flags & FW_OK) 1109 t4_fw_bye(sc, sc->mbox); 1110 1111 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 1112 pci_release_msi(dev); 1113 1114 if (sc->regs_res) 1115 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 1116 sc->regs_res); 1117 1118 if (sc->udbs_res) 1119 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 1120 sc->udbs_res); 1121 1122 if (sc->msix_res) 1123 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 1124 sc->msix_res); 1125 1126 if (sc->l2t) 1127 t4_free_l2t(sc->l2t); 1128 1129 #ifdef TCP_OFFLOAD 1130 free(sc->sge.ofld_rxq, M_CXGBE); 1131 free(sc->sge.ofld_txq, M_CXGBE); 1132 #endif 1133 #ifdef DEV_NETMAP 1134 free(sc->sge.nm_rxq, M_CXGBE); 1135 free(sc->sge.nm_txq, M_CXGBE); 1136 #endif 1137 free(sc->irq, M_CXGBE); 1138 free(sc->sge.rxq, M_CXGBE); 1139 free(sc->sge.txq, M_CXGBE); 1140 free(sc->sge.ctrlq, M_CXGBE); 1141 free(sc->sge.iqmap, M_CXGBE); 1142 free(sc->sge.eqmap, M_CXGBE); 1143 free(sc->tids.ftid_tab, M_CXGBE); 1144 t4_destroy_dma_tag(sc); 1145 if (mtx_initialized(&sc->sc_lock)) { 1146 sx_xlock(&t4_list_lock); 1147 SLIST_REMOVE(&t4_list, sc, adapter, link); 1148 sx_xunlock(&t4_list_lock); 1149 mtx_destroy(&sc->sc_lock); 1150 } 1151 1152 callout_drain(&sc->sfl_callout); 1153 if (mtx_initialized(&sc->tids.ftid_lock)) 1154 mtx_destroy(&sc->tids.ftid_lock); 1155 if (mtx_initialized(&sc->sfl_lock)) 1156 mtx_destroy(&sc->sfl_lock); 1157 if (mtx_initialized(&sc->ifp_lock)) 1158 mtx_destroy(&sc->ifp_lock); 1159 if (mtx_initialized(&sc->regwin_lock)) 1160 mtx_destroy(&sc->regwin_lock); 1161 1162 bzero(sc, sizeof(*sc)); 1163 1164 return (0); 1165 } 1166 1167 static int 1168 cxgbe_probe(device_t dev) 1169 { 1170 char buf[128]; 1171 struct port_info *pi = device_get_softc(dev); 1172 1173 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 1174 device_set_desc_copy(dev, buf); 1175 1176 return (BUS_PROBE_DEFAULT); 1177 } 1178 1179 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 1180 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 1181 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS) 1182 #define T4_CAP_ENABLE (T4_CAP) 1183 1184 static int 1185 cxgbe_vi_attach(device_t dev, struct vi_info *vi) 1186 { 1187 struct ifnet *ifp; 1188 struct sbuf *sb; 1189 1190 vi->xact_addr_filt = -1; 1191 callout_init(&vi->tick, 1); 1192 1193 /* Allocate an ifnet and set it up */ 1194 ifp = if_alloc(IFT_ETHER); 1195 if (ifp == NULL) { 1196 device_printf(dev, "Cannot allocate ifnet\n"); 1197 return (ENOMEM); 1198 } 1199 vi->ifp = ifp; 1200 ifp->if_softc = vi; 1201 1202 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1203 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1204 1205 ifp->if_init = cxgbe_init; 1206 ifp->if_ioctl = cxgbe_ioctl; 1207 ifp->if_transmit = cxgbe_transmit; 1208 ifp->if_qflush = cxgbe_qflush; 1209 ifp->if_get_counter = cxgbe_get_counter; 1210 1211 ifp->if_capabilities = T4_CAP; 1212 #ifdef TCP_OFFLOAD 1213 if (vi->nofldrxq != 0) 1214 ifp->if_capabilities |= IFCAP_TOE; 1215 #endif 1216 ifp->if_capenable = T4_CAP_ENABLE; 1217 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1218 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1219 1220 ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1221 ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS; 1222 ifp->if_hw_tsomaxsegsize = 65536; 1223 1224 /* Initialize ifmedia for this VI */ 1225 ifmedia_init(&vi->media, IFM_IMASK, cxgbe_media_change, 1226 cxgbe_media_status); 1227 build_medialist(vi->pi, &vi->media); 1228 1229 vi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 1230 EVENTHANDLER_PRI_ANY); 1231 1232 ether_ifattach(ifp, vi->hw_addr); 1233 1234 sb = sbuf_new_auto(); 1235 sbuf_printf(sb, "%d txq, %d rxq (NIC)", vi->ntxq, vi->nrxq); 1236 #ifdef TCP_OFFLOAD 1237 if (ifp->if_capabilities & IFCAP_TOE) 1238 sbuf_printf(sb, "; %d txq, %d rxq (TOE)", 1239 vi->nofldtxq, vi->nofldrxq); 1240 #endif 1241 sbuf_finish(sb); 1242 device_printf(dev, "%s\n", sbuf_data(sb)); 1243 sbuf_delete(sb); 1244 1245 vi_sysctls(vi); 1246 1247 return (0); 1248 } 1249 1250 static int 1251 cxgbe_attach(device_t dev) 1252 { 1253 struct port_info *pi = device_get_softc(dev); 1254 struct vi_info *vi; 1255 int i, rc; 1256 1257 callout_init_mtx(&pi->tick, &pi->pi_lock, 0); 1258 1259 rc = cxgbe_vi_attach(dev, &pi->vi[0]); 1260 if (rc) 1261 return (rc); 1262 1263 for_each_vi(pi, i, vi) { 1264 if (i == 0) 1265 continue; 1266 #ifdef DEV_NETMAP 1267 if (vi->flags & VI_NETMAP) { 1268 /* 1269 * media handled here to keep 1270 * implementation private to this file 1271 */ 1272 ifmedia_init(&vi->media, IFM_IMASK, cxgbe_media_change, 1273 cxgbe_media_status); 1274 build_medialist(pi, &vi->media); 1275 vi->dev = device_add_child(dev, is_t4(pi->adapter) ? 1276 "ncxgbe" : "ncxl", device_get_unit(dev)); 1277 } else 1278 #endif 1279 vi->dev = device_add_child(dev, is_t4(pi->adapter) ? 1280 "vcxgbe" : "vcxl", -1); 1281 if (vi->dev == NULL) { 1282 device_printf(dev, "failed to add VI %d\n", i); 1283 continue; 1284 } 1285 device_set_softc(vi->dev, vi); 1286 } 1287 1288 cxgbe_sysctls(pi); 1289 1290 bus_generic_attach(dev); 1291 1292 return (0); 1293 } 1294 1295 static void 1296 cxgbe_vi_detach(struct vi_info *vi) 1297 { 1298 struct ifnet *ifp = vi->ifp; 1299 1300 ether_ifdetach(ifp); 1301 1302 if (vi->vlan_c) 1303 EVENTHANDLER_DEREGISTER(vlan_config, vi->vlan_c); 1304 1305 /* Let detach proceed even if these fail. */ 1306 cxgbe_uninit_synchronized(vi); 1307 callout_drain(&vi->tick); 1308 vi_full_uninit(vi); 1309 1310 ifmedia_removeall(&vi->media); 1311 if_free(vi->ifp); 1312 vi->ifp = NULL; 1313 } 1314 1315 static int 1316 cxgbe_detach(device_t dev) 1317 { 1318 struct port_info *pi = device_get_softc(dev); 1319 struct adapter *sc = pi->adapter; 1320 int rc; 1321 1322 /* Detach the extra VIs first. */ 1323 rc = bus_generic_detach(dev); 1324 if (rc) 1325 return (rc); 1326 device_delete_children(dev); 1327 1328 doom_vi(sc, &pi->vi[0]); 1329 1330 if (pi->flags & HAS_TRACEQ) { 1331 sc->traceq = -1; /* cloner should not create ifnet */ 1332 t4_tracer_port_detach(sc); 1333 } 1334 1335 cxgbe_vi_detach(&pi->vi[0]); 1336 callout_drain(&pi->tick); 1337 1338 end_synchronized_op(sc, 0); 1339 1340 return (0); 1341 } 1342 1343 static void 1344 cxgbe_init(void *arg) 1345 { 1346 struct vi_info *vi = arg; 1347 struct adapter *sc = vi->pi->adapter; 1348 1349 if (begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4init") != 0) 1350 return; 1351 cxgbe_init_synchronized(vi); 1352 end_synchronized_op(sc, 0); 1353 } 1354 1355 static int 1356 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1357 { 1358 int rc = 0, mtu, flags, can_sleep; 1359 struct vi_info *vi = ifp->if_softc; 1360 struct adapter *sc = vi->pi->adapter; 1361 struct ifreq *ifr = (struct ifreq *)data; 1362 uint32_t mask; 1363 1364 switch (cmd) { 1365 case SIOCSIFMTU: 1366 mtu = ifr->ifr_mtu; 1367 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1368 return (EINVAL); 1369 1370 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4mtu"); 1371 if (rc) 1372 return (rc); 1373 ifp->if_mtu = mtu; 1374 if (vi->flags & VI_INIT_DONE) { 1375 t4_update_fl_bufsize(ifp); 1376 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1377 rc = update_mac_settings(ifp, XGMAC_MTU); 1378 } 1379 end_synchronized_op(sc, 0); 1380 break; 1381 1382 case SIOCSIFFLAGS: 1383 can_sleep = 0; 1384 redo_sifflags: 1385 rc = begin_synchronized_op(sc, vi, 1386 can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg"); 1387 if (rc) 1388 return (rc); 1389 1390 if (ifp->if_flags & IFF_UP) { 1391 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1392 flags = vi->if_flags; 1393 if ((ifp->if_flags ^ flags) & 1394 (IFF_PROMISC | IFF_ALLMULTI)) { 1395 if (can_sleep == 1) { 1396 end_synchronized_op(sc, 0); 1397 can_sleep = 0; 1398 goto redo_sifflags; 1399 } 1400 rc = update_mac_settings(ifp, 1401 XGMAC_PROMISC | XGMAC_ALLMULTI); 1402 } 1403 } else { 1404 if (can_sleep == 0) { 1405 end_synchronized_op(sc, LOCK_HELD); 1406 can_sleep = 1; 1407 goto redo_sifflags; 1408 } 1409 rc = cxgbe_init_synchronized(vi); 1410 } 1411 vi->if_flags = ifp->if_flags; 1412 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1413 if (can_sleep == 0) { 1414 end_synchronized_op(sc, LOCK_HELD); 1415 can_sleep = 1; 1416 goto redo_sifflags; 1417 } 1418 rc = cxgbe_uninit_synchronized(vi); 1419 } 1420 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD); 1421 break; 1422 1423 case SIOCADDMULTI: 1424 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1425 rc = begin_synchronized_op(sc, vi, HOLD_LOCK, "t4multi"); 1426 if (rc) 1427 return (rc); 1428 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1429 rc = update_mac_settings(ifp, XGMAC_MCADDRS); 1430 end_synchronized_op(sc, LOCK_HELD); 1431 break; 1432 1433 case SIOCSIFCAP: 1434 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4cap"); 1435 if (rc) 1436 return (rc); 1437 1438 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1439 if (mask & IFCAP_TXCSUM) { 1440 ifp->if_capenable ^= IFCAP_TXCSUM; 1441 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1442 1443 if (IFCAP_TSO4 & ifp->if_capenable && 1444 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1445 ifp->if_capenable &= ~IFCAP_TSO4; 1446 if_printf(ifp, 1447 "tso4 disabled due to -txcsum.\n"); 1448 } 1449 } 1450 if (mask & IFCAP_TXCSUM_IPV6) { 1451 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1452 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1453 1454 if (IFCAP_TSO6 & ifp->if_capenable && 1455 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1456 ifp->if_capenable &= ~IFCAP_TSO6; 1457 if_printf(ifp, 1458 "tso6 disabled due to -txcsum6.\n"); 1459 } 1460 } 1461 if (mask & IFCAP_RXCSUM) 1462 ifp->if_capenable ^= IFCAP_RXCSUM; 1463 if (mask & IFCAP_RXCSUM_IPV6) 1464 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1465 1466 /* 1467 * Note that we leave CSUM_TSO alone (it is always set). The 1468 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1469 * sending a TSO request our way, so it's sufficient to toggle 1470 * IFCAP_TSOx only. 1471 */ 1472 if (mask & IFCAP_TSO4) { 1473 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1474 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1475 if_printf(ifp, "enable txcsum first.\n"); 1476 rc = EAGAIN; 1477 goto fail; 1478 } 1479 ifp->if_capenable ^= IFCAP_TSO4; 1480 } 1481 if (mask & IFCAP_TSO6) { 1482 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1483 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1484 if_printf(ifp, "enable txcsum6 first.\n"); 1485 rc = EAGAIN; 1486 goto fail; 1487 } 1488 ifp->if_capenable ^= IFCAP_TSO6; 1489 } 1490 if (mask & IFCAP_LRO) { 1491 #if defined(INET) || defined(INET6) 1492 int i; 1493 struct sge_rxq *rxq; 1494 1495 ifp->if_capenable ^= IFCAP_LRO; 1496 for_each_rxq(vi, i, rxq) { 1497 if (ifp->if_capenable & IFCAP_LRO) 1498 rxq->iq.flags |= IQ_LRO_ENABLED; 1499 else 1500 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1501 } 1502 #endif 1503 } 1504 #ifdef TCP_OFFLOAD 1505 if (mask & IFCAP_TOE) { 1506 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1507 1508 rc = toe_capability(vi, enable); 1509 if (rc != 0) 1510 goto fail; 1511 1512 ifp->if_capenable ^= mask; 1513 } 1514 #endif 1515 if (mask & IFCAP_VLAN_HWTAGGING) { 1516 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1517 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1518 rc = update_mac_settings(ifp, XGMAC_VLANEX); 1519 } 1520 if (mask & IFCAP_VLAN_MTU) { 1521 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1522 1523 /* Need to find out how to disable auto-mtu-inflation */ 1524 } 1525 if (mask & IFCAP_VLAN_HWTSO) 1526 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1527 if (mask & IFCAP_VLAN_HWCSUM) 1528 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1529 1530 #ifdef VLAN_CAPABILITIES 1531 VLAN_CAPABILITIES(ifp); 1532 #endif 1533 fail: 1534 end_synchronized_op(sc, 0); 1535 break; 1536 1537 case SIOCSIFMEDIA: 1538 case SIOCGIFMEDIA: 1539 ifmedia_ioctl(ifp, ifr, &vi->media, cmd); 1540 break; 1541 1542 case SIOCGI2C: { 1543 struct ifi2creq i2c; 1544 1545 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); 1546 if (rc != 0) 1547 break; 1548 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) { 1549 rc = EPERM; 1550 break; 1551 } 1552 if (i2c.len > sizeof(i2c.data)) { 1553 rc = EINVAL; 1554 break; 1555 } 1556 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4i2c"); 1557 if (rc) 1558 return (rc); 1559 rc = -t4_i2c_rd(sc, sc->mbox, vi->pi->port_id, i2c.dev_addr, 1560 i2c.offset, i2c.len, &i2c.data[0]); 1561 end_synchronized_op(sc, 0); 1562 if (rc == 0) 1563 rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); 1564 break; 1565 } 1566 1567 default: 1568 rc = ether_ioctl(ifp, cmd, data); 1569 } 1570 1571 return (rc); 1572 } 1573 1574 static int 1575 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1576 { 1577 struct vi_info *vi = ifp->if_softc; 1578 struct port_info *pi = vi->pi; 1579 struct adapter *sc = pi->adapter; 1580 struct sge_txq *txq; 1581 void *items[1]; 1582 int rc; 1583 1584 M_ASSERTPKTHDR(m); 1585 MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */ 1586 1587 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1588 m_freem(m); 1589 return (ENETDOWN); 1590 } 1591 1592 rc = parse_pkt(&m); 1593 if (__predict_false(rc != 0)) { 1594 MPASS(m == NULL); /* was freed already */ 1595 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */ 1596 return (rc); 1597 } 1598 1599 /* Select a txq. */ 1600 txq = &sc->sge.txq[vi->first_txq]; 1601 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 1602 txq += ((m->m_pkthdr.flowid % (vi->ntxq - vi->rsrv_noflowq)) + 1603 vi->rsrv_noflowq); 1604 1605 items[0] = m; 1606 rc = mp_ring_enqueue(txq->r, items, 1, 4096); 1607 if (__predict_false(rc != 0)) 1608 m_freem(m); 1609 1610 return (rc); 1611 } 1612 1613 static void 1614 cxgbe_qflush(struct ifnet *ifp) 1615 { 1616 struct vi_info *vi = ifp->if_softc; 1617 struct sge_txq *txq; 1618 int i; 1619 1620 /* queues do not exist if !VI_INIT_DONE. */ 1621 if (vi->flags & VI_INIT_DONE) { 1622 for_each_txq(vi, i, txq) { 1623 TXQ_LOCK(txq); 1624 txq->eq.flags &= ~EQ_ENABLED; 1625 TXQ_UNLOCK(txq); 1626 while (!mp_ring_is_idle(txq->r)) { 1627 mp_ring_check_drainage(txq->r, 0); 1628 pause("qflush", 1); 1629 } 1630 } 1631 } 1632 if_qflush(ifp); 1633 } 1634 1635 static uint64_t 1636 vi_get_counter(struct ifnet *ifp, ift_counter c) 1637 { 1638 struct vi_info *vi = ifp->if_softc; 1639 struct fw_vi_stats_vf *s = &vi->stats; 1640 1641 vi_refresh_stats(vi->pi->adapter, vi); 1642 1643 switch (c) { 1644 case IFCOUNTER_IPACKETS: 1645 return (s->rx_bcast_frames + s->rx_mcast_frames + 1646 s->rx_ucast_frames); 1647 case IFCOUNTER_IERRORS: 1648 return (s->rx_err_frames); 1649 case IFCOUNTER_OPACKETS: 1650 return (s->tx_bcast_frames + s->tx_mcast_frames + 1651 s->tx_ucast_frames + s->tx_offload_frames); 1652 case IFCOUNTER_OERRORS: 1653 return (s->tx_drop_frames); 1654 case IFCOUNTER_IBYTES: 1655 return (s->rx_bcast_bytes + s->rx_mcast_bytes + 1656 s->rx_ucast_bytes); 1657 case IFCOUNTER_OBYTES: 1658 return (s->tx_bcast_bytes + s->tx_mcast_bytes + 1659 s->tx_ucast_bytes + s->tx_offload_bytes); 1660 case IFCOUNTER_IMCASTS: 1661 return (s->rx_mcast_frames); 1662 case IFCOUNTER_OMCASTS: 1663 return (s->tx_mcast_frames); 1664 case IFCOUNTER_OQDROPS: { 1665 uint64_t drops; 1666 1667 drops = 0; 1668 if ((vi->flags & (VI_INIT_DONE | VI_NETMAP)) == VI_INIT_DONE) { 1669 int i; 1670 struct sge_txq *txq; 1671 1672 for_each_txq(vi, i, txq) 1673 drops += counter_u64_fetch(txq->r->drops); 1674 } 1675 1676 return (drops); 1677 1678 } 1679 1680 default: 1681 return (if_get_counter_default(ifp, c)); 1682 } 1683 } 1684 1685 uint64_t 1686 cxgbe_get_counter(struct ifnet *ifp, ift_counter c) 1687 { 1688 struct vi_info *vi = ifp->if_softc; 1689 struct port_info *pi = vi->pi; 1690 struct adapter *sc = pi->adapter; 1691 struct port_stats *s = &pi->stats; 1692 1693 if (pi->nvi > 1) 1694 return (vi_get_counter(ifp, c)); 1695 1696 cxgbe_refresh_stats(sc, pi); 1697 1698 switch (c) { 1699 case IFCOUNTER_IPACKETS: 1700 return (s->rx_frames - s->rx_pause); 1701 1702 case IFCOUNTER_IERRORS: 1703 return (s->rx_jabber + s->rx_runt + s->rx_too_long + 1704 s->rx_fcs_err + s->rx_len_err); 1705 1706 case IFCOUNTER_OPACKETS: 1707 return (s->tx_frames - s->tx_pause); 1708 1709 case IFCOUNTER_OERRORS: 1710 return (s->tx_error_frames); 1711 1712 case IFCOUNTER_IBYTES: 1713 return (s->rx_octets - s->rx_pause * 64); 1714 1715 case IFCOUNTER_OBYTES: 1716 return (s->tx_octets - s->tx_pause * 64); 1717 1718 case IFCOUNTER_IMCASTS: 1719 return (s->rx_mcast_frames - s->rx_pause); 1720 1721 case IFCOUNTER_OMCASTS: 1722 return (s->tx_mcast_frames - s->tx_pause); 1723 1724 case IFCOUNTER_IQDROPS: 1725 return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 1726 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 1727 s->rx_trunc3 + pi->tnl_cong_drops); 1728 1729 case IFCOUNTER_OQDROPS: { 1730 uint64_t drops; 1731 1732 drops = s->tx_drop; 1733 if (vi->flags & VI_INIT_DONE) { 1734 int i; 1735 struct sge_txq *txq; 1736 1737 for_each_txq(vi, i, txq) 1738 drops += counter_u64_fetch(txq->r->drops); 1739 } 1740 1741 return (drops); 1742 1743 } 1744 1745 default: 1746 return (if_get_counter_default(ifp, c)); 1747 } 1748 } 1749 1750 static int 1751 cxgbe_media_change(struct ifnet *ifp) 1752 { 1753 struct vi_info *vi = ifp->if_softc; 1754 1755 device_printf(vi->dev, "%s unimplemented.\n", __func__); 1756 1757 return (EOPNOTSUPP); 1758 } 1759 1760 static void 1761 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1762 { 1763 struct vi_info *vi = ifp->if_softc; 1764 struct port_info *pi = vi->pi; 1765 struct ifmedia_entry *cur; 1766 int speed = pi->link_cfg.speed; 1767 1768 cur = vi->media.ifm_cur; 1769 1770 ifmr->ifm_status = IFM_AVALID; 1771 if (!pi->link_cfg.link_ok) 1772 return; 1773 1774 ifmr->ifm_status |= IFM_ACTIVE; 1775 1776 /* active and current will differ iff current media is autoselect. */ 1777 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1778 return; 1779 1780 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1781 if (speed == SPEED_10000) 1782 ifmr->ifm_active |= IFM_10G_T; 1783 else if (speed == SPEED_1000) 1784 ifmr->ifm_active |= IFM_1000_T; 1785 else if (speed == SPEED_100) 1786 ifmr->ifm_active |= IFM_100_TX; 1787 else if (speed == SPEED_10) 1788 ifmr->ifm_active |= IFM_10_T; 1789 else 1790 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1791 speed)); 1792 } 1793 1794 static int 1795 vcxgbe_probe(device_t dev) 1796 { 1797 char buf[128]; 1798 struct vi_info *vi = device_get_softc(dev); 1799 1800 snprintf(buf, sizeof(buf), "port %d vi %td", vi->pi->port_id, 1801 vi - vi->pi->vi); 1802 device_set_desc_copy(dev, buf); 1803 1804 return (BUS_PROBE_DEFAULT); 1805 } 1806 1807 static int 1808 vcxgbe_attach(device_t dev) 1809 { 1810 struct vi_info *vi; 1811 struct port_info *pi; 1812 struct adapter *sc; 1813 int func, index, rc; 1814 u32 param, val; 1815 1816 vi = device_get_softc(dev); 1817 pi = vi->pi; 1818 sc = pi->adapter; 1819 1820 index = vi - pi->vi; 1821 KASSERT(index < nitems(vi_mac_funcs), 1822 ("%s: VI %s doesn't have a MAC func", __func__, 1823 device_get_nameunit(dev))); 1824 func = vi_mac_funcs[index]; 1825 rc = t4_alloc_vi_func(sc, sc->mbox, pi->tx_chan, sc->pf, 0, 1, 1826 vi->hw_addr, &vi->rss_size, func, 0); 1827 if (rc < 0) { 1828 device_printf(dev, "Failed to allocate virtual interface " 1829 "for port %d: %d\n", pi->port_id, -rc); 1830 return (-rc); 1831 } 1832 vi->viid = rc; 1833 1834 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 1835 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_RSSINFO) | 1836 V_FW_PARAMS_PARAM_YZ(vi->viid); 1837 rc = t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 1838 if (rc) 1839 vi->rss_base = 0xffff; 1840 else { 1841 /* MPASS((val >> 16) == rss_size); */ 1842 vi->rss_base = val & 0xffff; 1843 } 1844 1845 rc = cxgbe_vi_attach(dev, vi); 1846 if (rc) { 1847 t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid); 1848 return (rc); 1849 } 1850 return (0); 1851 } 1852 1853 static int 1854 vcxgbe_detach(device_t dev) 1855 { 1856 struct vi_info *vi; 1857 struct adapter *sc; 1858 1859 vi = device_get_softc(dev); 1860 sc = vi->pi->adapter; 1861 1862 doom_vi(sc, vi); 1863 1864 cxgbe_vi_detach(vi); 1865 t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid); 1866 1867 end_synchronized_op(sc, 0); 1868 1869 return (0); 1870 } 1871 1872 void 1873 t4_fatal_err(struct adapter *sc) 1874 { 1875 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1876 t4_intr_disable(sc); 1877 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1878 device_get_nameunit(sc->dev)); 1879 } 1880 1881 static int 1882 map_bars_0_and_4(struct adapter *sc) 1883 { 1884 sc->regs_rid = PCIR_BAR(0); 1885 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1886 &sc->regs_rid, RF_ACTIVE); 1887 if (sc->regs_res == NULL) { 1888 device_printf(sc->dev, "cannot map registers.\n"); 1889 return (ENXIO); 1890 } 1891 sc->bt = rman_get_bustag(sc->regs_res); 1892 sc->bh = rman_get_bushandle(sc->regs_res); 1893 sc->mmio_len = rman_get_size(sc->regs_res); 1894 setbit(&sc->doorbells, DOORBELL_KDB); 1895 1896 sc->msix_rid = PCIR_BAR(4); 1897 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1898 &sc->msix_rid, RF_ACTIVE); 1899 if (sc->msix_res == NULL) { 1900 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1901 return (ENXIO); 1902 } 1903 1904 return (0); 1905 } 1906 1907 static int 1908 map_bar_2(struct adapter *sc) 1909 { 1910 1911 /* 1912 * T4: only iWARP driver uses the userspace doorbells. There is no need 1913 * to map it if RDMA is disabled. 1914 */ 1915 if (is_t4(sc) && sc->rdmacaps == 0) 1916 return (0); 1917 1918 sc->udbs_rid = PCIR_BAR(2); 1919 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1920 &sc->udbs_rid, RF_ACTIVE); 1921 if (sc->udbs_res == NULL) { 1922 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1923 return (ENXIO); 1924 } 1925 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1926 1927 if (is_t5(sc)) { 1928 setbit(&sc->doorbells, DOORBELL_UDB); 1929 #if defined(__i386__) || defined(__amd64__) 1930 if (t5_write_combine) { 1931 int rc; 1932 1933 /* 1934 * Enable write combining on BAR2. This is the 1935 * userspace doorbell BAR and is split into 128B 1936 * (UDBS_SEG_SIZE) doorbell regions, each associated 1937 * with an egress queue. The first 64B has the doorbell 1938 * and the second 64B can be used to submit a tx work 1939 * request with an implicit doorbell. 1940 */ 1941 1942 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1943 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1944 if (rc == 0) { 1945 clrbit(&sc->doorbells, DOORBELL_UDB); 1946 setbit(&sc->doorbells, DOORBELL_WCWR); 1947 setbit(&sc->doorbells, DOORBELL_UDBWC); 1948 } else { 1949 device_printf(sc->dev, 1950 "couldn't enable write combining: %d\n", 1951 rc); 1952 } 1953 1954 t4_write_reg(sc, A_SGE_STAT_CFG, 1955 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1956 } 1957 #endif 1958 } 1959 1960 return (0); 1961 } 1962 1963 static const struct memwin t4_memwin[] = { 1964 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1965 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1966 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1967 }; 1968 1969 static const struct memwin t5_memwin[] = { 1970 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1971 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1972 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1973 }; 1974 1975 static void 1976 setup_memwin(struct adapter *sc) 1977 { 1978 const struct memwin *mw; 1979 int i, n; 1980 uint32_t bar0; 1981 1982 if (is_t4(sc)) { 1983 /* 1984 * Read low 32b of bar0 indirectly via the hardware backdoor 1985 * mechanism. Works from within PCI passthrough environments 1986 * too, where rman_get_start() can return a different value. We 1987 * need to program the T4 memory window decoders with the actual 1988 * addresses that will be coming across the PCIe link. 1989 */ 1990 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1991 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1992 1993 mw = &t4_memwin[0]; 1994 n = nitems(t4_memwin); 1995 } else { 1996 /* T5 uses the relative offset inside the PCIe BAR */ 1997 bar0 = 0; 1998 1999 mw = &t5_memwin[0]; 2000 n = nitems(t5_memwin); 2001 } 2002 2003 for (i = 0; i < n; i++, mw++) { 2004 t4_write_reg(sc, 2005 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 2006 (mw->base + bar0) | V_BIR(0) | 2007 V_WINDOW(ilog2(mw->aperture) - 10)); 2008 } 2009 2010 /* flush */ 2011 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 2012 } 2013 2014 /* 2015 * Verify that the memory range specified by the addr/len pair is valid and lies 2016 * entirely within a single region (EDCx or MCx). 2017 */ 2018 static int 2019 validate_mem_range(struct adapter *sc, uint32_t addr, int len) 2020 { 2021 uint32_t em, addr_len, maddr, mlen; 2022 2023 /* Memory can only be accessed in naturally aligned 4 byte units */ 2024 if (addr & 3 || len & 3 || len == 0) 2025 return (EINVAL); 2026 2027 /* Enabled memories */ 2028 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 2029 if (em & F_EDRAM0_ENABLE) { 2030 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 2031 maddr = G_EDRAM0_BASE(addr_len) << 20; 2032 mlen = G_EDRAM0_SIZE(addr_len) << 20; 2033 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 2034 addr + len <= maddr + mlen) 2035 return (0); 2036 } 2037 if (em & F_EDRAM1_ENABLE) { 2038 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 2039 maddr = G_EDRAM1_BASE(addr_len) << 20; 2040 mlen = G_EDRAM1_SIZE(addr_len) << 20; 2041 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 2042 addr + len <= maddr + mlen) 2043 return (0); 2044 } 2045 if (em & F_EXT_MEM_ENABLE) { 2046 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 2047 maddr = G_EXT_MEM_BASE(addr_len) << 20; 2048 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 2049 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 2050 addr + len <= maddr + mlen) 2051 return (0); 2052 } 2053 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 2054 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 2055 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 2056 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 2057 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 2058 addr + len <= maddr + mlen) 2059 return (0); 2060 } 2061 2062 return (EFAULT); 2063 } 2064 2065 static int 2066 fwmtype_to_hwmtype(int mtype) 2067 { 2068 2069 switch (mtype) { 2070 case FW_MEMTYPE_EDC0: 2071 return (MEM_EDC0); 2072 case FW_MEMTYPE_EDC1: 2073 return (MEM_EDC1); 2074 case FW_MEMTYPE_EXTMEM: 2075 return (MEM_MC0); 2076 case FW_MEMTYPE_EXTMEM1: 2077 return (MEM_MC1); 2078 default: 2079 panic("%s: cannot translate fw mtype %d.", __func__, mtype); 2080 } 2081 } 2082 2083 /* 2084 * Verify that the memory range specified by the memtype/offset/len pair is 2085 * valid and lies entirely within the memtype specified. The global address of 2086 * the start of the range is returned in addr. 2087 */ 2088 static int 2089 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 2090 uint32_t *addr) 2091 { 2092 uint32_t em, addr_len, maddr, mlen; 2093 2094 /* Memory can only be accessed in naturally aligned 4 byte units */ 2095 if (off & 3 || len & 3 || len == 0) 2096 return (EINVAL); 2097 2098 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 2099 switch (fwmtype_to_hwmtype(mtype)) { 2100 case MEM_EDC0: 2101 if (!(em & F_EDRAM0_ENABLE)) 2102 return (EINVAL); 2103 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 2104 maddr = G_EDRAM0_BASE(addr_len) << 20; 2105 mlen = G_EDRAM0_SIZE(addr_len) << 20; 2106 break; 2107 case MEM_EDC1: 2108 if (!(em & F_EDRAM1_ENABLE)) 2109 return (EINVAL); 2110 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 2111 maddr = G_EDRAM1_BASE(addr_len) << 20; 2112 mlen = G_EDRAM1_SIZE(addr_len) << 20; 2113 break; 2114 case MEM_MC: 2115 if (!(em & F_EXT_MEM_ENABLE)) 2116 return (EINVAL); 2117 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 2118 maddr = G_EXT_MEM_BASE(addr_len) << 20; 2119 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 2120 break; 2121 case MEM_MC1: 2122 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 2123 return (EINVAL); 2124 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 2125 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 2126 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 2127 break; 2128 default: 2129 return (EINVAL); 2130 } 2131 2132 if (mlen > 0 && off < mlen && off + len <= mlen) { 2133 *addr = maddr + off; /* global address */ 2134 return (0); 2135 } 2136 2137 return (EFAULT); 2138 } 2139 2140 static void 2141 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 2142 { 2143 const struct memwin *mw; 2144 2145 if (is_t4(sc)) { 2146 KASSERT(win >= 0 && win < nitems(t4_memwin), 2147 ("%s: incorrect memwin# (%d)", __func__, win)); 2148 mw = &t4_memwin[win]; 2149 } else { 2150 KASSERT(win >= 0 && win < nitems(t5_memwin), 2151 ("%s: incorrect memwin# (%d)", __func__, win)); 2152 mw = &t5_memwin[win]; 2153 } 2154 2155 if (base != NULL) 2156 *base = mw->base; 2157 if (aperture != NULL) 2158 *aperture = mw->aperture; 2159 } 2160 2161 /* 2162 * Positions the memory window such that it can be used to access the specified 2163 * address in the chip's address space. The return value is the offset of addr 2164 * from the start of the window. 2165 */ 2166 static uint32_t 2167 position_memwin(struct adapter *sc, int n, uint32_t addr) 2168 { 2169 uint32_t start, pf; 2170 uint32_t reg; 2171 2172 KASSERT(n >= 0 && n <= 3, 2173 ("%s: invalid window %d.", __func__, n)); 2174 KASSERT((addr & 3) == 0, 2175 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 2176 2177 if (is_t4(sc)) { 2178 pf = 0; 2179 start = addr & ~0xf; /* start must be 16B aligned */ 2180 } else { 2181 pf = V_PFNUM(sc->pf); 2182 start = addr & ~0x7f; /* start must be 128B aligned */ 2183 } 2184 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 2185 2186 t4_write_reg(sc, reg, start | pf); 2187 t4_read_reg(sc, reg); 2188 2189 return (addr - start); 2190 } 2191 2192 static int 2193 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, int num_vis, 2194 struct intrs_and_queues *iaq) 2195 { 2196 int rc, itype, navail, nrxq10g, nrxq1g, n; 2197 int nofldrxq10g = 0, nofldrxq1g = 0; 2198 int nnmrxq10g = 0, nnmrxq1g = 0; 2199 2200 bzero(iaq, sizeof(*iaq)); 2201 2202 iaq->ntxq10g = t4_ntxq10g; 2203 iaq->ntxq1g = t4_ntxq1g; 2204 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 2205 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 2206 iaq->rsrv_noflowq = t4_rsrv_noflowq; 2207 #ifdef TCP_OFFLOAD 2208 if (is_offload(sc)) { 2209 iaq->nofldtxq10g = t4_nofldtxq10g; 2210 iaq->nofldtxq1g = t4_nofldtxq1g; 2211 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 2212 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 2213 } 2214 #endif 2215 #ifdef DEV_NETMAP 2216 iaq->nnmtxq10g = t4_nnmtxq10g; 2217 iaq->nnmtxq1g = t4_nnmtxq1g; 2218 iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g; 2219 iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g; 2220 #endif 2221 2222 for (itype = INTR_MSIX; itype; itype >>= 1) { 2223 2224 if ((itype & t4_intr_types) == 0) 2225 continue; /* not allowed */ 2226 2227 if (itype == INTR_MSIX) 2228 navail = pci_msix_count(sc->dev); 2229 else if (itype == INTR_MSI) 2230 navail = pci_msi_count(sc->dev); 2231 else 2232 navail = 1; 2233 restart: 2234 if (navail == 0) 2235 continue; 2236 2237 iaq->intr_type = itype; 2238 iaq->intr_flags_10g = 0; 2239 iaq->intr_flags_1g = 0; 2240 2241 /* 2242 * Best option: an interrupt vector for errors, one for the 2243 * firmware event queue, and one for every rxq (NIC, TOE, and 2244 * netmap). 2245 */ 2246 iaq->nirq = T4_EXTRA_INTR; 2247 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g); 2248 iaq->nirq += n10g * 2 * (num_vis - 1); 2249 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g); 2250 iaq->nirq += n1g * 2 * (num_vis - 1); 2251 if (iaq->nirq <= navail && 2252 (itype != INTR_MSI || powerof2(iaq->nirq))) { 2253 iaq->intr_flags_10g = INTR_ALL; 2254 iaq->intr_flags_1g = INTR_ALL; 2255 goto allocate; 2256 } 2257 2258 /* 2259 * Second best option: a vector for errors, one for the firmware 2260 * event queue, and vectors for either all the NIC rx queues or 2261 * all the TOE rx queues. The queues that don't get vectors 2262 * will forward their interrupts to those that do. 2263 * 2264 * Note: netmap rx queues cannot be created early and so they 2265 * can't be setup to receive forwarded interrupts for others. 2266 */ 2267 iaq->nirq = T4_EXTRA_INTR; 2268 if (nrxq10g >= nofldrxq10g) { 2269 iaq->intr_flags_10g = INTR_RXQ; 2270 iaq->nirq += n10g * nrxq10g; 2271 iaq->nirq += n10g * (num_vis - 1); 2272 #ifdef DEV_NETMAP 2273 iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g); 2274 #endif 2275 } else { 2276 iaq->intr_flags_10g = INTR_OFLD_RXQ; 2277 iaq->nirq += n10g * nofldrxq10g; 2278 #ifdef DEV_NETMAP 2279 iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g); 2280 #endif 2281 } 2282 if (nrxq1g >= nofldrxq1g) { 2283 iaq->intr_flags_1g = INTR_RXQ; 2284 iaq->nirq += n1g * nrxq1g; 2285 iaq->nirq += n1g * (num_vis - 1); 2286 #ifdef DEV_NETMAP 2287 iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g); 2288 #endif 2289 } else { 2290 iaq->intr_flags_1g = INTR_OFLD_RXQ; 2291 iaq->nirq += n1g * nofldrxq1g; 2292 #ifdef DEV_NETMAP 2293 iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g); 2294 #endif 2295 } 2296 if (iaq->nirq <= navail && 2297 (itype != INTR_MSI || powerof2(iaq->nirq))) 2298 goto allocate; 2299 2300 /* 2301 * Next best option: an interrupt vector for errors, one for the 2302 * firmware event queue, and at least one per VI. At this 2303 * point we know we'll have to downsize nrxq and/or nofldrxq 2304 * and/or nnmrxq to fit what's available to us. 2305 */ 2306 iaq->nirq = T4_EXTRA_INTR; 2307 iaq->nirq += (n10g + n1g) * num_vis; 2308 if (iaq->nirq <= navail) { 2309 int leftover = navail - iaq->nirq; 2310 2311 if (n10g > 0) { 2312 int target = max(nrxq10g, nofldrxq10g); 2313 2314 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ? 2315 INTR_RXQ : INTR_OFLD_RXQ; 2316 2317 n = 1; 2318 while (n < target && leftover >= n10g) { 2319 leftover -= n10g; 2320 iaq->nirq += n10g; 2321 n++; 2322 } 2323 iaq->nrxq10g = min(n, nrxq10g); 2324 #ifdef TCP_OFFLOAD 2325 iaq->nofldrxq10g = min(n, nofldrxq10g); 2326 #endif 2327 #ifdef DEV_NETMAP 2328 iaq->nnmrxq10g = min(n, nnmrxq10g); 2329 #endif 2330 } 2331 2332 if (n1g > 0) { 2333 int target = max(nrxq1g, nofldrxq1g); 2334 2335 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ? 2336 INTR_RXQ : INTR_OFLD_RXQ; 2337 2338 n = 1; 2339 while (n < target && leftover >= n1g) { 2340 leftover -= n1g; 2341 iaq->nirq += n1g; 2342 n++; 2343 } 2344 iaq->nrxq1g = min(n, nrxq1g); 2345 #ifdef TCP_OFFLOAD 2346 iaq->nofldrxq1g = min(n, nofldrxq1g); 2347 #endif 2348 #ifdef DEV_NETMAP 2349 iaq->nnmrxq1g = min(n, nnmrxq1g); 2350 #endif 2351 } 2352 2353 if (itype != INTR_MSI || powerof2(iaq->nirq)) 2354 goto allocate; 2355 } 2356 2357 /* 2358 * Least desirable option: one interrupt vector for everything. 2359 */ 2360 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 2361 iaq->intr_flags_10g = iaq->intr_flags_1g = 0; 2362 #ifdef TCP_OFFLOAD 2363 if (is_offload(sc)) 2364 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 2365 #endif 2366 #ifdef DEV_NETMAP 2367 iaq->nnmrxq10g = iaq->nnmrxq1g = 1; 2368 #endif 2369 2370 allocate: 2371 navail = iaq->nirq; 2372 rc = 0; 2373 if (itype == INTR_MSIX) 2374 rc = pci_alloc_msix(sc->dev, &navail); 2375 else if (itype == INTR_MSI) 2376 rc = pci_alloc_msi(sc->dev, &navail); 2377 2378 if (rc == 0) { 2379 if (navail == iaq->nirq) 2380 return (0); 2381 2382 /* 2383 * Didn't get the number requested. Use whatever number 2384 * the kernel is willing to allocate (it's in navail). 2385 */ 2386 device_printf(sc->dev, "fewer vectors than requested, " 2387 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 2388 itype, iaq->nirq, navail); 2389 pci_release_msi(sc->dev); 2390 goto restart; 2391 } 2392 2393 device_printf(sc->dev, 2394 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 2395 itype, rc, iaq->nirq, navail); 2396 } 2397 2398 device_printf(sc->dev, 2399 "failed to find a usable interrupt type. " 2400 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 2401 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 2402 2403 return (ENXIO); 2404 } 2405 2406 #define FW_VERSION(chip) ( \ 2407 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ 2408 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ 2409 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ 2410 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) 2411 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) 2412 2413 struct fw_info { 2414 uint8_t chip; 2415 char *kld_name; 2416 char *fw_mod_name; 2417 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 2418 } fw_info[] = { 2419 { 2420 .chip = CHELSIO_T4, 2421 .kld_name = "t4fw_cfg", 2422 .fw_mod_name = "t4fw", 2423 .fw_hdr = { 2424 .chip = FW_HDR_CHIP_T4, 2425 .fw_ver = htobe32_const(FW_VERSION(T4)), 2426 .intfver_nic = FW_INTFVER(T4, NIC), 2427 .intfver_vnic = FW_INTFVER(T4, VNIC), 2428 .intfver_ofld = FW_INTFVER(T4, OFLD), 2429 .intfver_ri = FW_INTFVER(T4, RI), 2430 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 2431 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 2432 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 2433 .intfver_fcoe = FW_INTFVER(T4, FCOE), 2434 }, 2435 }, { 2436 .chip = CHELSIO_T5, 2437 .kld_name = "t5fw_cfg", 2438 .fw_mod_name = "t5fw", 2439 .fw_hdr = { 2440 .chip = FW_HDR_CHIP_T5, 2441 .fw_ver = htobe32_const(FW_VERSION(T5)), 2442 .intfver_nic = FW_INTFVER(T5, NIC), 2443 .intfver_vnic = FW_INTFVER(T5, VNIC), 2444 .intfver_ofld = FW_INTFVER(T5, OFLD), 2445 .intfver_ri = FW_INTFVER(T5, RI), 2446 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 2447 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 2448 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 2449 .intfver_fcoe = FW_INTFVER(T5, FCOE), 2450 }, 2451 } 2452 }; 2453 2454 static struct fw_info * 2455 find_fw_info(int chip) 2456 { 2457 int i; 2458 2459 for (i = 0; i < nitems(fw_info); i++) { 2460 if (fw_info[i].chip == chip) 2461 return (&fw_info[i]); 2462 } 2463 return (NULL); 2464 } 2465 2466 /* 2467 * Is the given firmware API compatible with the one the driver was compiled 2468 * with? 2469 */ 2470 static int 2471 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 2472 { 2473 2474 /* short circuit if it's the exact same firmware version */ 2475 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 2476 return (1); 2477 2478 /* 2479 * XXX: Is this too conservative? Perhaps I should limit this to the 2480 * features that are supported in the driver. 2481 */ 2482 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 2483 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 2484 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 2485 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 2486 return (1); 2487 #undef SAME_INTF 2488 2489 return (0); 2490 } 2491 2492 /* 2493 * The firmware in the KLD is usable, but should it be installed? This routine 2494 * explains itself in detail if it indicates the KLD firmware should be 2495 * installed. 2496 */ 2497 static int 2498 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 2499 { 2500 const char *reason; 2501 2502 if (!card_fw_usable) { 2503 reason = "incompatible or unusable"; 2504 goto install; 2505 } 2506 2507 if (k > c) { 2508 reason = "older than the version bundled with this driver"; 2509 goto install; 2510 } 2511 2512 if (t4_fw_install == 2 && k != c) { 2513 reason = "different than the version bundled with this driver"; 2514 goto install; 2515 } 2516 2517 return (0); 2518 2519 install: 2520 if (t4_fw_install == 0) { 2521 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2522 "but the driver is prohibited from installing a different " 2523 "firmware on the card.\n", 2524 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2525 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 2526 2527 return (0); 2528 } 2529 2530 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2531 "installing firmware %u.%u.%u.%u on card.\n", 2532 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2533 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 2534 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2535 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2536 2537 return (1); 2538 } 2539 /* 2540 * Establish contact with the firmware and determine if we are the master driver 2541 * or not, and whether we are responsible for chip initialization. 2542 */ 2543 static int 2544 prep_firmware(struct adapter *sc) 2545 { 2546 const struct firmware *fw = NULL, *default_cfg; 2547 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 2548 enum dev_state state; 2549 struct fw_info *fw_info; 2550 struct fw_hdr *card_fw; /* fw on the card */ 2551 const struct fw_hdr *kld_fw; /* fw in the KLD */ 2552 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 2553 against */ 2554 2555 /* Contact firmware. */ 2556 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 2557 if (rc < 0 || state == DEV_STATE_ERR) { 2558 rc = -rc; 2559 device_printf(sc->dev, 2560 "failed to connect to the firmware: %d, %d.\n", rc, state); 2561 return (rc); 2562 } 2563 pf = rc; 2564 if (pf == sc->mbox) 2565 sc->flags |= MASTER_PF; 2566 else if (state == DEV_STATE_UNINIT) { 2567 /* 2568 * We didn't get to be the master so we definitely won't be 2569 * configuring the chip. It's a bug if someone else hasn't 2570 * configured it already. 2571 */ 2572 device_printf(sc->dev, "couldn't be master(%d), " 2573 "device not already initialized either(%d).\n", rc, state); 2574 return (EDOOFUS); 2575 } 2576 2577 /* This is the firmware whose headers the driver was compiled against */ 2578 fw_info = find_fw_info(chip_id(sc)); 2579 if (fw_info == NULL) { 2580 device_printf(sc->dev, 2581 "unable to look up firmware information for chip %d.\n", 2582 chip_id(sc)); 2583 return (EINVAL); 2584 } 2585 drv_fw = &fw_info->fw_hdr; 2586 2587 /* 2588 * The firmware KLD contains many modules. The KLD name is also the 2589 * name of the module that contains the default config file. 2590 */ 2591 default_cfg = firmware_get(fw_info->kld_name); 2592 2593 /* Read the header of the firmware on the card */ 2594 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 2595 rc = -t4_read_flash(sc, FLASH_FW_START, 2596 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 2597 if (rc == 0) 2598 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 2599 else { 2600 device_printf(sc->dev, 2601 "Unable to read card's firmware header: %d\n", rc); 2602 card_fw_usable = 0; 2603 } 2604 2605 /* This is the firmware in the KLD */ 2606 fw = firmware_get(fw_info->fw_mod_name); 2607 if (fw != NULL) { 2608 kld_fw = (const void *)fw->data; 2609 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 2610 } else { 2611 kld_fw = NULL; 2612 kld_fw_usable = 0; 2613 } 2614 2615 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2616 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) { 2617 /* 2618 * Common case: the firmware on the card is an exact match and 2619 * the KLD is an exact match too, or the KLD is 2620 * absent/incompatible. Note that t4_fw_install = 2 is ignored 2621 * here -- use cxgbetool loadfw if you want to reinstall the 2622 * same firmware as the one on the card. 2623 */ 2624 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2625 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2626 be32toh(card_fw->fw_ver))) { 2627 2628 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 2629 if (rc != 0) { 2630 device_printf(sc->dev, 2631 "failed to install firmware: %d\n", rc); 2632 goto done; 2633 } 2634 2635 /* Installed successfully, update the cached header too. */ 2636 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2637 card_fw_usable = 1; 2638 need_fw_reset = 0; /* already reset as part of load_fw */ 2639 } 2640 2641 if (!card_fw_usable) { 2642 uint32_t d, c, k; 2643 2644 d = ntohl(drv_fw->fw_ver); 2645 c = ntohl(card_fw->fw_ver); 2646 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2647 2648 device_printf(sc->dev, "Cannot find a usable firmware: " 2649 "fw_install %d, chip state %d, " 2650 "driver compiled with %d.%d.%d.%d, " 2651 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2652 t4_fw_install, state, 2653 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2654 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2655 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2656 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2657 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2658 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2659 rc = EINVAL; 2660 goto done; 2661 } 2662 2663 /* We're using whatever's on the card and it's known to be good. */ 2664 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2665 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2666 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2667 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2668 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2669 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2670 t4_get_tp_version(sc, &sc->params.tp_vers); 2671 2672 /* Reset device */ 2673 if (need_fw_reset && 2674 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2675 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2676 if (rc != ETIMEDOUT && rc != EIO) 2677 t4_fw_bye(sc, sc->mbox); 2678 goto done; 2679 } 2680 sc->flags |= FW_OK; 2681 2682 rc = get_params__pre_init(sc); 2683 if (rc != 0) 2684 goto done; /* error message displayed already */ 2685 2686 /* Partition adapter resources as specified in the config file. */ 2687 if (state == DEV_STATE_UNINIT) { 2688 2689 KASSERT(sc->flags & MASTER_PF, 2690 ("%s: trying to change chip settings when not master.", 2691 __func__)); 2692 2693 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2694 if (rc != 0) 2695 goto done; /* error message displayed already */ 2696 2697 t4_tweak_chip_settings(sc); 2698 2699 /* get basic stuff going */ 2700 rc = -t4_fw_initialize(sc, sc->mbox); 2701 if (rc != 0) { 2702 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2703 goto done; 2704 } 2705 } else { 2706 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2707 sc->cfcsum = 0; 2708 } 2709 2710 done: 2711 free(card_fw, M_CXGBE); 2712 if (fw != NULL) 2713 firmware_put(fw, FIRMWARE_UNLOAD); 2714 if (default_cfg != NULL) 2715 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2716 2717 return (rc); 2718 } 2719 2720 #define FW_PARAM_DEV(param) \ 2721 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2722 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2723 #define FW_PARAM_PFVF(param) \ 2724 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2725 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2726 2727 /* 2728 * Partition chip resources for use between various PFs, VFs, etc. 2729 */ 2730 static int 2731 partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2732 const char *name_prefix) 2733 { 2734 const struct firmware *cfg = NULL; 2735 int rc = 0; 2736 struct fw_caps_config_cmd caps; 2737 uint32_t mtype, moff, finicsum, cfcsum; 2738 2739 /* 2740 * Figure out what configuration file to use. Pick the default config 2741 * file for the card if the user hasn't specified one explicitly. 2742 */ 2743 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2744 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2745 /* Card specific overrides go here. */ 2746 if (pci_get_device(sc->dev) == 0x440a) 2747 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2748 if (is_fpga(sc)) 2749 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2750 } 2751 2752 /* 2753 * We need to load another module if the profile is anything except 2754 * "default" or "flash". 2755 */ 2756 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2757 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2758 char s[32]; 2759 2760 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2761 cfg = firmware_get(s); 2762 if (cfg == NULL) { 2763 if (default_cfg != NULL) { 2764 device_printf(sc->dev, 2765 "unable to load module \"%s\" for " 2766 "configuration profile \"%s\", will use " 2767 "the default config file instead.\n", 2768 s, sc->cfg_file); 2769 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2770 "%s", DEFAULT_CF); 2771 } else { 2772 device_printf(sc->dev, 2773 "unable to load module \"%s\" for " 2774 "configuration profile \"%s\", will use " 2775 "the config file on the card's flash " 2776 "instead.\n", s, sc->cfg_file); 2777 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2778 "%s", FLASH_CF); 2779 } 2780 } 2781 } 2782 2783 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2784 default_cfg == NULL) { 2785 device_printf(sc->dev, 2786 "default config file not available, will use the config " 2787 "file on the card's flash instead.\n"); 2788 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2789 } 2790 2791 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2792 u_int cflen, i, n; 2793 const uint32_t *cfdata; 2794 uint32_t param, val, addr, off, mw_base, mw_aperture; 2795 2796 KASSERT(cfg != NULL || default_cfg != NULL, 2797 ("%s: no config to upload", __func__)); 2798 2799 /* 2800 * Ask the firmware where it wants us to upload the config file. 2801 */ 2802 param = FW_PARAM_DEV(CF); 2803 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2804 if (rc != 0) { 2805 /* No support for config file? Shouldn't happen. */ 2806 device_printf(sc->dev, 2807 "failed to query config file location: %d.\n", rc); 2808 goto done; 2809 } 2810 mtype = G_FW_PARAMS_PARAM_Y(val); 2811 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2812 2813 /* 2814 * XXX: sheer laziness. We deliberately added 4 bytes of 2815 * useless stuffing/comments at the end of the config file so 2816 * it's ok to simply throw away the last remaining bytes when 2817 * the config file is not an exact multiple of 4. This also 2818 * helps with the validate_mt_off_len check. 2819 */ 2820 if (cfg != NULL) { 2821 cflen = cfg->datasize & ~3; 2822 cfdata = cfg->data; 2823 } else { 2824 cflen = default_cfg->datasize & ~3; 2825 cfdata = default_cfg->data; 2826 } 2827 2828 if (cflen > FLASH_CFG_MAX_SIZE) { 2829 device_printf(sc->dev, 2830 "config file too long (%d, max allowed is %d). " 2831 "Will try to use the config on the card, if any.\n", 2832 cflen, FLASH_CFG_MAX_SIZE); 2833 goto use_config_on_flash; 2834 } 2835 2836 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2837 if (rc != 0) { 2838 device_printf(sc->dev, 2839 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2840 "Will try to use the config on the card, if any.\n", 2841 __func__, mtype, moff, cflen, rc); 2842 goto use_config_on_flash; 2843 } 2844 2845 memwin_info(sc, 2, &mw_base, &mw_aperture); 2846 while (cflen) { 2847 off = position_memwin(sc, 2, addr); 2848 n = min(cflen, mw_aperture - off); 2849 for (i = 0; i < n; i += 4) 2850 t4_write_reg(sc, mw_base + off + i, *cfdata++); 2851 cflen -= n; 2852 addr += n; 2853 } 2854 } else { 2855 use_config_on_flash: 2856 mtype = FW_MEMTYPE_FLASH; 2857 moff = t4_flash_cfg_addr(sc); 2858 } 2859 2860 bzero(&caps, sizeof(caps)); 2861 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2862 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2863 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2864 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2865 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2866 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2867 if (rc != 0) { 2868 device_printf(sc->dev, 2869 "failed to pre-process config file: %d " 2870 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2871 goto done; 2872 } 2873 2874 finicsum = be32toh(caps.finicsum); 2875 cfcsum = be32toh(caps.cfcsum); 2876 if (finicsum != cfcsum) { 2877 device_printf(sc->dev, 2878 "WARNING: config file checksum mismatch: %08x %08x\n", 2879 finicsum, cfcsum); 2880 } 2881 sc->cfcsum = cfcsum; 2882 2883 #define LIMIT_CAPS(x) do { \ 2884 caps.x &= htobe16(t4_##x##_allowed); \ 2885 } while (0) 2886 2887 /* 2888 * Let the firmware know what features will (not) be used so it can tune 2889 * things accordingly. 2890 */ 2891 LIMIT_CAPS(linkcaps); 2892 LIMIT_CAPS(niccaps); 2893 LIMIT_CAPS(toecaps); 2894 LIMIT_CAPS(rdmacaps); 2895 LIMIT_CAPS(iscsicaps); 2896 LIMIT_CAPS(fcoecaps); 2897 #undef LIMIT_CAPS 2898 2899 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2900 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2901 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2902 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2903 if (rc != 0) { 2904 device_printf(sc->dev, 2905 "failed to process config file: %d.\n", rc); 2906 } 2907 done: 2908 if (cfg != NULL) 2909 firmware_put(cfg, FIRMWARE_UNLOAD); 2910 return (rc); 2911 } 2912 2913 /* 2914 * Retrieve parameters that are needed (or nice to have) very early. 2915 */ 2916 static int 2917 get_params__pre_init(struct adapter *sc) 2918 { 2919 int rc; 2920 uint32_t param[2], val[2]; 2921 struct fw_devlog_cmd cmd; 2922 struct devlog_params *dlog = &sc->params.devlog; 2923 2924 param[0] = FW_PARAM_DEV(PORTVEC); 2925 param[1] = FW_PARAM_DEV(CCLK); 2926 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2927 if (rc != 0) { 2928 device_printf(sc->dev, 2929 "failed to query parameters (pre_init): %d.\n", rc); 2930 return (rc); 2931 } 2932 2933 sc->params.portvec = val[0]; 2934 sc->params.nports = bitcount32(val[0]); 2935 sc->params.vpd.cclk = val[1]; 2936 2937 /* Read device log parameters. */ 2938 bzero(&cmd, sizeof(cmd)); 2939 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2940 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2941 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2942 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2943 if (rc != 0) { 2944 device_printf(sc->dev, 2945 "failed to get devlog parameters: %d.\n", rc); 2946 bzero(dlog, sizeof (*dlog)); 2947 rc = 0; /* devlog isn't critical for device operation */ 2948 } else { 2949 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2950 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2951 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2952 dlog->size = be32toh(cmd.memsize_devlog); 2953 } 2954 2955 return (rc); 2956 } 2957 2958 /* 2959 * Retrieve various parameters that are of interest to the driver. The device 2960 * has been initialized by the firmware at this point. 2961 */ 2962 static int 2963 get_params__post_init(struct adapter *sc) 2964 { 2965 int rc; 2966 uint32_t param[7], val[7]; 2967 struct fw_caps_config_cmd caps; 2968 2969 param[0] = FW_PARAM_PFVF(IQFLINT_START); 2970 param[1] = FW_PARAM_PFVF(EQ_START); 2971 param[2] = FW_PARAM_PFVF(FILTER_START); 2972 param[3] = FW_PARAM_PFVF(FILTER_END); 2973 param[4] = FW_PARAM_PFVF(L2T_START); 2974 param[5] = FW_PARAM_PFVF(L2T_END); 2975 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2976 if (rc != 0) { 2977 device_printf(sc->dev, 2978 "failed to query parameters (post_init): %d.\n", rc); 2979 return (rc); 2980 } 2981 2982 sc->sge.iq_start = val[0]; 2983 sc->sge.eq_start = val[1]; 2984 sc->tids.ftid_base = val[2]; 2985 sc->tids.nftids = val[3] - val[2] + 1; 2986 sc->params.ftid_min = val[2]; 2987 sc->params.ftid_max = val[3]; 2988 sc->vres.l2t.start = val[4]; 2989 sc->vres.l2t.size = val[5] - val[4] + 1; 2990 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2991 ("%s: L2 table size (%u) larger than expected (%u)", 2992 __func__, sc->vres.l2t.size, L2T_SIZE)); 2993 2994 /* get capabilites */ 2995 bzero(&caps, sizeof(caps)); 2996 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2997 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2998 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2999 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 3000 if (rc != 0) { 3001 device_printf(sc->dev, 3002 "failed to get card capabilities: %d.\n", rc); 3003 return (rc); 3004 } 3005 3006 #define READ_CAPS(x) do { \ 3007 sc->x = htobe16(caps.x); \ 3008 } while (0) 3009 READ_CAPS(linkcaps); 3010 READ_CAPS(niccaps); 3011 READ_CAPS(toecaps); 3012 READ_CAPS(rdmacaps); 3013 READ_CAPS(iscsicaps); 3014 READ_CAPS(fcoecaps); 3015 3016 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { 3017 param[0] = FW_PARAM_PFVF(ETHOFLD_START); 3018 param[1] = FW_PARAM_PFVF(ETHOFLD_END); 3019 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 3020 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); 3021 if (rc != 0) { 3022 device_printf(sc->dev, 3023 "failed to query NIC parameters: %d.\n", rc); 3024 return (rc); 3025 } 3026 sc->tids.etid_base = val[0]; 3027 sc->params.etid_min = val[0]; 3028 sc->tids.netids = val[1] - val[0] + 1; 3029 sc->params.netids = sc->tids.netids; 3030 sc->params.eo_wr_cred = val[2]; 3031 sc->params.ethoffload = 1; 3032 } 3033 3034 if (sc->toecaps) { 3035 /* query offload-related parameters */ 3036 param[0] = FW_PARAM_DEV(NTID); 3037 param[1] = FW_PARAM_PFVF(SERVER_START); 3038 param[2] = FW_PARAM_PFVF(SERVER_END); 3039 param[3] = FW_PARAM_PFVF(TDDP_START); 3040 param[4] = FW_PARAM_PFVF(TDDP_END); 3041 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 3042 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 3043 if (rc != 0) { 3044 device_printf(sc->dev, 3045 "failed to query TOE parameters: %d.\n", rc); 3046 return (rc); 3047 } 3048 sc->tids.ntids = val[0]; 3049 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 3050 sc->tids.stid_base = val[1]; 3051 sc->tids.nstids = val[2] - val[1] + 1; 3052 sc->vres.ddp.start = val[3]; 3053 sc->vres.ddp.size = val[4] - val[3] + 1; 3054 sc->params.ofldq_wr_cred = val[5]; 3055 sc->params.offload = 1; 3056 } 3057 if (sc->rdmacaps) { 3058 param[0] = FW_PARAM_PFVF(STAG_START); 3059 param[1] = FW_PARAM_PFVF(STAG_END); 3060 param[2] = FW_PARAM_PFVF(RQ_START); 3061 param[3] = FW_PARAM_PFVF(RQ_END); 3062 param[4] = FW_PARAM_PFVF(PBL_START); 3063 param[5] = FW_PARAM_PFVF(PBL_END); 3064 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 3065 if (rc != 0) { 3066 device_printf(sc->dev, 3067 "failed to query RDMA parameters(1): %d.\n", rc); 3068 return (rc); 3069 } 3070 sc->vres.stag.start = val[0]; 3071 sc->vres.stag.size = val[1] - val[0] + 1; 3072 sc->vres.rq.start = val[2]; 3073 sc->vres.rq.size = val[3] - val[2] + 1; 3074 sc->vres.pbl.start = val[4]; 3075 sc->vres.pbl.size = val[5] - val[4] + 1; 3076 3077 param[0] = FW_PARAM_PFVF(SQRQ_START); 3078 param[1] = FW_PARAM_PFVF(SQRQ_END); 3079 param[2] = FW_PARAM_PFVF(CQ_START); 3080 param[3] = FW_PARAM_PFVF(CQ_END); 3081 param[4] = FW_PARAM_PFVF(OCQ_START); 3082 param[5] = FW_PARAM_PFVF(OCQ_END); 3083 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 3084 if (rc != 0) { 3085 device_printf(sc->dev, 3086 "failed to query RDMA parameters(2): %d.\n", rc); 3087 return (rc); 3088 } 3089 sc->vres.qp.start = val[0]; 3090 sc->vres.qp.size = val[1] - val[0] + 1; 3091 sc->vres.cq.start = val[2]; 3092 sc->vres.cq.size = val[3] - val[2] + 1; 3093 sc->vres.ocq.start = val[4]; 3094 sc->vres.ocq.size = val[5] - val[4] + 1; 3095 } 3096 if (sc->iscsicaps) { 3097 param[0] = FW_PARAM_PFVF(ISCSI_START); 3098 param[1] = FW_PARAM_PFVF(ISCSI_END); 3099 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 3100 if (rc != 0) { 3101 device_printf(sc->dev, 3102 "failed to query iSCSI parameters: %d.\n", rc); 3103 return (rc); 3104 } 3105 sc->vres.iscsi.start = val[0]; 3106 sc->vres.iscsi.size = val[1] - val[0] + 1; 3107 } 3108 3109 /* 3110 * We've got the params we wanted to query via the firmware. Now grab 3111 * some others directly from the chip. 3112 */ 3113 rc = t4_read_chip_settings(sc); 3114 3115 return (rc); 3116 } 3117 3118 static int 3119 set_params__post_init(struct adapter *sc) 3120 { 3121 uint32_t param, val; 3122 3123 /* ask for encapsulated CPLs */ 3124 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 3125 val = 1; 3126 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 3127 3128 return (0); 3129 } 3130 3131 #undef FW_PARAM_PFVF 3132 #undef FW_PARAM_DEV 3133 3134 static void 3135 t4_set_desc(struct adapter *sc) 3136 { 3137 char buf[128]; 3138 struct adapter_params *p = &sc->params; 3139 3140 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, " 3141 "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "", 3142 chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec); 3143 3144 device_set_desc_copy(sc->dev, buf); 3145 } 3146 3147 static void 3148 build_medialist(struct port_info *pi, struct ifmedia *media) 3149 { 3150 int m; 3151 3152 PORT_LOCK(pi); 3153 3154 ifmedia_removeall(media); 3155 3156 m = IFM_ETHER | IFM_FDX; 3157 3158 switch(pi->port_type) { 3159 case FW_PORT_TYPE_BT_XFI: 3160 case FW_PORT_TYPE_BT_XAUI: 3161 ifmedia_add(media, m | IFM_10G_T, 0, NULL); 3162 /* fall through */ 3163 3164 case FW_PORT_TYPE_BT_SGMII: 3165 ifmedia_add(media, m | IFM_1000_T, 0, NULL); 3166 ifmedia_add(media, m | IFM_100_TX, 0, NULL); 3167 ifmedia_add(media, IFM_ETHER | IFM_AUTO, 0, NULL); 3168 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 3169 break; 3170 3171 case FW_PORT_TYPE_CX4: 3172 ifmedia_add(media, m | IFM_10G_CX4, 0, NULL); 3173 ifmedia_set(media, m | IFM_10G_CX4); 3174 break; 3175 3176 case FW_PORT_TYPE_QSFP_10G: 3177 case FW_PORT_TYPE_SFP: 3178 case FW_PORT_TYPE_FIBER_XFI: 3179 case FW_PORT_TYPE_FIBER_XAUI: 3180 switch (pi->mod_type) { 3181 3182 case FW_PORT_MOD_TYPE_LR: 3183 ifmedia_add(media, m | IFM_10G_LR, 0, NULL); 3184 ifmedia_set(media, m | IFM_10G_LR); 3185 break; 3186 3187 case FW_PORT_MOD_TYPE_SR: 3188 ifmedia_add(media, m | IFM_10G_SR, 0, NULL); 3189 ifmedia_set(media, m | IFM_10G_SR); 3190 break; 3191 3192 case FW_PORT_MOD_TYPE_LRM: 3193 ifmedia_add(media, m | IFM_10G_LRM, 0, NULL); 3194 ifmedia_set(media, m | IFM_10G_LRM); 3195 break; 3196 3197 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 3198 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 3199 ifmedia_add(media, m | IFM_10G_TWINAX, 0, NULL); 3200 ifmedia_set(media, m | IFM_10G_TWINAX); 3201 break; 3202 3203 case FW_PORT_MOD_TYPE_NONE: 3204 m &= ~IFM_FDX; 3205 ifmedia_add(media, m | IFM_NONE, 0, NULL); 3206 ifmedia_set(media, m | IFM_NONE); 3207 break; 3208 3209 case FW_PORT_MOD_TYPE_NA: 3210 case FW_PORT_MOD_TYPE_ER: 3211 default: 3212 device_printf(pi->dev, 3213 "unknown port_type (%d), mod_type (%d)\n", 3214 pi->port_type, pi->mod_type); 3215 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 3216 ifmedia_set(media, m | IFM_UNKNOWN); 3217 break; 3218 } 3219 break; 3220 3221 case FW_PORT_TYPE_QSFP: 3222 switch (pi->mod_type) { 3223 3224 case FW_PORT_MOD_TYPE_LR: 3225 ifmedia_add(media, m | IFM_40G_LR4, 0, NULL); 3226 ifmedia_set(media, m | IFM_40G_LR4); 3227 break; 3228 3229 case FW_PORT_MOD_TYPE_SR: 3230 ifmedia_add(media, m | IFM_40G_SR4, 0, NULL); 3231 ifmedia_set(media, m | IFM_40G_SR4); 3232 break; 3233 3234 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 3235 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 3236 ifmedia_add(media, m | IFM_40G_CR4, 0, NULL); 3237 ifmedia_set(media, m | IFM_40G_CR4); 3238 break; 3239 3240 case FW_PORT_MOD_TYPE_NONE: 3241 m &= ~IFM_FDX; 3242 ifmedia_add(media, m | IFM_NONE, 0, NULL); 3243 ifmedia_set(media, m | IFM_NONE); 3244 break; 3245 3246 default: 3247 device_printf(pi->dev, 3248 "unknown port_type (%d), mod_type (%d)\n", 3249 pi->port_type, pi->mod_type); 3250 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 3251 ifmedia_set(media, m | IFM_UNKNOWN); 3252 break; 3253 } 3254 break; 3255 3256 default: 3257 device_printf(pi->dev, 3258 "unknown port_type (%d), mod_type (%d)\n", pi->port_type, 3259 pi->mod_type); 3260 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 3261 ifmedia_set(media, m | IFM_UNKNOWN); 3262 break; 3263 } 3264 3265 PORT_UNLOCK(pi); 3266 } 3267 3268 #define FW_MAC_EXACT_CHUNK 7 3269 3270 /* 3271 * Program the port's XGMAC based on parameters in ifnet. The caller also 3272 * indicates which parameters should be programmed (the rest are left alone). 3273 */ 3274 int 3275 update_mac_settings(struct ifnet *ifp, int flags) 3276 { 3277 int rc = 0; 3278 struct vi_info *vi = ifp->if_softc; 3279 struct port_info *pi = vi->pi; 3280 struct adapter *sc = pi->adapter; 3281 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 3282 3283 ASSERT_SYNCHRONIZED_OP(sc); 3284 KASSERT(flags, ("%s: not told what to update.", __func__)); 3285 3286 if (flags & XGMAC_MTU) 3287 mtu = ifp->if_mtu; 3288 3289 if (flags & XGMAC_PROMISC) 3290 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 3291 3292 if (flags & XGMAC_ALLMULTI) 3293 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 3294 3295 if (flags & XGMAC_VLANEX) 3296 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 3297 3298 if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) { 3299 rc = -t4_set_rxmode(sc, sc->mbox, vi->viid, mtu, promisc, 3300 allmulti, 1, vlanex, false); 3301 if (rc) { 3302 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, 3303 rc); 3304 return (rc); 3305 } 3306 } 3307 3308 if (flags & XGMAC_UCADDR) { 3309 uint8_t ucaddr[ETHER_ADDR_LEN]; 3310 3311 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 3312 rc = t4_change_mac(sc, sc->mbox, vi->viid, vi->xact_addr_filt, 3313 ucaddr, true, true); 3314 if (rc < 0) { 3315 rc = -rc; 3316 if_printf(ifp, "change_mac failed: %d\n", rc); 3317 return (rc); 3318 } else { 3319 vi->xact_addr_filt = rc; 3320 rc = 0; 3321 } 3322 } 3323 3324 if (flags & XGMAC_MCADDRS) { 3325 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 3326 int del = 1; 3327 uint64_t hash = 0; 3328 struct ifmultiaddr *ifma; 3329 int i = 0, j; 3330 3331 if_maddr_rlock(ifp); 3332 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3333 if (ifma->ifma_addr->sa_family != AF_LINK) 3334 continue; 3335 mcaddr[i] = 3336 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 3337 MPASS(ETHER_IS_MULTICAST(mcaddr[i])); 3338 i++; 3339 3340 if (i == FW_MAC_EXACT_CHUNK) { 3341 rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid, 3342 del, i, mcaddr, NULL, &hash, 0); 3343 if (rc < 0) { 3344 rc = -rc; 3345 for (j = 0; j < i; j++) { 3346 if_printf(ifp, 3347 "failed to add mc address" 3348 " %02x:%02x:%02x:" 3349 "%02x:%02x:%02x rc=%d\n", 3350 mcaddr[j][0], mcaddr[j][1], 3351 mcaddr[j][2], mcaddr[j][3], 3352 mcaddr[j][4], mcaddr[j][5], 3353 rc); 3354 } 3355 goto mcfail; 3356 } 3357 del = 0; 3358 i = 0; 3359 } 3360 } 3361 if (i > 0) { 3362 rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid, del, i, 3363 mcaddr, NULL, &hash, 0); 3364 if (rc < 0) { 3365 rc = -rc; 3366 for (j = 0; j < i; j++) { 3367 if_printf(ifp, 3368 "failed to add mc address" 3369 " %02x:%02x:%02x:" 3370 "%02x:%02x:%02x rc=%d\n", 3371 mcaddr[j][0], mcaddr[j][1], 3372 mcaddr[j][2], mcaddr[j][3], 3373 mcaddr[j][4], mcaddr[j][5], 3374 rc); 3375 } 3376 goto mcfail; 3377 } 3378 } 3379 3380 rc = -t4_set_addr_hash(sc, sc->mbox, vi->viid, 0, hash, 0); 3381 if (rc != 0) 3382 if_printf(ifp, "failed to set mc address hash: %d", rc); 3383 mcfail: 3384 if_maddr_runlock(ifp); 3385 } 3386 3387 return (rc); 3388 } 3389 3390 /* 3391 * {begin|end}_synchronized_op must be called from the same thread. 3392 */ 3393 int 3394 begin_synchronized_op(struct adapter *sc, struct vi_info *vi, int flags, 3395 char *wmesg) 3396 { 3397 int rc, pri; 3398 3399 #ifdef WITNESS 3400 /* the caller thinks it's ok to sleep, but is it really? */ 3401 if (flags & SLEEP_OK) 3402 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 3403 "begin_synchronized_op"); 3404 #endif 3405 3406 if (INTR_OK) 3407 pri = PCATCH; 3408 else 3409 pri = 0; 3410 3411 ADAPTER_LOCK(sc); 3412 for (;;) { 3413 3414 if (vi && IS_DOOMED(vi)) { 3415 rc = ENXIO; 3416 goto done; 3417 } 3418 3419 if (!IS_BUSY(sc)) { 3420 rc = 0; 3421 break; 3422 } 3423 3424 if (!(flags & SLEEP_OK)) { 3425 rc = EBUSY; 3426 goto done; 3427 } 3428 3429 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 3430 rc = EINTR; 3431 goto done; 3432 } 3433 } 3434 3435 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 3436 SET_BUSY(sc); 3437 #ifdef INVARIANTS 3438 sc->last_op = wmesg; 3439 sc->last_op_thr = curthread; 3440 sc->last_op_flags = flags; 3441 #endif 3442 3443 done: 3444 if (!(flags & HOLD_LOCK) || rc) 3445 ADAPTER_UNLOCK(sc); 3446 3447 return (rc); 3448 } 3449 3450 /* 3451 * Tell if_ioctl and if_init that the VI is going away. This is 3452 * special variant of begin_synchronized_op and must be paired with a 3453 * call to end_synchronized_op. 3454 */ 3455 void 3456 doom_vi(struct adapter *sc, struct vi_info *vi) 3457 { 3458 3459 ADAPTER_LOCK(sc); 3460 SET_DOOMED(vi); 3461 wakeup(&sc->flags); 3462 while (IS_BUSY(sc)) 3463 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 3464 SET_BUSY(sc); 3465 #ifdef INVARIANTS 3466 sc->last_op = "t4detach"; 3467 sc->last_op_thr = curthread; 3468 sc->last_op_flags = 0; 3469 #endif 3470 ADAPTER_UNLOCK(sc); 3471 } 3472 3473 /* 3474 * {begin|end}_synchronized_op must be called from the same thread. 3475 */ 3476 void 3477 end_synchronized_op(struct adapter *sc, int flags) 3478 { 3479 3480 if (flags & LOCK_HELD) 3481 ADAPTER_LOCK_ASSERT_OWNED(sc); 3482 else 3483 ADAPTER_LOCK(sc); 3484 3485 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 3486 CLR_BUSY(sc); 3487 wakeup(&sc->flags); 3488 ADAPTER_UNLOCK(sc); 3489 } 3490 3491 static int 3492 cxgbe_init_synchronized(struct vi_info *vi) 3493 { 3494 struct port_info *pi = vi->pi; 3495 struct adapter *sc = pi->adapter; 3496 struct ifnet *ifp = vi->ifp; 3497 int rc = 0, i; 3498 struct sge_txq *txq; 3499 3500 ASSERT_SYNCHRONIZED_OP(sc); 3501 3502 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 3503 return (0); /* already running */ 3504 3505 if (!(sc->flags & FULL_INIT_DONE) && 3506 ((rc = adapter_full_init(sc)) != 0)) 3507 return (rc); /* error message displayed already */ 3508 3509 if (!(vi->flags & VI_INIT_DONE) && 3510 ((rc = vi_full_init(vi)) != 0)) 3511 return (rc); /* error message displayed already */ 3512 3513 rc = update_mac_settings(ifp, XGMAC_ALL); 3514 if (rc) 3515 goto done; /* error message displayed already */ 3516 3517 rc = -t4_enable_vi(sc, sc->mbox, vi->viid, true, true); 3518 if (rc != 0) { 3519 if_printf(ifp, "enable_vi failed: %d\n", rc); 3520 goto done; 3521 } 3522 3523 /* 3524 * Can't fail from this point onwards. Review cxgbe_uninit_synchronized 3525 * if this changes. 3526 */ 3527 3528 for_each_txq(vi, i, txq) { 3529 TXQ_LOCK(txq); 3530 txq->eq.flags |= EQ_ENABLED; 3531 TXQ_UNLOCK(txq); 3532 } 3533 3534 /* 3535 * The first iq of the first port to come up is used for tracing. 3536 */ 3537 if (sc->traceq < 0 && IS_MAIN_VI(vi)) { 3538 sc->traceq = sc->sge.rxq[vi->first_rxq].iq.abs_id; 3539 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : 3540 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | 3541 V_QUEUENUMBER(sc->traceq)); 3542 pi->flags |= HAS_TRACEQ; 3543 } 3544 3545 /* all ok */ 3546 PORT_LOCK(pi); 3547 ifp->if_drv_flags |= IFF_DRV_RUNNING; 3548 pi->up_vis++; 3549 3550 if (pi->nvi > 1) 3551 callout_reset(&vi->tick, hz, vi_tick, vi); 3552 else 3553 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 3554 PORT_UNLOCK(pi); 3555 done: 3556 if (rc != 0) 3557 cxgbe_uninit_synchronized(vi); 3558 3559 return (rc); 3560 } 3561 3562 /* 3563 * Idempotent. 3564 */ 3565 static int 3566 cxgbe_uninit_synchronized(struct vi_info *vi) 3567 { 3568 struct port_info *pi = vi->pi; 3569 struct adapter *sc = pi->adapter; 3570 struct ifnet *ifp = vi->ifp; 3571 int rc, i; 3572 struct sge_txq *txq; 3573 3574 ASSERT_SYNCHRONIZED_OP(sc); 3575 3576 if (!(vi->flags & VI_INIT_DONE)) { 3577 KASSERT(!(ifp->if_drv_flags & IFF_DRV_RUNNING), 3578 ("uninited VI is running")); 3579 return (0); 3580 } 3581 3582 /* 3583 * Disable the VI so that all its data in either direction is discarded 3584 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 3585 * tick) intact as the TP can deliver negative advice or data that it's 3586 * holding in its RAM (for an offloaded connection) even after the VI is 3587 * disabled. 3588 */ 3589 rc = -t4_enable_vi(sc, sc->mbox, vi->viid, false, false); 3590 if (rc) { 3591 if_printf(ifp, "disable_vi failed: %d\n", rc); 3592 return (rc); 3593 } 3594 3595 for_each_txq(vi, i, txq) { 3596 TXQ_LOCK(txq); 3597 txq->eq.flags &= ~EQ_ENABLED; 3598 TXQ_UNLOCK(txq); 3599 } 3600 3601 PORT_LOCK(pi); 3602 if (pi->nvi == 1) 3603 callout_stop(&pi->tick); 3604 else 3605 callout_stop(&vi->tick); 3606 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 3607 PORT_UNLOCK(pi); 3608 return (0); 3609 } 3610 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3611 pi->up_vis--; 3612 if (pi->up_vis > 0) { 3613 PORT_UNLOCK(pi); 3614 return (0); 3615 } 3616 PORT_UNLOCK(pi); 3617 3618 pi->link_cfg.link_ok = 0; 3619 pi->link_cfg.speed = 0; 3620 pi->linkdnrc = -1; 3621 t4_os_link_changed(sc, pi->port_id, 0, -1); 3622 3623 return (0); 3624 } 3625 3626 /* 3627 * It is ok for this function to fail midway and return right away. t4_detach 3628 * will walk the entire sc->irq list and clean up whatever is valid. 3629 */ 3630 static int 3631 setup_intr_handlers(struct adapter *sc) 3632 { 3633 int rc, rid, p, q, v; 3634 char s[8]; 3635 struct irq *irq; 3636 struct port_info *pi; 3637 struct vi_info *vi; 3638 struct sge_rxq *rxq; 3639 #ifdef TCP_OFFLOAD 3640 struct sge_ofld_rxq *ofld_rxq; 3641 #endif 3642 #ifdef DEV_NETMAP 3643 struct sge_nm_rxq *nm_rxq; 3644 #endif 3645 #ifdef RSS 3646 int nbuckets = rss_getnumbuckets(); 3647 #endif 3648 3649 /* 3650 * Setup interrupts. 3651 */ 3652 irq = &sc->irq[0]; 3653 rid = sc->intr_type == INTR_INTX ? 0 : 1; 3654 if (sc->intr_count == 1) 3655 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all")); 3656 3657 /* Multiple interrupts. */ 3658 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 3659 ("%s: too few intr.", __func__)); 3660 3661 /* The first one is always error intr */ 3662 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 3663 if (rc != 0) 3664 return (rc); 3665 irq++; 3666 rid++; 3667 3668 /* The second one is always the firmware event queue */ 3669 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt"); 3670 if (rc != 0) 3671 return (rc); 3672 irq++; 3673 rid++; 3674 3675 for_each_port(sc, p) { 3676 pi = sc->port[p]; 3677 for_each_vi(pi, v, vi) { 3678 vi->first_intr = rid - 1; 3679 #ifdef DEV_NETMAP 3680 if (vi->flags & VI_NETMAP) { 3681 for_each_nm_rxq(vi, q, nm_rxq) { 3682 snprintf(s, sizeof(s), "%d-%d", p, q); 3683 rc = t4_alloc_irq(sc, irq, rid, 3684 t4_nm_intr, nm_rxq, s); 3685 if (rc != 0) 3686 return (rc); 3687 irq++; 3688 rid++; 3689 vi->nintr++; 3690 } 3691 continue; 3692 } 3693 #endif 3694 if (vi->flags & INTR_RXQ) { 3695 for_each_rxq(vi, q, rxq) { 3696 if (v == 0) 3697 snprintf(s, sizeof(s), "%d.%d", 3698 p, q); 3699 else 3700 snprintf(s, sizeof(s), 3701 "%d(%d).%d", p, v, q); 3702 rc = t4_alloc_irq(sc, irq, rid, 3703 t4_intr, rxq, s); 3704 if (rc != 0) 3705 return (rc); 3706 #ifdef RSS 3707 bus_bind_intr(sc->dev, irq->res, 3708 rss_getcpu(q % nbuckets)); 3709 #endif 3710 irq++; 3711 rid++; 3712 vi->nintr++; 3713 } 3714 } 3715 #ifdef TCP_OFFLOAD 3716 if (vi->flags & INTR_OFLD_RXQ) { 3717 for_each_ofld_rxq(vi, q, ofld_rxq) { 3718 snprintf(s, sizeof(s), "%d,%d", p, q); 3719 rc = t4_alloc_irq(sc, irq, rid, 3720 t4_intr, ofld_rxq, s); 3721 if (rc != 0) 3722 return (rc); 3723 irq++; 3724 rid++; 3725 vi->nintr++; 3726 } 3727 } 3728 #endif 3729 } 3730 } 3731 MPASS(irq == &sc->irq[sc->intr_count]); 3732 3733 return (0); 3734 } 3735 3736 int 3737 adapter_full_init(struct adapter *sc) 3738 { 3739 int rc, i; 3740 3741 ASSERT_SYNCHRONIZED_OP(sc); 3742 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3743 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 3744 ("%s: FULL_INIT_DONE already", __func__)); 3745 3746 /* 3747 * queues that belong to the adapter (not any particular port). 3748 */ 3749 rc = t4_setup_adapter_queues(sc); 3750 if (rc != 0) 3751 goto done; 3752 3753 for (i = 0; i < nitems(sc->tq); i++) { 3754 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 3755 taskqueue_thread_enqueue, &sc->tq[i]); 3756 if (sc->tq[i] == NULL) { 3757 device_printf(sc->dev, 3758 "failed to allocate task queue %d\n", i); 3759 rc = ENOMEM; 3760 goto done; 3761 } 3762 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3763 device_get_nameunit(sc->dev), i); 3764 } 3765 3766 t4_intr_enable(sc); 3767 sc->flags |= FULL_INIT_DONE; 3768 done: 3769 if (rc != 0) 3770 adapter_full_uninit(sc); 3771 3772 return (rc); 3773 } 3774 3775 int 3776 adapter_full_uninit(struct adapter *sc) 3777 { 3778 int i; 3779 3780 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3781 3782 t4_teardown_adapter_queues(sc); 3783 3784 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3785 taskqueue_free(sc->tq[i]); 3786 sc->tq[i] = NULL; 3787 } 3788 3789 sc->flags &= ~FULL_INIT_DONE; 3790 3791 return (0); 3792 } 3793 3794 #ifdef RSS 3795 #define SUPPORTED_RSS_HASHTYPES (RSS_HASHTYPE_RSS_IPV4 | \ 3796 RSS_HASHTYPE_RSS_TCP_IPV4 | RSS_HASHTYPE_RSS_IPV6 | \ 3797 RSS_HASHTYPE_RSS_TCP_IPV6 | RSS_HASHTYPE_RSS_UDP_IPV4 | \ 3798 RSS_HASHTYPE_RSS_UDP_IPV6) 3799 3800 /* Translates kernel hash types to hardware. */ 3801 static int 3802 hashconfig_to_hashen(int hashconfig) 3803 { 3804 int hashen = 0; 3805 3806 if (hashconfig & RSS_HASHTYPE_RSS_IPV4) 3807 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN; 3808 if (hashconfig & RSS_HASHTYPE_RSS_IPV6) 3809 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN; 3810 if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV4) { 3811 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN | 3812 F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN; 3813 } 3814 if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV6) { 3815 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN | 3816 F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN; 3817 } 3818 if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV4) 3819 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN; 3820 if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV6) 3821 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN; 3822 3823 return (hashen); 3824 } 3825 3826 /* Translates hardware hash types to kernel. */ 3827 static int 3828 hashen_to_hashconfig(int hashen) 3829 { 3830 int hashconfig = 0; 3831 3832 if (hashen & F_FW_RSS_VI_CONFIG_CMD_UDPEN) { 3833 /* 3834 * If UDP hashing was enabled it must have been enabled for 3835 * either IPv4 or IPv6 (inclusive or). Enabling UDP without 3836 * enabling any 4-tuple hash is nonsense configuration. 3837 */ 3838 MPASS(hashen & (F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN | 3839 F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)); 3840 3841 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) 3842 hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV4; 3843 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) 3844 hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV6; 3845 } 3846 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) 3847 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV4; 3848 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) 3849 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV6; 3850 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN) 3851 hashconfig |= RSS_HASHTYPE_RSS_IPV4; 3852 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN) 3853 hashconfig |= RSS_HASHTYPE_RSS_IPV6; 3854 3855 return (hashconfig); 3856 } 3857 #endif 3858 3859 int 3860 vi_full_init(struct vi_info *vi) 3861 { 3862 struct adapter *sc = vi->pi->adapter; 3863 struct ifnet *ifp = vi->ifp; 3864 uint16_t *rss; 3865 struct sge_rxq *rxq; 3866 int rc, i, j, hashen; 3867 #ifdef RSS 3868 int nbuckets = rss_getnumbuckets(); 3869 int hashconfig = rss_gethashconfig(); 3870 int extra; 3871 uint32_t raw_rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; 3872 uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; 3873 #endif 3874 3875 ASSERT_SYNCHRONIZED_OP(sc); 3876 KASSERT((vi->flags & VI_INIT_DONE) == 0, 3877 ("%s: VI_INIT_DONE already", __func__)); 3878 3879 sysctl_ctx_init(&vi->ctx); 3880 vi->flags |= VI_SYSCTL_CTX; 3881 3882 /* 3883 * Allocate tx/rx/fl queues for this VI. 3884 */ 3885 rc = t4_setup_vi_queues(vi); 3886 if (rc != 0) 3887 goto done; /* error message displayed already */ 3888 3889 #ifdef DEV_NETMAP 3890 /* Netmap VIs configure RSS when netmap is enabled. */ 3891 if (vi->flags & VI_NETMAP) { 3892 vi->flags |= VI_INIT_DONE; 3893 return (0); 3894 } 3895 #endif 3896 3897 /* 3898 * Setup RSS for this VI. Save a copy of the RSS table for later use. 3899 */ 3900 if (vi->nrxq > vi->rss_size) { 3901 if_printf(ifp, "nrxq (%d) > hw RSS table size (%d); " 3902 "some queues will never receive traffic.\n", vi->nrxq, 3903 vi->rss_size); 3904 } else if (vi->rss_size % vi->nrxq) { 3905 if_printf(ifp, "nrxq (%d), hw RSS table size (%d); " 3906 "expect uneven traffic distribution.\n", vi->nrxq, 3907 vi->rss_size); 3908 } 3909 #ifdef RSS 3910 MPASS(RSS_KEYSIZE == 40); 3911 if (vi->nrxq != nbuckets) { 3912 if_printf(ifp, "nrxq (%d) != kernel RSS buckets (%d);" 3913 "performance will be impacted.\n", vi->nrxq, nbuckets); 3914 } 3915 3916 rss_getkey((void *)&raw_rss_key[0]); 3917 for (i = 0; i < nitems(rss_key); i++) { 3918 rss_key[i] = htobe32(raw_rss_key[nitems(rss_key) - 1 - i]); 3919 } 3920 t4_write_rss_key(sc, (void *)&rss_key[0], -1); 3921 #endif 3922 rss = malloc(vi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); 3923 for (i = 0; i < vi->rss_size;) { 3924 #ifdef RSS 3925 j = rss_get_indirection_to_bucket(i); 3926 j %= vi->nrxq; 3927 rxq = &sc->sge.rxq[vi->first_rxq + j]; 3928 rss[i++] = rxq->iq.abs_id; 3929 #else 3930 for_each_rxq(vi, j, rxq) { 3931 rss[i++] = rxq->iq.abs_id; 3932 if (i == vi->rss_size) 3933 break; 3934 } 3935 #endif 3936 } 3937 3938 rc = -t4_config_rss_range(sc, sc->mbox, vi->viid, 0, vi->rss_size, rss, 3939 vi->rss_size); 3940 if (rc != 0) { 3941 if_printf(ifp, "rss_config failed: %d\n", rc); 3942 goto done; 3943 } 3944 3945 #ifdef RSS 3946 hashen = hashconfig_to_hashen(hashconfig); 3947 3948 /* 3949 * We may have had to enable some hashes even though the global config 3950 * wants them disabled. This is a potential problem that must be 3951 * reported to the user. 3952 */ 3953 extra = hashen_to_hashconfig(hashen) ^ hashconfig; 3954 3955 /* 3956 * If we consider only the supported hash types, then the enabled hashes 3957 * are a superset of the requested hashes. In other words, there cannot 3958 * be any supported hash that was requested but not enabled, but there 3959 * can be hashes that were not requested but had to be enabled. 3960 */ 3961 extra &= SUPPORTED_RSS_HASHTYPES; 3962 MPASS((extra & hashconfig) == 0); 3963 3964 if (extra) { 3965 if_printf(ifp, 3966 "global RSS config (0x%x) cannot be accomodated.\n", 3967 hashconfig); 3968 } 3969 if (extra & RSS_HASHTYPE_RSS_IPV4) 3970 if_printf(ifp, "IPv4 2-tuple hashing forced on.\n"); 3971 if (extra & RSS_HASHTYPE_RSS_TCP_IPV4) 3972 if_printf(ifp, "TCP/IPv4 4-tuple hashing forced on.\n"); 3973 if (extra & RSS_HASHTYPE_RSS_IPV6) 3974 if_printf(ifp, "IPv6 2-tuple hashing forced on.\n"); 3975 if (extra & RSS_HASHTYPE_RSS_TCP_IPV6) 3976 if_printf(ifp, "TCP/IPv6 4-tuple hashing forced on.\n"); 3977 if (extra & RSS_HASHTYPE_RSS_UDP_IPV4) 3978 if_printf(ifp, "UDP/IPv4 4-tuple hashing forced on.\n"); 3979 if (extra & RSS_HASHTYPE_RSS_UDP_IPV6) 3980 if_printf(ifp, "UDP/IPv6 4-tuple hashing forced on.\n"); 3981 #else 3982 hashen = F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN | 3983 F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN | 3984 F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN | 3985 F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN | F_FW_RSS_VI_CONFIG_CMD_UDPEN; 3986 #endif 3987 rc = -t4_config_vi_rss(sc, sc->mbox, vi->viid, hashen, rss[0]); 3988 if (rc != 0) { 3989 if_printf(ifp, "rss hash/defaultq config failed: %d\n", rc); 3990 goto done; 3991 } 3992 3993 vi->rss = rss; 3994 vi->flags |= VI_INIT_DONE; 3995 done: 3996 if (rc != 0) 3997 vi_full_uninit(vi); 3998 3999 return (rc); 4000 } 4001 4002 /* 4003 * Idempotent. 4004 */ 4005 int 4006 vi_full_uninit(struct vi_info *vi) 4007 { 4008 struct port_info *pi = vi->pi; 4009 struct adapter *sc = pi->adapter; 4010 int i; 4011 struct sge_rxq *rxq; 4012 struct sge_txq *txq; 4013 #ifdef TCP_OFFLOAD 4014 struct sge_ofld_rxq *ofld_rxq; 4015 struct sge_wrq *ofld_txq; 4016 #endif 4017 4018 if (vi->flags & VI_INIT_DONE) { 4019 4020 /* Need to quiesce queues. */ 4021 #ifdef DEV_NETMAP 4022 if (vi->flags & VI_NETMAP) 4023 goto skip; 4024 #endif 4025 4026 /* XXX: Only for the first VI? */ 4027 if (IS_MAIN_VI(vi)) 4028 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]); 4029 4030 for_each_txq(vi, i, txq) { 4031 quiesce_txq(sc, txq); 4032 } 4033 4034 #ifdef TCP_OFFLOAD 4035 for_each_ofld_txq(vi, i, ofld_txq) { 4036 quiesce_wrq(sc, ofld_txq); 4037 } 4038 #endif 4039 4040 for_each_rxq(vi, i, rxq) { 4041 quiesce_iq(sc, &rxq->iq); 4042 quiesce_fl(sc, &rxq->fl); 4043 } 4044 4045 #ifdef TCP_OFFLOAD 4046 for_each_ofld_rxq(vi, i, ofld_rxq) { 4047 quiesce_iq(sc, &ofld_rxq->iq); 4048 quiesce_fl(sc, &ofld_rxq->fl); 4049 } 4050 #endif 4051 free(vi->rss, M_CXGBE); 4052 } 4053 #ifdef DEV_NETMAP 4054 skip: 4055 #endif 4056 4057 t4_teardown_vi_queues(vi); 4058 vi->flags &= ~VI_INIT_DONE; 4059 4060 return (0); 4061 } 4062 4063 static void 4064 quiesce_txq(struct adapter *sc, struct sge_txq *txq) 4065 { 4066 struct sge_eq *eq = &txq->eq; 4067 struct sge_qstat *spg = (void *)&eq->desc[eq->sidx]; 4068 4069 (void) sc; /* unused */ 4070 4071 #ifdef INVARIANTS 4072 TXQ_LOCK(txq); 4073 MPASS((eq->flags & EQ_ENABLED) == 0); 4074 TXQ_UNLOCK(txq); 4075 #endif 4076 4077 /* Wait for the mp_ring to empty. */ 4078 while (!mp_ring_is_idle(txq->r)) { 4079 mp_ring_check_drainage(txq->r, 0); 4080 pause("rquiesce", 1); 4081 } 4082 4083 /* Then wait for the hardware to finish. */ 4084 while (spg->cidx != htobe16(eq->pidx)) 4085 pause("equiesce", 1); 4086 4087 /* Finally, wait for the driver to reclaim all descriptors. */ 4088 while (eq->cidx != eq->pidx) 4089 pause("dquiesce", 1); 4090 } 4091 4092 static void 4093 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq) 4094 { 4095 4096 /* XXXTX */ 4097 } 4098 4099 static void 4100 quiesce_iq(struct adapter *sc, struct sge_iq *iq) 4101 { 4102 (void) sc; /* unused */ 4103 4104 /* Synchronize with the interrupt handler */ 4105 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 4106 pause("iqfree", 1); 4107 } 4108 4109 static void 4110 quiesce_fl(struct adapter *sc, struct sge_fl *fl) 4111 { 4112 mtx_lock(&sc->sfl_lock); 4113 FL_LOCK(fl); 4114 fl->flags |= FL_DOOMED; 4115 FL_UNLOCK(fl); 4116 callout_stop(&sc->sfl_callout); 4117 mtx_unlock(&sc->sfl_lock); 4118 4119 KASSERT((fl->flags & FL_STARVING) == 0, 4120 ("%s: still starving", __func__)); 4121 } 4122 4123 static int 4124 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 4125 driver_intr_t *handler, void *arg, char *name) 4126 { 4127 int rc; 4128 4129 irq->rid = rid; 4130 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 4131 RF_SHAREABLE | RF_ACTIVE); 4132 if (irq->res == NULL) { 4133 device_printf(sc->dev, 4134 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 4135 return (ENOMEM); 4136 } 4137 4138 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 4139 NULL, handler, arg, &irq->tag); 4140 if (rc != 0) { 4141 device_printf(sc->dev, 4142 "failed to setup interrupt for rid %d, name %s: %d\n", 4143 rid, name, rc); 4144 } else if (name) 4145 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 4146 4147 return (rc); 4148 } 4149 4150 static int 4151 t4_free_irq(struct adapter *sc, struct irq *irq) 4152 { 4153 if (irq->tag) 4154 bus_teardown_intr(sc->dev, irq->res, irq->tag); 4155 if (irq->res) 4156 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 4157 4158 bzero(irq, sizeof(*irq)); 4159 4160 return (0); 4161 } 4162 4163 static void 4164 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 4165 unsigned int end) 4166 { 4167 uint32_t *p = (uint32_t *)(buf + start); 4168 4169 for ( ; start <= end; start += sizeof(uint32_t)) 4170 *p++ = t4_read_reg(sc, start); 4171 } 4172 4173 static void 4174 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 4175 { 4176 int i, n; 4177 const unsigned int *reg_ranges; 4178 static const unsigned int t4_reg_ranges[] = { 4179 0x1008, 0x1108, 4180 0x1180, 0x11b4, 4181 0x11fc, 0x123c, 4182 0x1300, 0x173c, 4183 0x1800, 0x18fc, 4184 0x3000, 0x30d8, 4185 0x30e0, 0x5924, 4186 0x5960, 0x59d4, 4187 0x5a00, 0x5af8, 4188 0x6000, 0x6098, 4189 0x6100, 0x6150, 4190 0x6200, 0x6208, 4191 0x6240, 0x6248, 4192 0x6280, 0x6338, 4193 0x6370, 0x638c, 4194 0x6400, 0x643c, 4195 0x6500, 0x6524, 4196 0x6a00, 0x6a38, 4197 0x6a60, 0x6a78, 4198 0x6b00, 0x6b84, 4199 0x6bf0, 0x6c84, 4200 0x6cf0, 0x6d84, 4201 0x6df0, 0x6e84, 4202 0x6ef0, 0x6f84, 4203 0x6ff0, 0x7084, 4204 0x70f0, 0x7184, 4205 0x71f0, 0x7284, 4206 0x72f0, 0x7384, 4207 0x73f0, 0x7450, 4208 0x7500, 0x7530, 4209 0x7600, 0x761c, 4210 0x7680, 0x76cc, 4211 0x7700, 0x7798, 4212 0x77c0, 0x77fc, 4213 0x7900, 0x79fc, 4214 0x7b00, 0x7c38, 4215 0x7d00, 0x7efc, 4216 0x8dc0, 0x8e1c, 4217 0x8e30, 0x8e78, 4218 0x8ea0, 0x8f6c, 4219 0x8fc0, 0x9074, 4220 0x90fc, 0x90fc, 4221 0x9400, 0x9458, 4222 0x9600, 0x96bc, 4223 0x9800, 0x9808, 4224 0x9820, 0x983c, 4225 0x9850, 0x9864, 4226 0x9c00, 0x9c6c, 4227 0x9c80, 0x9cec, 4228 0x9d00, 0x9d6c, 4229 0x9d80, 0x9dec, 4230 0x9e00, 0x9e6c, 4231 0x9e80, 0x9eec, 4232 0x9f00, 0x9f6c, 4233 0x9f80, 0x9fec, 4234 0xd004, 0xd03c, 4235 0xdfc0, 0xdfe0, 4236 0xe000, 0xea7c, 4237 0xf000, 0x11110, 4238 0x11118, 0x11190, 4239 0x19040, 0x1906c, 4240 0x19078, 0x19080, 4241 0x1908c, 0x19124, 4242 0x19150, 0x191b0, 4243 0x191d0, 0x191e8, 4244 0x19238, 0x1924c, 4245 0x193f8, 0x19474, 4246 0x19490, 0x194f8, 4247 0x19800, 0x19f30, 4248 0x1a000, 0x1a06c, 4249 0x1a0b0, 0x1a120, 4250 0x1a128, 0x1a138, 4251 0x1a190, 0x1a1c4, 4252 0x1a1fc, 0x1a1fc, 4253 0x1e040, 0x1e04c, 4254 0x1e284, 0x1e28c, 4255 0x1e2c0, 0x1e2c0, 4256 0x1e2e0, 0x1e2e0, 4257 0x1e300, 0x1e384, 4258 0x1e3c0, 0x1e3c8, 4259 0x1e440, 0x1e44c, 4260 0x1e684, 0x1e68c, 4261 0x1e6c0, 0x1e6c0, 4262 0x1e6e0, 0x1e6e0, 4263 0x1e700, 0x1e784, 4264 0x1e7c0, 0x1e7c8, 4265 0x1e840, 0x1e84c, 4266 0x1ea84, 0x1ea8c, 4267 0x1eac0, 0x1eac0, 4268 0x1eae0, 0x1eae0, 4269 0x1eb00, 0x1eb84, 4270 0x1ebc0, 0x1ebc8, 4271 0x1ec40, 0x1ec4c, 4272 0x1ee84, 0x1ee8c, 4273 0x1eec0, 0x1eec0, 4274 0x1eee0, 0x1eee0, 4275 0x1ef00, 0x1ef84, 4276 0x1efc0, 0x1efc8, 4277 0x1f040, 0x1f04c, 4278 0x1f284, 0x1f28c, 4279 0x1f2c0, 0x1f2c0, 4280 0x1f2e0, 0x1f2e0, 4281 0x1f300, 0x1f384, 4282 0x1f3c0, 0x1f3c8, 4283 0x1f440, 0x1f44c, 4284 0x1f684, 0x1f68c, 4285 0x1f6c0, 0x1f6c0, 4286 0x1f6e0, 0x1f6e0, 4287 0x1f700, 0x1f784, 4288 0x1f7c0, 0x1f7c8, 4289 0x1f840, 0x1f84c, 4290 0x1fa84, 0x1fa8c, 4291 0x1fac0, 0x1fac0, 4292 0x1fae0, 0x1fae0, 4293 0x1fb00, 0x1fb84, 4294 0x1fbc0, 0x1fbc8, 4295 0x1fc40, 0x1fc4c, 4296 0x1fe84, 0x1fe8c, 4297 0x1fec0, 0x1fec0, 4298 0x1fee0, 0x1fee0, 4299 0x1ff00, 0x1ff84, 4300 0x1ffc0, 0x1ffc8, 4301 0x20000, 0x2002c, 4302 0x20100, 0x2013c, 4303 0x20190, 0x201c8, 4304 0x20200, 0x20318, 4305 0x20400, 0x20528, 4306 0x20540, 0x20614, 4307 0x21000, 0x21040, 4308 0x2104c, 0x21060, 4309 0x210c0, 0x210ec, 4310 0x21200, 0x21268, 4311 0x21270, 0x21284, 4312 0x212fc, 0x21388, 4313 0x21400, 0x21404, 4314 0x21500, 0x21518, 4315 0x2152c, 0x2153c, 4316 0x21550, 0x21554, 4317 0x21600, 0x21600, 4318 0x21608, 0x21628, 4319 0x21630, 0x2163c, 4320 0x21700, 0x2171c, 4321 0x21780, 0x2178c, 4322 0x21800, 0x21c38, 4323 0x21c80, 0x21d7c, 4324 0x21e00, 0x21e04, 4325 0x22000, 0x2202c, 4326 0x22100, 0x2213c, 4327 0x22190, 0x221c8, 4328 0x22200, 0x22318, 4329 0x22400, 0x22528, 4330 0x22540, 0x22614, 4331 0x23000, 0x23040, 4332 0x2304c, 0x23060, 4333 0x230c0, 0x230ec, 4334 0x23200, 0x23268, 4335 0x23270, 0x23284, 4336 0x232fc, 0x23388, 4337 0x23400, 0x23404, 4338 0x23500, 0x23518, 4339 0x2352c, 0x2353c, 4340 0x23550, 0x23554, 4341 0x23600, 0x23600, 4342 0x23608, 0x23628, 4343 0x23630, 0x2363c, 4344 0x23700, 0x2371c, 4345 0x23780, 0x2378c, 4346 0x23800, 0x23c38, 4347 0x23c80, 0x23d7c, 4348 0x23e00, 0x23e04, 4349 0x24000, 0x2402c, 4350 0x24100, 0x2413c, 4351 0x24190, 0x241c8, 4352 0x24200, 0x24318, 4353 0x24400, 0x24528, 4354 0x24540, 0x24614, 4355 0x25000, 0x25040, 4356 0x2504c, 0x25060, 4357 0x250c0, 0x250ec, 4358 0x25200, 0x25268, 4359 0x25270, 0x25284, 4360 0x252fc, 0x25388, 4361 0x25400, 0x25404, 4362 0x25500, 0x25518, 4363 0x2552c, 0x2553c, 4364 0x25550, 0x25554, 4365 0x25600, 0x25600, 4366 0x25608, 0x25628, 4367 0x25630, 0x2563c, 4368 0x25700, 0x2571c, 4369 0x25780, 0x2578c, 4370 0x25800, 0x25c38, 4371 0x25c80, 0x25d7c, 4372 0x25e00, 0x25e04, 4373 0x26000, 0x2602c, 4374 0x26100, 0x2613c, 4375 0x26190, 0x261c8, 4376 0x26200, 0x26318, 4377 0x26400, 0x26528, 4378 0x26540, 0x26614, 4379 0x27000, 0x27040, 4380 0x2704c, 0x27060, 4381 0x270c0, 0x270ec, 4382 0x27200, 0x27268, 4383 0x27270, 0x27284, 4384 0x272fc, 0x27388, 4385 0x27400, 0x27404, 4386 0x27500, 0x27518, 4387 0x2752c, 0x2753c, 4388 0x27550, 0x27554, 4389 0x27600, 0x27600, 4390 0x27608, 0x27628, 4391 0x27630, 0x2763c, 4392 0x27700, 0x2771c, 4393 0x27780, 0x2778c, 4394 0x27800, 0x27c38, 4395 0x27c80, 0x27d7c, 4396 0x27e00, 0x27e04 4397 }; 4398 static const unsigned int t5_reg_ranges[] = { 4399 0x1008, 0x1148, 4400 0x1180, 0x11b4, 4401 0x11fc, 0x123c, 4402 0x1280, 0x173c, 4403 0x1800, 0x18fc, 4404 0x3000, 0x3028, 4405 0x3060, 0x30d8, 4406 0x30e0, 0x30fc, 4407 0x3140, 0x357c, 4408 0x35a8, 0x35cc, 4409 0x35ec, 0x35ec, 4410 0x3600, 0x5624, 4411 0x56cc, 0x575c, 4412 0x580c, 0x5814, 4413 0x5890, 0x58bc, 4414 0x5940, 0x59dc, 4415 0x59fc, 0x5a18, 4416 0x5a60, 0x5a9c, 4417 0x5b94, 0x5bfc, 4418 0x6000, 0x6040, 4419 0x6058, 0x614c, 4420 0x7700, 0x7798, 4421 0x77c0, 0x78fc, 4422 0x7b00, 0x7c54, 4423 0x7d00, 0x7efc, 4424 0x8dc0, 0x8de0, 4425 0x8df8, 0x8e84, 4426 0x8ea0, 0x8f84, 4427 0x8fc0, 0x90f8, 4428 0x9400, 0x9470, 4429 0x9600, 0x96f4, 4430 0x9800, 0x9808, 4431 0x9820, 0x983c, 4432 0x9850, 0x9864, 4433 0x9c00, 0x9c6c, 4434 0x9c80, 0x9cec, 4435 0x9d00, 0x9d6c, 4436 0x9d80, 0x9dec, 4437 0x9e00, 0x9e6c, 4438 0x9e80, 0x9eec, 4439 0x9f00, 0x9f6c, 4440 0x9f80, 0xa020, 4441 0xd004, 0xd03c, 4442 0xdfc0, 0xdfe0, 4443 0xe000, 0x11088, 4444 0x1109c, 0x11110, 4445 0x11118, 0x1117c, 4446 0x11190, 0x11204, 4447 0x19040, 0x1906c, 4448 0x19078, 0x19080, 4449 0x1908c, 0x19124, 4450 0x19150, 0x191b0, 4451 0x191d0, 0x191e8, 4452 0x19238, 0x19290, 4453 0x193f8, 0x19474, 4454 0x19490, 0x194cc, 4455 0x194f0, 0x194f8, 4456 0x19c00, 0x19c60, 4457 0x19c94, 0x19e10, 4458 0x19e50, 0x19f34, 4459 0x19f40, 0x19f50, 4460 0x19f90, 0x19fe4, 4461 0x1a000, 0x1a06c, 4462 0x1a0b0, 0x1a120, 4463 0x1a128, 0x1a138, 4464 0x1a190, 0x1a1c4, 4465 0x1a1fc, 0x1a1fc, 4466 0x1e008, 0x1e00c, 4467 0x1e040, 0x1e04c, 4468 0x1e284, 0x1e290, 4469 0x1e2c0, 0x1e2c0, 4470 0x1e2e0, 0x1e2e0, 4471 0x1e300, 0x1e384, 4472 0x1e3c0, 0x1e3c8, 4473 0x1e408, 0x1e40c, 4474 0x1e440, 0x1e44c, 4475 0x1e684, 0x1e690, 4476 0x1e6c0, 0x1e6c0, 4477 0x1e6e0, 0x1e6e0, 4478 0x1e700, 0x1e784, 4479 0x1e7c0, 0x1e7c8, 4480 0x1e808, 0x1e80c, 4481 0x1e840, 0x1e84c, 4482 0x1ea84, 0x1ea90, 4483 0x1eac0, 0x1eac0, 4484 0x1eae0, 0x1eae0, 4485 0x1eb00, 0x1eb84, 4486 0x1ebc0, 0x1ebc8, 4487 0x1ec08, 0x1ec0c, 4488 0x1ec40, 0x1ec4c, 4489 0x1ee84, 0x1ee90, 4490 0x1eec0, 0x1eec0, 4491 0x1eee0, 0x1eee0, 4492 0x1ef00, 0x1ef84, 4493 0x1efc0, 0x1efc8, 4494 0x1f008, 0x1f00c, 4495 0x1f040, 0x1f04c, 4496 0x1f284, 0x1f290, 4497 0x1f2c0, 0x1f2c0, 4498 0x1f2e0, 0x1f2e0, 4499 0x1f300, 0x1f384, 4500 0x1f3c0, 0x1f3c8, 4501 0x1f408, 0x1f40c, 4502 0x1f440, 0x1f44c, 4503 0x1f684, 0x1f690, 4504 0x1f6c0, 0x1f6c0, 4505 0x1f6e0, 0x1f6e0, 4506 0x1f700, 0x1f784, 4507 0x1f7c0, 0x1f7c8, 4508 0x1f808, 0x1f80c, 4509 0x1f840, 0x1f84c, 4510 0x1fa84, 0x1fa90, 4511 0x1fac0, 0x1fac0, 4512 0x1fae0, 0x1fae0, 4513 0x1fb00, 0x1fb84, 4514 0x1fbc0, 0x1fbc8, 4515 0x1fc08, 0x1fc0c, 4516 0x1fc40, 0x1fc4c, 4517 0x1fe84, 0x1fe90, 4518 0x1fec0, 0x1fec0, 4519 0x1fee0, 0x1fee0, 4520 0x1ff00, 0x1ff84, 4521 0x1ffc0, 0x1ffc8, 4522 0x30000, 0x30030, 4523 0x30100, 0x30144, 4524 0x30190, 0x301d0, 4525 0x30200, 0x30318, 4526 0x30400, 0x3052c, 4527 0x30540, 0x3061c, 4528 0x30800, 0x30834, 4529 0x308c0, 0x30908, 4530 0x30910, 0x309ac, 4531 0x30a00, 0x30a2c, 4532 0x30a44, 0x30a50, 4533 0x30a74, 0x30c24, 4534 0x30d00, 0x30d00, 4535 0x30d08, 0x30d14, 4536 0x30d1c, 0x30d20, 4537 0x30d3c, 0x30d50, 4538 0x31200, 0x3120c, 4539 0x31220, 0x31220, 4540 0x31240, 0x31240, 4541 0x31600, 0x3160c, 4542 0x31a00, 0x31a1c, 4543 0x31e00, 0x31e20, 4544 0x31e38, 0x31e3c, 4545 0x31e80, 0x31e80, 4546 0x31e88, 0x31ea8, 4547 0x31eb0, 0x31eb4, 4548 0x31ec8, 0x31ed4, 4549 0x31fb8, 0x32004, 4550 0x32200, 0x32200, 4551 0x32208, 0x32240, 4552 0x32248, 0x32280, 4553 0x32288, 0x322c0, 4554 0x322c8, 0x322fc, 4555 0x32600, 0x32630, 4556 0x32a00, 0x32abc, 4557 0x32b00, 0x32b70, 4558 0x33000, 0x33048, 4559 0x33060, 0x3309c, 4560 0x330f0, 0x33148, 4561 0x33160, 0x3319c, 4562 0x331f0, 0x332e4, 4563 0x332f8, 0x333e4, 4564 0x333f8, 0x33448, 4565 0x33460, 0x3349c, 4566 0x334f0, 0x33548, 4567 0x33560, 0x3359c, 4568 0x335f0, 0x336e4, 4569 0x336f8, 0x337e4, 4570 0x337f8, 0x337fc, 4571 0x33814, 0x33814, 4572 0x3382c, 0x3382c, 4573 0x33880, 0x3388c, 4574 0x338e8, 0x338ec, 4575 0x33900, 0x33948, 4576 0x33960, 0x3399c, 4577 0x339f0, 0x33ae4, 4578 0x33af8, 0x33b10, 4579 0x33b28, 0x33b28, 4580 0x33b3c, 0x33b50, 4581 0x33bf0, 0x33c10, 4582 0x33c28, 0x33c28, 4583 0x33c3c, 0x33c50, 4584 0x33cf0, 0x33cfc, 4585 0x34000, 0x34030, 4586 0x34100, 0x34144, 4587 0x34190, 0x341d0, 4588 0x34200, 0x34318, 4589 0x34400, 0x3452c, 4590 0x34540, 0x3461c, 4591 0x34800, 0x34834, 4592 0x348c0, 0x34908, 4593 0x34910, 0x349ac, 4594 0x34a00, 0x34a2c, 4595 0x34a44, 0x34a50, 4596 0x34a74, 0x34c24, 4597 0x34d00, 0x34d00, 4598 0x34d08, 0x34d14, 4599 0x34d1c, 0x34d20, 4600 0x34d3c, 0x34d50, 4601 0x35200, 0x3520c, 4602 0x35220, 0x35220, 4603 0x35240, 0x35240, 4604 0x35600, 0x3560c, 4605 0x35a00, 0x35a1c, 4606 0x35e00, 0x35e20, 4607 0x35e38, 0x35e3c, 4608 0x35e80, 0x35e80, 4609 0x35e88, 0x35ea8, 4610 0x35eb0, 0x35eb4, 4611 0x35ec8, 0x35ed4, 4612 0x35fb8, 0x36004, 4613 0x36200, 0x36200, 4614 0x36208, 0x36240, 4615 0x36248, 0x36280, 4616 0x36288, 0x362c0, 4617 0x362c8, 0x362fc, 4618 0x36600, 0x36630, 4619 0x36a00, 0x36abc, 4620 0x36b00, 0x36b70, 4621 0x37000, 0x37048, 4622 0x37060, 0x3709c, 4623 0x370f0, 0x37148, 4624 0x37160, 0x3719c, 4625 0x371f0, 0x372e4, 4626 0x372f8, 0x373e4, 4627 0x373f8, 0x37448, 4628 0x37460, 0x3749c, 4629 0x374f0, 0x37548, 4630 0x37560, 0x3759c, 4631 0x375f0, 0x376e4, 4632 0x376f8, 0x377e4, 4633 0x377f8, 0x377fc, 4634 0x37814, 0x37814, 4635 0x3782c, 0x3782c, 4636 0x37880, 0x3788c, 4637 0x378e8, 0x378ec, 4638 0x37900, 0x37948, 4639 0x37960, 0x3799c, 4640 0x379f0, 0x37ae4, 4641 0x37af8, 0x37b10, 4642 0x37b28, 0x37b28, 4643 0x37b3c, 0x37b50, 4644 0x37bf0, 0x37c10, 4645 0x37c28, 0x37c28, 4646 0x37c3c, 0x37c50, 4647 0x37cf0, 0x37cfc, 4648 0x38000, 0x38030, 4649 0x38100, 0x38144, 4650 0x38190, 0x381d0, 4651 0x38200, 0x38318, 4652 0x38400, 0x3852c, 4653 0x38540, 0x3861c, 4654 0x38800, 0x38834, 4655 0x388c0, 0x38908, 4656 0x38910, 0x389ac, 4657 0x38a00, 0x38a2c, 4658 0x38a44, 0x38a50, 4659 0x38a74, 0x38c24, 4660 0x38d00, 0x38d00, 4661 0x38d08, 0x38d14, 4662 0x38d1c, 0x38d20, 4663 0x38d3c, 0x38d50, 4664 0x39200, 0x3920c, 4665 0x39220, 0x39220, 4666 0x39240, 0x39240, 4667 0x39600, 0x3960c, 4668 0x39a00, 0x39a1c, 4669 0x39e00, 0x39e20, 4670 0x39e38, 0x39e3c, 4671 0x39e80, 0x39e80, 4672 0x39e88, 0x39ea8, 4673 0x39eb0, 0x39eb4, 4674 0x39ec8, 0x39ed4, 4675 0x39fb8, 0x3a004, 4676 0x3a200, 0x3a200, 4677 0x3a208, 0x3a240, 4678 0x3a248, 0x3a280, 4679 0x3a288, 0x3a2c0, 4680 0x3a2c8, 0x3a2fc, 4681 0x3a600, 0x3a630, 4682 0x3aa00, 0x3aabc, 4683 0x3ab00, 0x3ab70, 4684 0x3b000, 0x3b048, 4685 0x3b060, 0x3b09c, 4686 0x3b0f0, 0x3b148, 4687 0x3b160, 0x3b19c, 4688 0x3b1f0, 0x3b2e4, 4689 0x3b2f8, 0x3b3e4, 4690 0x3b3f8, 0x3b448, 4691 0x3b460, 0x3b49c, 4692 0x3b4f0, 0x3b548, 4693 0x3b560, 0x3b59c, 4694 0x3b5f0, 0x3b6e4, 4695 0x3b6f8, 0x3b7e4, 4696 0x3b7f8, 0x3b7fc, 4697 0x3b814, 0x3b814, 4698 0x3b82c, 0x3b82c, 4699 0x3b880, 0x3b88c, 4700 0x3b8e8, 0x3b8ec, 4701 0x3b900, 0x3b948, 4702 0x3b960, 0x3b99c, 4703 0x3b9f0, 0x3bae4, 4704 0x3baf8, 0x3bb10, 4705 0x3bb28, 0x3bb28, 4706 0x3bb3c, 0x3bb50, 4707 0x3bbf0, 0x3bc10, 4708 0x3bc28, 0x3bc28, 4709 0x3bc3c, 0x3bc50, 4710 0x3bcf0, 0x3bcfc, 4711 0x3c000, 0x3c030, 4712 0x3c100, 0x3c144, 4713 0x3c190, 0x3c1d0, 4714 0x3c200, 0x3c318, 4715 0x3c400, 0x3c52c, 4716 0x3c540, 0x3c61c, 4717 0x3c800, 0x3c834, 4718 0x3c8c0, 0x3c908, 4719 0x3c910, 0x3c9ac, 4720 0x3ca00, 0x3ca2c, 4721 0x3ca44, 0x3ca50, 4722 0x3ca74, 0x3cc24, 4723 0x3cd00, 0x3cd00, 4724 0x3cd08, 0x3cd14, 4725 0x3cd1c, 0x3cd20, 4726 0x3cd3c, 0x3cd50, 4727 0x3d200, 0x3d20c, 4728 0x3d220, 0x3d220, 4729 0x3d240, 0x3d240, 4730 0x3d600, 0x3d60c, 4731 0x3da00, 0x3da1c, 4732 0x3de00, 0x3de20, 4733 0x3de38, 0x3de3c, 4734 0x3de80, 0x3de80, 4735 0x3de88, 0x3dea8, 4736 0x3deb0, 0x3deb4, 4737 0x3dec8, 0x3ded4, 4738 0x3dfb8, 0x3e004, 4739 0x3e200, 0x3e200, 4740 0x3e208, 0x3e240, 4741 0x3e248, 0x3e280, 4742 0x3e288, 0x3e2c0, 4743 0x3e2c8, 0x3e2fc, 4744 0x3e600, 0x3e630, 4745 0x3ea00, 0x3eabc, 4746 0x3eb00, 0x3eb70, 4747 0x3f000, 0x3f048, 4748 0x3f060, 0x3f09c, 4749 0x3f0f0, 0x3f148, 4750 0x3f160, 0x3f19c, 4751 0x3f1f0, 0x3f2e4, 4752 0x3f2f8, 0x3f3e4, 4753 0x3f3f8, 0x3f448, 4754 0x3f460, 0x3f49c, 4755 0x3f4f0, 0x3f548, 4756 0x3f560, 0x3f59c, 4757 0x3f5f0, 0x3f6e4, 4758 0x3f6f8, 0x3f7e4, 4759 0x3f7f8, 0x3f7fc, 4760 0x3f814, 0x3f814, 4761 0x3f82c, 0x3f82c, 4762 0x3f880, 0x3f88c, 4763 0x3f8e8, 0x3f8ec, 4764 0x3f900, 0x3f948, 4765 0x3f960, 0x3f99c, 4766 0x3f9f0, 0x3fae4, 4767 0x3faf8, 0x3fb10, 4768 0x3fb28, 0x3fb28, 4769 0x3fb3c, 0x3fb50, 4770 0x3fbf0, 0x3fc10, 4771 0x3fc28, 0x3fc28, 4772 0x3fc3c, 0x3fc50, 4773 0x3fcf0, 0x3fcfc, 4774 0x40000, 0x4000c, 4775 0x40040, 0x40068, 4776 0x4007c, 0x40144, 4777 0x40180, 0x4018c, 4778 0x40200, 0x40298, 4779 0x402ac, 0x4033c, 4780 0x403f8, 0x403fc, 4781 0x41304, 0x413c4, 4782 0x41400, 0x4141c, 4783 0x41480, 0x414d0, 4784 0x44000, 0x44078, 4785 0x440c0, 0x44278, 4786 0x442c0, 0x44478, 4787 0x444c0, 0x44678, 4788 0x446c0, 0x44878, 4789 0x448c0, 0x449fc, 4790 0x45000, 0x45068, 4791 0x45080, 0x45084, 4792 0x450a0, 0x450b0, 4793 0x45200, 0x45268, 4794 0x45280, 0x45284, 4795 0x452a0, 0x452b0, 4796 0x460c0, 0x460e4, 4797 0x47000, 0x4708c, 4798 0x47200, 0x47250, 4799 0x47400, 0x47420, 4800 0x47600, 0x47618, 4801 0x47800, 0x47814, 4802 0x48000, 0x4800c, 4803 0x48040, 0x48068, 4804 0x4807c, 0x48144, 4805 0x48180, 0x4818c, 4806 0x48200, 0x48298, 4807 0x482ac, 0x4833c, 4808 0x483f8, 0x483fc, 4809 0x49304, 0x493c4, 4810 0x49400, 0x4941c, 4811 0x49480, 0x494d0, 4812 0x4c000, 0x4c078, 4813 0x4c0c0, 0x4c278, 4814 0x4c2c0, 0x4c478, 4815 0x4c4c0, 0x4c678, 4816 0x4c6c0, 0x4c878, 4817 0x4c8c0, 0x4c9fc, 4818 0x4d000, 0x4d068, 4819 0x4d080, 0x4d084, 4820 0x4d0a0, 0x4d0b0, 4821 0x4d200, 0x4d268, 4822 0x4d280, 0x4d284, 4823 0x4d2a0, 0x4d2b0, 4824 0x4e0c0, 0x4e0e4, 4825 0x4f000, 0x4f08c, 4826 0x4f200, 0x4f250, 4827 0x4f400, 0x4f420, 4828 0x4f600, 0x4f618, 4829 0x4f800, 0x4f814, 4830 0x50000, 0x500cc, 4831 0x50400, 0x50400, 4832 0x50800, 0x508cc, 4833 0x50c00, 0x50c00, 4834 0x51000, 0x5101c, 4835 0x51300, 0x51308, 4836 }; 4837 4838 if (is_t4(sc)) { 4839 reg_ranges = &t4_reg_ranges[0]; 4840 n = nitems(t4_reg_ranges); 4841 } else { 4842 reg_ranges = &t5_reg_ranges[0]; 4843 n = nitems(t5_reg_ranges); 4844 } 4845 4846 regs->version = chip_id(sc) | chip_rev(sc) << 10; 4847 for (i = 0; i < n; i += 2) 4848 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 4849 } 4850 4851 #define A_PL_INDIR_CMD 0x1f8 4852 4853 #define S_PL_AUTOINC 31 4854 #define M_PL_AUTOINC 0x1U 4855 #define V_PL_AUTOINC(x) ((x) << S_PL_AUTOINC) 4856 #define G_PL_AUTOINC(x) (((x) >> S_PL_AUTOINC) & M_PL_AUTOINC) 4857 4858 #define S_PL_VFID 20 4859 #define M_PL_VFID 0xffU 4860 #define V_PL_VFID(x) ((x) << S_PL_VFID) 4861 #define G_PL_VFID(x) (((x) >> S_PL_VFID) & M_PL_VFID) 4862 4863 #define S_PL_ADDR 0 4864 #define M_PL_ADDR 0xfffffU 4865 #define V_PL_ADDR(x) ((x) << S_PL_ADDR) 4866 #define G_PL_ADDR(x) (((x) >> S_PL_ADDR) & M_PL_ADDR) 4867 4868 #define A_PL_INDIR_DATA 0x1fc 4869 4870 static uint64_t 4871 read_vf_stat(struct adapter *sc, unsigned int viid, int reg) 4872 { 4873 u32 stats[2]; 4874 4875 mtx_assert(&sc->regwin_lock, MA_OWNED); 4876 t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) | 4877 V_PL_VFID(G_FW_VIID_VIN(viid)) | V_PL_ADDR(VF_MPS_REG(reg))); 4878 stats[0] = t4_read_reg(sc, A_PL_INDIR_DATA); 4879 stats[1] = t4_read_reg(sc, A_PL_INDIR_DATA); 4880 return (((uint64_t)stats[1]) << 32 | stats[0]); 4881 } 4882 4883 static void 4884 t4_get_vi_stats(struct adapter *sc, unsigned int viid, 4885 struct fw_vi_stats_vf *stats) 4886 { 4887 4888 #define GET_STAT(name) \ 4889 read_vf_stat(sc, viid, A_MPS_VF_STAT_##name##_L) 4890 4891 stats->tx_bcast_bytes = GET_STAT(TX_VF_BCAST_BYTES); 4892 stats->tx_bcast_frames = GET_STAT(TX_VF_BCAST_FRAMES); 4893 stats->tx_mcast_bytes = GET_STAT(TX_VF_MCAST_BYTES); 4894 stats->tx_mcast_frames = GET_STAT(TX_VF_MCAST_FRAMES); 4895 stats->tx_ucast_bytes = GET_STAT(TX_VF_UCAST_BYTES); 4896 stats->tx_ucast_frames = GET_STAT(TX_VF_UCAST_FRAMES); 4897 stats->tx_drop_frames = GET_STAT(TX_VF_DROP_FRAMES); 4898 stats->tx_offload_bytes = GET_STAT(TX_VF_OFFLOAD_BYTES); 4899 stats->tx_offload_frames = GET_STAT(TX_VF_OFFLOAD_FRAMES); 4900 stats->rx_bcast_bytes = GET_STAT(RX_VF_BCAST_BYTES); 4901 stats->rx_bcast_frames = GET_STAT(RX_VF_BCAST_FRAMES); 4902 stats->rx_mcast_bytes = GET_STAT(RX_VF_MCAST_BYTES); 4903 stats->rx_mcast_frames = GET_STAT(RX_VF_MCAST_FRAMES); 4904 stats->rx_ucast_bytes = GET_STAT(RX_VF_UCAST_BYTES); 4905 stats->rx_ucast_frames = GET_STAT(RX_VF_UCAST_FRAMES); 4906 stats->rx_err_frames = GET_STAT(RX_VF_ERR_FRAMES); 4907 4908 #undef GET_STAT 4909 } 4910 4911 static void 4912 t4_clr_vi_stats(struct adapter *sc, unsigned int viid) 4913 { 4914 int reg; 4915 4916 t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) | 4917 V_PL_VFID(G_FW_VIID_VIN(viid)) | 4918 V_PL_ADDR(VF_MPS_REG(A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L))); 4919 for (reg = A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L; 4920 reg <= A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H; reg += 4) 4921 t4_write_reg(sc, A_PL_INDIR_DATA, 0); 4922 } 4923 4924 static void 4925 vi_refresh_stats(struct adapter *sc, struct vi_info *vi) 4926 { 4927 struct timeval tv; 4928 const struct timeval interval = {0, 250000}; /* 250ms */ 4929 4930 if (!(vi->flags & VI_INIT_DONE)) 4931 return; 4932 4933 getmicrotime(&tv); 4934 timevalsub(&tv, &interval); 4935 if (timevalcmp(&tv, &vi->last_refreshed, <)) 4936 return; 4937 4938 mtx_lock(&sc->regwin_lock); 4939 t4_get_vi_stats(sc, vi->viid, &vi->stats); 4940 getmicrotime(&vi->last_refreshed); 4941 mtx_unlock(&sc->regwin_lock); 4942 } 4943 4944 static void 4945 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi) 4946 { 4947 int i; 4948 u_int v, tnl_cong_drops; 4949 struct timeval tv; 4950 const struct timeval interval = {0, 250000}; /* 250ms */ 4951 4952 getmicrotime(&tv); 4953 timevalsub(&tv, &interval); 4954 if (timevalcmp(&tv, &pi->last_refreshed, <)) 4955 return; 4956 4957 tnl_cong_drops = 0; 4958 t4_get_port_stats(sc, pi->tx_chan, &pi->stats); 4959 for (i = 0; i < NCHAN; i++) { 4960 if (pi->rx_chan_map & (1 << i)) { 4961 mtx_lock(&sc->regwin_lock); 4962 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 4963 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 4964 mtx_unlock(&sc->regwin_lock); 4965 tnl_cong_drops += v; 4966 } 4967 } 4968 pi->tnl_cong_drops = tnl_cong_drops; 4969 getmicrotime(&pi->last_refreshed); 4970 } 4971 4972 static void 4973 cxgbe_tick(void *arg) 4974 { 4975 struct port_info *pi = arg; 4976 struct adapter *sc = pi->adapter; 4977 4978 PORT_LOCK_ASSERT_OWNED(pi); 4979 cxgbe_refresh_stats(sc, pi); 4980 4981 callout_schedule(&pi->tick, hz); 4982 } 4983 4984 void 4985 vi_tick(void *arg) 4986 { 4987 struct vi_info *vi = arg; 4988 struct adapter *sc = vi->pi->adapter; 4989 4990 vi_refresh_stats(sc, vi); 4991 4992 callout_schedule(&vi->tick, hz); 4993 } 4994 4995 static void 4996 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 4997 { 4998 struct ifnet *vlan; 4999 5000 if (arg != ifp || ifp->if_type != IFT_ETHER) 5001 return; 5002 5003 vlan = VLAN_DEVAT(ifp, vid); 5004 VLAN_SETCOOKIE(vlan, ifp); 5005 } 5006 5007 static int 5008 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 5009 { 5010 5011 #ifdef INVARIANTS 5012 panic("%s: opcode 0x%02x on iq %p with payload %p", 5013 __func__, rss->opcode, iq, m); 5014 #else 5015 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 5016 __func__, rss->opcode, iq, m); 5017 m_freem(m); 5018 #endif 5019 return (EDOOFUS); 5020 } 5021 5022 int 5023 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 5024 { 5025 uintptr_t *loc, new; 5026 5027 if (opcode >= nitems(sc->cpl_handler)) 5028 return (EINVAL); 5029 5030 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 5031 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 5032 atomic_store_rel_ptr(loc, new); 5033 5034 return (0); 5035 } 5036 5037 static int 5038 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 5039 { 5040 5041 #ifdef INVARIANTS 5042 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 5043 #else 5044 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 5045 __func__, iq, ctrl); 5046 #endif 5047 return (EDOOFUS); 5048 } 5049 5050 int 5051 t4_register_an_handler(struct adapter *sc, an_handler_t h) 5052 { 5053 uintptr_t *loc, new; 5054 5055 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 5056 loc = (uintptr_t *) &sc->an_handler; 5057 atomic_store_rel_ptr(loc, new); 5058 5059 return (0); 5060 } 5061 5062 static int 5063 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 5064 { 5065 const struct cpl_fw6_msg *cpl = 5066 __containerof(rpl, struct cpl_fw6_msg, data[0]); 5067 5068 #ifdef INVARIANTS 5069 panic("%s: fw_msg type %d", __func__, cpl->type); 5070 #else 5071 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 5072 #endif 5073 return (EDOOFUS); 5074 } 5075 5076 int 5077 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 5078 { 5079 uintptr_t *loc, new; 5080 5081 if (type >= nitems(sc->fw_msg_handler)) 5082 return (EINVAL); 5083 5084 /* 5085 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 5086 * handler dispatch table. Reject any attempt to install a handler for 5087 * this subtype. 5088 */ 5089 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 5090 return (EINVAL); 5091 5092 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 5093 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 5094 atomic_store_rel_ptr(loc, new); 5095 5096 return (0); 5097 } 5098 5099 static void 5100 t4_sysctls(struct adapter *sc) 5101 { 5102 struct sysctl_ctx_list *ctx; 5103 struct sysctl_oid *oid; 5104 struct sysctl_oid_list *children, *c0; 5105 static char *caps[] = { 5106 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 5107 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL" /* caps[1] niccaps */ 5108 "\6HASHFILTER\7ETHOFLD", 5109 "\20\1TOE", /* caps[2] toecaps */ 5110 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 5111 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 5112 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 5113 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 5114 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 5115 "\4PO_INITIAOR\5PO_TARGET" 5116 }; 5117 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 5118 5119 ctx = device_get_sysctl_ctx(sc->dev); 5120 5121 /* 5122 * dev.t4nex.X. 5123 */ 5124 oid = device_get_sysctl_tree(sc->dev); 5125 c0 = children = SYSCTL_CHILDREN(oid); 5126 5127 sc->sc_do_rxcopy = 1; 5128 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, 5129 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); 5130 5131 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 5132 sc->params.nports, "# of ports"); 5133 5134 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 5135 NULL, chip_rev(sc), "chip hardware revision"); 5136 5137 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 5138 CTLFLAG_RD, sc->fw_version, 0, "firmware version"); 5139 5140 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 5141 CTLFLAG_RD, sc->cfg_file, 0, "configuration file"); 5142 5143 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 5144 sc->cfcsum, "config file checksum"); 5145 5146 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 5147 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 5148 sysctl_bitfield, "A", "available doorbells"); 5149 5150 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 5151 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 5152 sysctl_bitfield, "A", "available link capabilities"); 5153 5154 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 5155 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 5156 sysctl_bitfield, "A", "available NIC capabilities"); 5157 5158 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 5159 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 5160 sysctl_bitfield, "A", "available TCP offload capabilities"); 5161 5162 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 5163 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 5164 sysctl_bitfield, "A", "available RDMA capabilities"); 5165 5166 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 5167 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 5168 sysctl_bitfield, "A", "available iSCSI capabilities"); 5169 5170 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 5171 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 5172 sysctl_bitfield, "A", "available FCoE capabilities"); 5173 5174 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 5175 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 5176 5177 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 5178 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 5179 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 5180 "interrupt holdoff timer values (us)"); 5181 5182 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 5183 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 5184 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 5185 "interrupt holdoff packet counter values"); 5186 5187 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, 5188 NULL, sc->tids.nftids, "number of filters"); 5189 5190 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | 5191 CTLFLAG_RD, sc, 0, sysctl_temperature, "I", 5192 "chip temperature (in Celsius)"); 5193 5194 t4_sge_sysctls(sc, ctx, children); 5195 5196 sc->lro_timeout = 100; 5197 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, 5198 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); 5199 5200 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "debug_flags", CTLFLAG_RW, 5201 &sc->debug_flags, 0, "flags to enable runtime debugging"); 5202 5203 #ifdef SBUF_DRAIN 5204 /* 5205 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 5206 */ 5207 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 5208 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 5209 "logs and miscellaneous information"); 5210 children = SYSCTL_CHILDREN(oid); 5211 5212 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 5213 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5214 sysctl_cctrl, "A", "congestion control"); 5215 5216 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 5217 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5218 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 5219 5220 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 5221 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 5222 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 5223 5224 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 5225 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 5226 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 5227 5228 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 5229 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 5230 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 5231 5232 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 5233 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 5234 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 5235 5236 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 5237 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 5238 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 5239 5240 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 5241 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5242 sysctl_cim_la, "A", "CIM logic analyzer"); 5243 5244 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", 5245 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5246 sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); 5247 5248 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 5249 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 5250 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 5251 5252 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 5253 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 5254 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 5255 5256 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 5257 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 5258 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 5259 5260 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 5261 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 5262 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 5263 5264 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 5265 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 5266 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 5267 5268 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 5269 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 5270 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 5271 5272 if (is_t5(sc)) { 5273 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 5274 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 5275 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 5276 5277 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 5278 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 5279 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 5280 } 5281 5282 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", 5283 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5284 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); 5285 5286 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 5287 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5288 sysctl_cim_qcfg, "A", "CIM queue configuration"); 5289 5290 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 5291 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5292 sysctl_cpl_stats, "A", "CPL statistics"); 5293 5294 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 5295 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5296 sysctl_ddp_stats, "A", "non-TCP DDP statistics"); 5297 5298 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 5299 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5300 sysctl_devlog, "A", "firmware's device log"); 5301 5302 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 5303 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5304 sysctl_fcoe_stats, "A", "FCoE statistics"); 5305 5306 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 5307 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5308 sysctl_hw_sched, "A", "hardware scheduler "); 5309 5310 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 5311 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5312 sysctl_l2t, "A", "hardware L2 table"); 5313 5314 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 5315 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5316 sysctl_lb_stats, "A", "loopback statistics"); 5317 5318 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 5319 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5320 sysctl_meminfo, "A", "memory regions"); 5321 5322 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", 5323 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5324 sysctl_mps_tcam, "A", "MPS TCAM entries"); 5325 5326 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 5327 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5328 sysctl_path_mtus, "A", "path MTUs"); 5329 5330 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 5331 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5332 sysctl_pm_stats, "A", "PM statistics"); 5333 5334 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 5335 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5336 sysctl_rdma_stats, "A", "RDMA statistics"); 5337 5338 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 5339 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5340 sysctl_tcp_stats, "A", "TCP statistics"); 5341 5342 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 5343 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5344 sysctl_tids, "A", "TID information"); 5345 5346 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 5347 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5348 sysctl_tp_err_stats, "A", "TP error statistics"); 5349 5350 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", 5351 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5352 sysctl_tp_la, "A", "TP logic analyzer"); 5353 5354 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 5355 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5356 sysctl_tx_rate, "A", "Tx rate"); 5357 5358 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", 5359 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5360 sysctl_ulprx_la, "A", "ULPRX logic analyzer"); 5361 5362 if (is_t5(sc)) { 5363 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 5364 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5365 sysctl_wcwr_stats, "A", "write combined work requests"); 5366 } 5367 #endif 5368 5369 #ifdef TCP_OFFLOAD 5370 if (is_offload(sc)) { 5371 /* 5372 * dev.t4nex.X.toe. 5373 */ 5374 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 5375 NULL, "TOE parameters"); 5376 children = SYSCTL_CHILDREN(oid); 5377 5378 sc->tt.sndbuf = 256 * 1024; 5379 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 5380 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 5381 5382 sc->tt.ddp = 0; 5383 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 5384 &sc->tt.ddp, 0, "DDP allowed"); 5385 5386 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 5387 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 5388 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 5389 5390 sc->tt.ddp_thres = 5391 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 5392 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 5393 &sc->tt.ddp_thres, 0, "DDP threshold"); 5394 5395 sc->tt.rx_coalesce = 1; 5396 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", 5397 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); 5398 5399 sc->tt.tx_align = 1; 5400 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align", 5401 CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload"); 5402 } 5403 #endif 5404 } 5405 5406 void 5407 vi_sysctls(struct vi_info *vi) 5408 { 5409 struct sysctl_ctx_list *ctx; 5410 struct sysctl_oid *oid; 5411 struct sysctl_oid_list *children; 5412 5413 ctx = device_get_sysctl_ctx(vi->dev); 5414 5415 /* 5416 * dev.[nv](cxgbe|cxl).X. 5417 */ 5418 oid = device_get_sysctl_tree(vi->dev); 5419 children = SYSCTL_CHILDREN(oid); 5420 5421 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "viid", CTLFLAG_RD, NULL, 5422 vi->viid, "VI identifer"); 5423 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 5424 &vi->nrxq, 0, "# of rx queues"); 5425 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 5426 &vi->ntxq, 0, "# of tx queues"); 5427 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 5428 &vi->first_rxq, 0, "index of first rx queue"); 5429 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 5430 &vi->first_txq, 0, "index of first tx queue"); 5431 5432 if (vi->flags & VI_NETMAP) 5433 return; 5434 5435 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT | 5436 CTLFLAG_RW, vi, 0, sysctl_noflowq, "IU", 5437 "Reserve queue 0 for non-flowid packets"); 5438 5439 #ifdef TCP_OFFLOAD 5440 if (vi->nofldrxq != 0) { 5441 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 5442 &vi->nofldrxq, 0, 5443 "# of rx queues for offloaded TCP connections"); 5444 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 5445 &vi->nofldtxq, 0, 5446 "# of tx queues for offloaded TCP connections"); 5447 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 5448 CTLFLAG_RD, &vi->first_ofld_rxq, 0, 5449 "index of first TOE rx queue"); 5450 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 5451 CTLFLAG_RD, &vi->first_ofld_txq, 0, 5452 "index of first TOE tx queue"); 5453 } 5454 #endif 5455 5456 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 5457 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_holdoff_tmr_idx, "I", 5458 "holdoff timer index"); 5459 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 5460 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_holdoff_pktc_idx, "I", 5461 "holdoff packet counter index"); 5462 5463 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 5464 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_qsize_rxq, "I", 5465 "rx queue size"); 5466 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 5467 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_qsize_txq, "I", 5468 "tx queue size"); 5469 } 5470 5471 static void 5472 cxgbe_sysctls(struct port_info *pi) 5473 { 5474 struct sysctl_ctx_list *ctx; 5475 struct sysctl_oid *oid; 5476 struct sysctl_oid_list *children; 5477 struct adapter *sc = pi->adapter; 5478 5479 ctx = device_get_sysctl_ctx(pi->dev); 5480 5481 /* 5482 * dev.cxgbe.X. 5483 */ 5484 oid = device_get_sysctl_tree(pi->dev); 5485 children = SYSCTL_CHILDREN(oid); 5486 5487 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | 5488 CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); 5489 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { 5490 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 5491 CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I", 5492 "PHY temperature (in Celsius)"); 5493 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", 5494 CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I", 5495 "PHY firmware version"); 5496 } 5497 5498 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings", 5499 CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings, 5500 "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)"); 5501 5502 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "max_speed", CTLFLAG_RD, NULL, 5503 port_top_speed(pi), "max speed (in Gbps)"); 5504 5505 /* 5506 * dev.cxgbe.X.stats. 5507 */ 5508 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 5509 NULL, "port statistics"); 5510 children = SYSCTL_CHILDREN(oid); 5511 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD, 5512 &pi->tx_parse_error, 0, 5513 "# of tx packets with invalid length or # of segments"); 5514 5515 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 5516 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 5517 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ 5518 sysctl_handle_t4_reg64, "QU", desc) 5519 5520 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 5521 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 5522 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 5523 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 5524 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 5525 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 5526 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 5527 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 5528 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 5529 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 5530 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 5531 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 5532 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 5533 "# of tx frames in this range", 5534 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 5535 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 5536 "# of tx frames in this range", 5537 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 5538 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 5539 "# of tx frames in this range", 5540 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 5541 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 5542 "# of tx frames in this range", 5543 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 5544 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 5545 "# of tx frames in this range", 5546 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 5547 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 5548 "# of tx frames in this range", 5549 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 5550 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 5551 "# of tx frames in this range", 5552 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 5553 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 5554 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 5555 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 5556 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 5557 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 5558 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 5559 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 5560 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 5561 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 5562 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 5563 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 5564 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 5565 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 5566 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 5567 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 5568 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 5569 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 5570 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 5571 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 5572 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 5573 5574 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 5575 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 5576 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 5577 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 5578 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 5579 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 5580 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 5581 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 5582 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 5583 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 5584 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 5585 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 5586 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 5587 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 5588 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 5589 "# of frames received with bad FCS", 5590 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 5591 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 5592 "# of frames received with length error", 5593 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 5594 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 5595 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 5596 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 5597 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 5598 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 5599 "# of rx frames in this range", 5600 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 5601 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 5602 "# of rx frames in this range", 5603 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 5604 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 5605 "# of rx frames in this range", 5606 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 5607 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 5608 "# of rx frames in this range", 5609 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 5610 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 5611 "# of rx frames in this range", 5612 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 5613 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 5614 "# of rx frames in this range", 5615 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 5616 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 5617 "# of rx frames in this range", 5618 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 5619 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 5620 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 5621 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 5622 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 5623 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 5624 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 5625 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 5626 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 5627 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 5628 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 5629 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 5630 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 5631 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 5632 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 5633 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 5634 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 5635 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 5636 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 5637 5638 #undef SYSCTL_ADD_T4_REG64 5639 5640 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 5641 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 5642 &pi->stats.name, desc) 5643 5644 /* We get these from port_stats and they may be stale by upto 1s */ 5645 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 5646 "# drops due to buffer-group 0 overflows"); 5647 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 5648 "# drops due to buffer-group 1 overflows"); 5649 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 5650 "# drops due to buffer-group 2 overflows"); 5651 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 5652 "# drops due to buffer-group 3 overflows"); 5653 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 5654 "# of buffer-group 0 truncated packets"); 5655 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 5656 "# of buffer-group 1 truncated packets"); 5657 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 5658 "# of buffer-group 2 truncated packets"); 5659 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 5660 "# of buffer-group 3 truncated packets"); 5661 5662 #undef SYSCTL_ADD_T4_PORTSTAT 5663 } 5664 5665 static int 5666 sysctl_int_array(SYSCTL_HANDLER_ARGS) 5667 { 5668 int rc, *i, space = 0; 5669 struct sbuf sb; 5670 5671 sbuf_new_for_sysctl(&sb, NULL, 64, req); 5672 for (i = arg1; arg2; arg2 -= sizeof(int), i++) { 5673 if (space) 5674 sbuf_printf(&sb, " "); 5675 sbuf_printf(&sb, "%d", *i); 5676 space = 1; 5677 } 5678 rc = sbuf_finish(&sb); 5679 sbuf_delete(&sb); 5680 return (rc); 5681 } 5682 5683 static int 5684 sysctl_bitfield(SYSCTL_HANDLER_ARGS) 5685 { 5686 int rc; 5687 struct sbuf *sb; 5688 5689 rc = sysctl_wire_old_buffer(req, 0); 5690 if (rc != 0) 5691 return(rc); 5692 5693 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5694 if (sb == NULL) 5695 return (ENOMEM); 5696 5697 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 5698 rc = sbuf_finish(sb); 5699 sbuf_delete(sb); 5700 5701 return (rc); 5702 } 5703 5704 static int 5705 sysctl_btphy(SYSCTL_HANDLER_ARGS) 5706 { 5707 struct port_info *pi = arg1; 5708 int op = arg2; 5709 struct adapter *sc = pi->adapter; 5710 u_int v; 5711 int rc; 5712 5713 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, "t4btt"); 5714 if (rc) 5715 return (rc); 5716 /* XXX: magic numbers */ 5717 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, 5718 &v); 5719 end_synchronized_op(sc, 0); 5720 if (rc) 5721 return (rc); 5722 if (op == 0) 5723 v /= 256; 5724 5725 rc = sysctl_handle_int(oidp, &v, 0, req); 5726 return (rc); 5727 } 5728 5729 static int 5730 sysctl_noflowq(SYSCTL_HANDLER_ARGS) 5731 { 5732 struct vi_info *vi = arg1; 5733 int rc, val; 5734 5735 val = vi->rsrv_noflowq; 5736 rc = sysctl_handle_int(oidp, &val, 0, req); 5737 if (rc != 0 || req->newptr == NULL) 5738 return (rc); 5739 5740 if ((val >= 1) && (vi->ntxq > 1)) 5741 vi->rsrv_noflowq = 1; 5742 else 5743 vi->rsrv_noflowq = 0; 5744 5745 return (rc); 5746 } 5747 5748 static int 5749 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 5750 { 5751 struct vi_info *vi = arg1; 5752 struct adapter *sc = vi->pi->adapter; 5753 int idx, rc, i; 5754 struct sge_rxq *rxq; 5755 #ifdef TCP_OFFLOAD 5756 struct sge_ofld_rxq *ofld_rxq; 5757 #endif 5758 uint8_t v; 5759 5760 idx = vi->tmr_idx; 5761 5762 rc = sysctl_handle_int(oidp, &idx, 0, req); 5763 if (rc != 0 || req->newptr == NULL) 5764 return (rc); 5765 5766 if (idx < 0 || idx >= SGE_NTIMERS) 5767 return (EINVAL); 5768 5769 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5770 "t4tmr"); 5771 if (rc) 5772 return (rc); 5773 5774 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(vi->pktc_idx != -1); 5775 for_each_rxq(vi, i, rxq) { 5776 #ifdef atomic_store_rel_8 5777 atomic_store_rel_8(&rxq->iq.intr_params, v); 5778 #else 5779 rxq->iq.intr_params = v; 5780 #endif 5781 } 5782 #ifdef TCP_OFFLOAD 5783 for_each_ofld_rxq(vi, i, ofld_rxq) { 5784 #ifdef atomic_store_rel_8 5785 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); 5786 #else 5787 ofld_rxq->iq.intr_params = v; 5788 #endif 5789 } 5790 #endif 5791 vi->tmr_idx = idx; 5792 5793 end_synchronized_op(sc, LOCK_HELD); 5794 return (0); 5795 } 5796 5797 static int 5798 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 5799 { 5800 struct vi_info *vi = arg1; 5801 struct adapter *sc = vi->pi->adapter; 5802 int idx, rc; 5803 5804 idx = vi->pktc_idx; 5805 5806 rc = sysctl_handle_int(oidp, &idx, 0, req); 5807 if (rc != 0 || req->newptr == NULL) 5808 return (rc); 5809 5810 if (idx < -1 || idx >= SGE_NCOUNTERS) 5811 return (EINVAL); 5812 5813 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5814 "t4pktc"); 5815 if (rc) 5816 return (rc); 5817 5818 if (vi->flags & VI_INIT_DONE) 5819 rc = EBUSY; /* cannot be changed once the queues are created */ 5820 else 5821 vi->pktc_idx = idx; 5822 5823 end_synchronized_op(sc, LOCK_HELD); 5824 return (rc); 5825 } 5826 5827 static int 5828 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 5829 { 5830 struct vi_info *vi = arg1; 5831 struct adapter *sc = vi->pi->adapter; 5832 int qsize, rc; 5833 5834 qsize = vi->qsize_rxq; 5835 5836 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5837 if (rc != 0 || req->newptr == NULL) 5838 return (rc); 5839 5840 if (qsize < 128 || (qsize & 7)) 5841 return (EINVAL); 5842 5843 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5844 "t4rxqs"); 5845 if (rc) 5846 return (rc); 5847 5848 if (vi->flags & VI_INIT_DONE) 5849 rc = EBUSY; /* cannot be changed once the queues are created */ 5850 else 5851 vi->qsize_rxq = qsize; 5852 5853 end_synchronized_op(sc, LOCK_HELD); 5854 return (rc); 5855 } 5856 5857 static int 5858 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 5859 { 5860 struct vi_info *vi = arg1; 5861 struct adapter *sc = vi->pi->adapter; 5862 int qsize, rc; 5863 5864 qsize = vi->qsize_txq; 5865 5866 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5867 if (rc != 0 || req->newptr == NULL) 5868 return (rc); 5869 5870 if (qsize < 128 || qsize > 65536) 5871 return (EINVAL); 5872 5873 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5874 "t4txqs"); 5875 if (rc) 5876 return (rc); 5877 5878 if (vi->flags & VI_INIT_DONE) 5879 rc = EBUSY; /* cannot be changed once the queues are created */ 5880 else 5881 vi->qsize_txq = qsize; 5882 5883 end_synchronized_op(sc, LOCK_HELD); 5884 return (rc); 5885 } 5886 5887 static int 5888 sysctl_pause_settings(SYSCTL_HANDLER_ARGS) 5889 { 5890 struct port_info *pi = arg1; 5891 struct adapter *sc = pi->adapter; 5892 struct link_config *lc = &pi->link_cfg; 5893 int rc; 5894 5895 if (req->newptr == NULL) { 5896 struct sbuf *sb; 5897 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX"; 5898 5899 rc = sysctl_wire_old_buffer(req, 0); 5900 if (rc != 0) 5901 return(rc); 5902 5903 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5904 if (sb == NULL) 5905 return (ENOMEM); 5906 5907 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits); 5908 rc = sbuf_finish(sb); 5909 sbuf_delete(sb); 5910 } else { 5911 char s[2]; 5912 int n; 5913 5914 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX)); 5915 s[1] = 0; 5916 5917 rc = sysctl_handle_string(oidp, s, sizeof(s), req); 5918 if (rc != 0) 5919 return(rc); 5920 5921 if (s[1] != 0) 5922 return (EINVAL); 5923 if (s[0] < '0' || s[0] > '9') 5924 return (EINVAL); /* not a number */ 5925 n = s[0] - '0'; 5926 if (n & ~(PAUSE_TX | PAUSE_RX)) 5927 return (EINVAL); /* some other bit is set too */ 5928 5929 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, 5930 "t4PAUSE"); 5931 if (rc) 5932 return (rc); 5933 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) { 5934 int link_ok = lc->link_ok; 5935 5936 lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX); 5937 lc->requested_fc |= n; 5938 rc = -t4_link_l1cfg(sc, sc->mbox, pi->tx_chan, lc); 5939 lc->link_ok = link_ok; /* restore */ 5940 } 5941 end_synchronized_op(sc, 0); 5942 } 5943 5944 return (rc); 5945 } 5946 5947 static int 5948 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 5949 { 5950 struct adapter *sc = arg1; 5951 int reg = arg2; 5952 uint64_t val; 5953 5954 val = t4_read_reg64(sc, reg); 5955 5956 return (sysctl_handle_64(oidp, &val, 0, req)); 5957 } 5958 5959 static int 5960 sysctl_temperature(SYSCTL_HANDLER_ARGS) 5961 { 5962 struct adapter *sc = arg1; 5963 int rc, t; 5964 uint32_t param, val; 5965 5966 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); 5967 if (rc) 5968 return (rc); 5969 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 5970 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 5971 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); 5972 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 5973 end_synchronized_op(sc, 0); 5974 if (rc) 5975 return (rc); 5976 5977 /* unknown is returned as 0 but we display -1 in that case */ 5978 t = val == 0 ? -1 : val; 5979 5980 rc = sysctl_handle_int(oidp, &t, 0, req); 5981 return (rc); 5982 } 5983 5984 #ifdef SBUF_DRAIN 5985 static int 5986 sysctl_cctrl(SYSCTL_HANDLER_ARGS) 5987 { 5988 struct adapter *sc = arg1; 5989 struct sbuf *sb; 5990 int rc, i; 5991 uint16_t incr[NMTUS][NCCTRL_WIN]; 5992 static const char *dec_fac[] = { 5993 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 5994 "0.9375" 5995 }; 5996 5997 rc = sysctl_wire_old_buffer(req, 0); 5998 if (rc != 0) 5999 return (rc); 6000 6001 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6002 if (sb == NULL) 6003 return (ENOMEM); 6004 6005 t4_read_cong_tbl(sc, incr); 6006 6007 for (i = 0; i < NCCTRL_WIN; ++i) { 6008 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 6009 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 6010 incr[5][i], incr[6][i], incr[7][i]); 6011 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 6012 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 6013 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 6014 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 6015 } 6016 6017 rc = sbuf_finish(sb); 6018 sbuf_delete(sb); 6019 6020 return (rc); 6021 } 6022 6023 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 6024 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 6025 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 6026 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 6027 }; 6028 6029 static int 6030 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 6031 { 6032 struct adapter *sc = arg1; 6033 struct sbuf *sb; 6034 int rc, i, n, qid = arg2; 6035 uint32_t *buf, *p; 6036 char *qtype; 6037 u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5; 6038 6039 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 6040 ("%s: bad qid %d\n", __func__, qid)); 6041 6042 if (qid < CIM_NUM_IBQ) { 6043 /* inbound queue */ 6044 qtype = "IBQ"; 6045 n = 4 * CIM_IBQ_SIZE; 6046 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 6047 rc = t4_read_cim_ibq(sc, qid, buf, n); 6048 } else { 6049 /* outbound queue */ 6050 qtype = "OBQ"; 6051 qid -= CIM_NUM_IBQ; 6052 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 6053 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 6054 rc = t4_read_cim_obq(sc, qid, buf, n); 6055 } 6056 6057 if (rc < 0) { 6058 rc = -rc; 6059 goto done; 6060 } 6061 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 6062 6063 rc = sysctl_wire_old_buffer(req, 0); 6064 if (rc != 0) 6065 goto done; 6066 6067 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 6068 if (sb == NULL) { 6069 rc = ENOMEM; 6070 goto done; 6071 } 6072 6073 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 6074 for (i = 0, p = buf; i < n; i += 16, p += 4) 6075 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 6076 p[2], p[3]); 6077 6078 rc = sbuf_finish(sb); 6079 sbuf_delete(sb); 6080 done: 6081 free(buf, M_CXGBE); 6082 return (rc); 6083 } 6084 6085 static int 6086 sysctl_cim_la(SYSCTL_HANDLER_ARGS) 6087 { 6088 struct adapter *sc = arg1; 6089 u_int cfg; 6090 struct sbuf *sb; 6091 uint32_t *buf, *p; 6092 int rc; 6093 6094 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 6095 if (rc != 0) 6096 return (rc); 6097 6098 rc = sysctl_wire_old_buffer(req, 0); 6099 if (rc != 0) 6100 return (rc); 6101 6102 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6103 if (sb == NULL) 6104 return (ENOMEM); 6105 6106 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 6107 M_ZERO | M_WAITOK); 6108 6109 rc = -t4_cim_read_la(sc, buf, NULL); 6110 if (rc != 0) 6111 goto done; 6112 6113 sbuf_printf(sb, "Status Data PC%s", 6114 cfg & F_UPDBGLACAPTPCONLY ? "" : 6115 " LS0Stat LS0Addr LS0Data"); 6116 6117 KASSERT((sc->params.cim_la_size & 7) == 0, 6118 ("%s: p will walk off the end of buf", __func__)); 6119 6120 for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) { 6121 if (cfg & F_UPDBGLACAPTPCONLY) { 6122 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 6123 p[6], p[7]); 6124 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 6125 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 6126 p[4] & 0xff, p[5] >> 8); 6127 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 6128 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 6129 p[1] & 0xf, p[2] >> 4); 6130 } else { 6131 sbuf_printf(sb, 6132 "\n %02x %x%07x %x%07x %08x %08x " 6133 "%08x%08x%08x%08x", 6134 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 6135 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 6136 p[6], p[7]); 6137 } 6138 } 6139 6140 rc = sbuf_finish(sb); 6141 sbuf_delete(sb); 6142 done: 6143 free(buf, M_CXGBE); 6144 return (rc); 6145 } 6146 6147 static int 6148 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) 6149 { 6150 struct adapter *sc = arg1; 6151 u_int i; 6152 struct sbuf *sb; 6153 uint32_t *buf, *p; 6154 int rc; 6155 6156 rc = sysctl_wire_old_buffer(req, 0); 6157 if (rc != 0) 6158 return (rc); 6159 6160 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6161 if (sb == NULL) 6162 return (ENOMEM); 6163 6164 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, 6165 M_ZERO | M_WAITOK); 6166 6167 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); 6168 p = buf; 6169 6170 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 6171 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], 6172 p[1], p[0]); 6173 } 6174 6175 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); 6176 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 6177 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", 6178 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, 6179 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, 6180 (p[1] >> 2) | ((p[2] & 3) << 30), 6181 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, 6182 p[0] & 1); 6183 } 6184 6185 rc = sbuf_finish(sb); 6186 sbuf_delete(sb); 6187 free(buf, M_CXGBE); 6188 return (rc); 6189 } 6190 6191 static int 6192 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) 6193 { 6194 struct adapter *sc = arg1; 6195 u_int i; 6196 struct sbuf *sb; 6197 uint32_t *buf, *p; 6198 int rc; 6199 6200 rc = sysctl_wire_old_buffer(req, 0); 6201 if (rc != 0) 6202 return (rc); 6203 6204 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6205 if (sb == NULL) 6206 return (ENOMEM); 6207 6208 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, 6209 M_ZERO | M_WAITOK); 6210 6211 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); 6212 p = buf; 6213 6214 sbuf_printf(sb, "Cntl ID DataBE Addr Data"); 6215 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 6216 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", 6217 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, 6218 p[4], p[3], p[2], p[1], p[0]); 6219 } 6220 6221 sbuf_printf(sb, "\n\nCntl ID Data"); 6222 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 6223 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", 6224 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); 6225 } 6226 6227 rc = sbuf_finish(sb); 6228 sbuf_delete(sb); 6229 free(buf, M_CXGBE); 6230 return (rc); 6231 } 6232 6233 static int 6234 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 6235 { 6236 struct adapter *sc = arg1; 6237 struct sbuf *sb; 6238 int rc, i; 6239 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 6240 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 6241 uint16_t thres[CIM_NUM_IBQ]; 6242 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 6243 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 6244 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 6245 6246 if (is_t4(sc)) { 6247 cim_num_obq = CIM_NUM_OBQ; 6248 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 6249 obq_rdaddr = A_UP_OBQ_0_REALADDR; 6250 } else { 6251 cim_num_obq = CIM_NUM_OBQ_T5; 6252 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 6253 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 6254 } 6255 nq = CIM_NUM_IBQ + cim_num_obq; 6256 6257 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 6258 if (rc == 0) 6259 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 6260 if (rc != 0) 6261 return (rc); 6262 6263 t4_read_cimq_cfg(sc, base, size, thres); 6264 6265 rc = sysctl_wire_old_buffer(req, 0); 6266 if (rc != 0) 6267 return (rc); 6268 6269 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 6270 if (sb == NULL) 6271 return (ENOMEM); 6272 6273 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 6274 6275 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 6276 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 6277 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 6278 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 6279 G_QUEREMFLITS(p[2]) * 16); 6280 for ( ; i < nq; i++, p += 4, wr += 2) 6281 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 6282 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 6283 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 6284 G_QUEREMFLITS(p[2]) * 16); 6285 6286 rc = sbuf_finish(sb); 6287 sbuf_delete(sb); 6288 6289 return (rc); 6290 } 6291 6292 static int 6293 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 6294 { 6295 struct adapter *sc = arg1; 6296 struct sbuf *sb; 6297 int rc; 6298 struct tp_cpl_stats stats; 6299 6300 rc = sysctl_wire_old_buffer(req, 0); 6301 if (rc != 0) 6302 return (rc); 6303 6304 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6305 if (sb == NULL) 6306 return (ENOMEM); 6307 6308 t4_tp_get_cpl_stats(sc, &stats); 6309 6310 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6311 "channel 3\n"); 6312 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 6313 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 6314 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 6315 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 6316 6317 rc = sbuf_finish(sb); 6318 sbuf_delete(sb); 6319 6320 return (rc); 6321 } 6322 6323 static int 6324 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 6325 { 6326 struct adapter *sc = arg1; 6327 struct sbuf *sb; 6328 int rc; 6329 struct tp_usm_stats stats; 6330 6331 rc = sysctl_wire_old_buffer(req, 0); 6332 if (rc != 0) 6333 return(rc); 6334 6335 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6336 if (sb == NULL) 6337 return (ENOMEM); 6338 6339 t4_get_usm_stats(sc, &stats); 6340 6341 sbuf_printf(sb, "Frames: %u\n", stats.frames); 6342 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 6343 sbuf_printf(sb, "Drops: %u", stats.drops); 6344 6345 rc = sbuf_finish(sb); 6346 sbuf_delete(sb); 6347 6348 return (rc); 6349 } 6350 6351 const char *devlog_level_strings[] = { 6352 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 6353 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 6354 [FW_DEVLOG_LEVEL_ERR] = "ERR", 6355 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 6356 [FW_DEVLOG_LEVEL_INFO] = "INFO", 6357 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 6358 }; 6359 6360 const char *devlog_facility_strings[] = { 6361 [FW_DEVLOG_FACILITY_CORE] = "CORE", 6362 [FW_DEVLOG_FACILITY_CF] = "CF", 6363 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 6364 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 6365 [FW_DEVLOG_FACILITY_RES] = "RES", 6366 [FW_DEVLOG_FACILITY_HW] = "HW", 6367 [FW_DEVLOG_FACILITY_FLR] = "FLR", 6368 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 6369 [FW_DEVLOG_FACILITY_PHY] = "PHY", 6370 [FW_DEVLOG_FACILITY_MAC] = "MAC", 6371 [FW_DEVLOG_FACILITY_PORT] = "PORT", 6372 [FW_DEVLOG_FACILITY_VI] = "VI", 6373 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 6374 [FW_DEVLOG_FACILITY_ACL] = "ACL", 6375 [FW_DEVLOG_FACILITY_TM] = "TM", 6376 [FW_DEVLOG_FACILITY_QFC] = "QFC", 6377 [FW_DEVLOG_FACILITY_DCB] = "DCB", 6378 [FW_DEVLOG_FACILITY_ETH] = "ETH", 6379 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 6380 [FW_DEVLOG_FACILITY_RI] = "RI", 6381 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 6382 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 6383 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 6384 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 6385 }; 6386 6387 static int 6388 sysctl_devlog(SYSCTL_HANDLER_ARGS) 6389 { 6390 struct adapter *sc = arg1; 6391 struct devlog_params *dparams = &sc->params.devlog; 6392 struct fw_devlog_e *buf, *e; 6393 int i, j, rc, nentries, first = 0, m; 6394 struct sbuf *sb; 6395 uint64_t ftstamp = UINT64_MAX; 6396 6397 if (dparams->start == 0) { 6398 dparams->memtype = FW_MEMTYPE_EDC0; 6399 dparams->start = 0x84000; 6400 dparams->size = 32768; 6401 } 6402 6403 nentries = dparams->size / sizeof(struct fw_devlog_e); 6404 6405 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 6406 if (buf == NULL) 6407 return (ENOMEM); 6408 6409 m = fwmtype_to_hwmtype(dparams->memtype); 6410 rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf); 6411 if (rc != 0) 6412 goto done; 6413 6414 for (i = 0; i < nentries; i++) { 6415 e = &buf[i]; 6416 6417 if (e->timestamp == 0) 6418 break; /* end */ 6419 6420 e->timestamp = be64toh(e->timestamp); 6421 e->seqno = be32toh(e->seqno); 6422 for (j = 0; j < 8; j++) 6423 e->params[j] = be32toh(e->params[j]); 6424 6425 if (e->timestamp < ftstamp) { 6426 ftstamp = e->timestamp; 6427 first = i; 6428 } 6429 } 6430 6431 if (buf[first].timestamp == 0) 6432 goto done; /* nothing in the log */ 6433 6434 rc = sysctl_wire_old_buffer(req, 0); 6435 if (rc != 0) 6436 goto done; 6437 6438 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6439 if (sb == NULL) { 6440 rc = ENOMEM; 6441 goto done; 6442 } 6443 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 6444 "Seq#", "Tstamp", "Level", "Facility", "Message"); 6445 6446 i = first; 6447 do { 6448 e = &buf[i]; 6449 if (e->timestamp == 0) 6450 break; /* end */ 6451 6452 sbuf_printf(sb, "%10d %15ju %8s %8s ", 6453 e->seqno, e->timestamp, 6454 (e->level < nitems(devlog_level_strings) ? 6455 devlog_level_strings[e->level] : "UNKNOWN"), 6456 (e->facility < nitems(devlog_facility_strings) ? 6457 devlog_facility_strings[e->facility] : "UNKNOWN")); 6458 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 6459 e->params[2], e->params[3], e->params[4], 6460 e->params[5], e->params[6], e->params[7]); 6461 6462 if (++i == nentries) 6463 i = 0; 6464 } while (i != first); 6465 6466 rc = sbuf_finish(sb); 6467 sbuf_delete(sb); 6468 done: 6469 free(buf, M_CXGBE); 6470 return (rc); 6471 } 6472 6473 static int 6474 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 6475 { 6476 struct adapter *sc = arg1; 6477 struct sbuf *sb; 6478 int rc; 6479 struct tp_fcoe_stats stats[4]; 6480 6481 rc = sysctl_wire_old_buffer(req, 0); 6482 if (rc != 0) 6483 return (rc); 6484 6485 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6486 if (sb == NULL) 6487 return (ENOMEM); 6488 6489 t4_get_fcoe_stats(sc, 0, &stats[0]); 6490 t4_get_fcoe_stats(sc, 1, &stats[1]); 6491 t4_get_fcoe_stats(sc, 2, &stats[2]); 6492 t4_get_fcoe_stats(sc, 3, &stats[3]); 6493 6494 sbuf_printf(sb, " channel 0 channel 1 " 6495 "channel 2 channel 3\n"); 6496 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 6497 stats[0].octets_ddp, stats[1].octets_ddp, stats[2].octets_ddp, 6498 stats[3].octets_ddp); 6499 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", 6500 stats[0].frames_ddp, stats[1].frames_ddp, stats[2].frames_ddp, 6501 stats[3].frames_ddp); 6502 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 6503 stats[0].frames_drop, stats[1].frames_drop, stats[2].frames_drop, 6504 stats[3].frames_drop); 6505 6506 rc = sbuf_finish(sb); 6507 sbuf_delete(sb); 6508 6509 return (rc); 6510 } 6511 6512 static int 6513 sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 6514 { 6515 struct adapter *sc = arg1; 6516 struct sbuf *sb; 6517 int rc, i; 6518 unsigned int map, kbps, ipg, mode; 6519 unsigned int pace_tab[NTX_SCHED]; 6520 6521 rc = sysctl_wire_old_buffer(req, 0); 6522 if (rc != 0) 6523 return (rc); 6524 6525 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6526 if (sb == NULL) 6527 return (ENOMEM); 6528 6529 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 6530 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 6531 t4_read_pace_tbl(sc, pace_tab); 6532 6533 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 6534 "Class IPG (0.1 ns) Flow IPG (us)"); 6535 6536 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 6537 t4_get_tx_sched(sc, i, &kbps, &ipg); 6538 sbuf_printf(sb, "\n %u %-5s %u ", i, 6539 (mode & (1 << i)) ? "flow" : "class", map & 3); 6540 if (kbps) 6541 sbuf_printf(sb, "%9u ", kbps); 6542 else 6543 sbuf_printf(sb, " disabled "); 6544 6545 if (ipg) 6546 sbuf_printf(sb, "%13u ", ipg); 6547 else 6548 sbuf_printf(sb, " disabled "); 6549 6550 if (pace_tab[i]) 6551 sbuf_printf(sb, "%10u", pace_tab[i]); 6552 else 6553 sbuf_printf(sb, " disabled"); 6554 } 6555 6556 rc = sbuf_finish(sb); 6557 sbuf_delete(sb); 6558 6559 return (rc); 6560 } 6561 6562 static int 6563 sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 6564 { 6565 struct adapter *sc = arg1; 6566 struct sbuf *sb; 6567 int rc, i, j; 6568 uint64_t *p0, *p1; 6569 struct lb_port_stats s[2]; 6570 static const char *stat_name[] = { 6571 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 6572 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 6573 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 6574 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 6575 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 6576 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 6577 "BG2FramesTrunc:", "BG3FramesTrunc:" 6578 }; 6579 6580 rc = sysctl_wire_old_buffer(req, 0); 6581 if (rc != 0) 6582 return (rc); 6583 6584 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6585 if (sb == NULL) 6586 return (ENOMEM); 6587 6588 memset(s, 0, sizeof(s)); 6589 6590 for (i = 0; i < 4; i += 2) { 6591 t4_get_lb_stats(sc, i, &s[0]); 6592 t4_get_lb_stats(sc, i + 1, &s[1]); 6593 6594 p0 = &s[0].octets; 6595 p1 = &s[1].octets; 6596 sbuf_printf(sb, "%s Loopback %u" 6597 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 6598 6599 for (j = 0; j < nitems(stat_name); j++) 6600 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 6601 *p0++, *p1++); 6602 } 6603 6604 rc = sbuf_finish(sb); 6605 sbuf_delete(sb); 6606 6607 return (rc); 6608 } 6609 6610 static int 6611 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) 6612 { 6613 int rc = 0; 6614 struct port_info *pi = arg1; 6615 struct sbuf *sb; 6616 static const char *linkdnreasons[] = { 6617 "non-specific", "remote fault", "autoneg failed", "reserved3", 6618 "PHY overheated", "unknown", "rx los", "reserved7" 6619 }; 6620 6621 rc = sysctl_wire_old_buffer(req, 0); 6622 if (rc != 0) 6623 return(rc); 6624 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); 6625 if (sb == NULL) 6626 return (ENOMEM); 6627 6628 if (pi->linkdnrc < 0) 6629 sbuf_printf(sb, "n/a"); 6630 else if (pi->linkdnrc < nitems(linkdnreasons)) 6631 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]); 6632 else 6633 sbuf_printf(sb, "%d", pi->linkdnrc); 6634 6635 rc = sbuf_finish(sb); 6636 sbuf_delete(sb); 6637 6638 return (rc); 6639 } 6640 6641 struct mem_desc { 6642 unsigned int base; 6643 unsigned int limit; 6644 unsigned int idx; 6645 }; 6646 6647 static int 6648 mem_desc_cmp(const void *a, const void *b) 6649 { 6650 return ((const struct mem_desc *)a)->base - 6651 ((const struct mem_desc *)b)->base; 6652 } 6653 6654 static void 6655 mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 6656 unsigned int to) 6657 { 6658 unsigned int size; 6659 6660 size = to - from + 1; 6661 if (size == 0) 6662 return; 6663 6664 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 6665 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 6666 } 6667 6668 static int 6669 sysctl_meminfo(SYSCTL_HANDLER_ARGS) 6670 { 6671 struct adapter *sc = arg1; 6672 struct sbuf *sb; 6673 int rc, i, n; 6674 uint32_t lo, hi, used, alloc; 6675 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 6676 static const char *region[] = { 6677 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 6678 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 6679 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 6680 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 6681 "RQUDP region:", "PBL region:", "TXPBL region:", 6682 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 6683 "On-chip queues:" 6684 }; 6685 struct mem_desc avail[4]; 6686 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 6687 struct mem_desc *md = mem; 6688 6689 rc = sysctl_wire_old_buffer(req, 0); 6690 if (rc != 0) 6691 return (rc); 6692 6693 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6694 if (sb == NULL) 6695 return (ENOMEM); 6696 6697 for (i = 0; i < nitems(mem); i++) { 6698 mem[i].limit = 0; 6699 mem[i].idx = i; 6700 } 6701 6702 /* Find and sort the populated memory ranges */ 6703 i = 0; 6704 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 6705 if (lo & F_EDRAM0_ENABLE) { 6706 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 6707 avail[i].base = G_EDRAM0_BASE(hi) << 20; 6708 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 6709 avail[i].idx = 0; 6710 i++; 6711 } 6712 if (lo & F_EDRAM1_ENABLE) { 6713 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 6714 avail[i].base = G_EDRAM1_BASE(hi) << 20; 6715 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 6716 avail[i].idx = 1; 6717 i++; 6718 } 6719 if (lo & F_EXT_MEM_ENABLE) { 6720 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 6721 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 6722 avail[i].limit = avail[i].base + 6723 (G_EXT_MEM_SIZE(hi) << 20); 6724 avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */ 6725 i++; 6726 } 6727 if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) { 6728 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 6729 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 6730 avail[i].limit = avail[i].base + 6731 (G_EXT_MEM1_SIZE(hi) << 20); 6732 avail[i].idx = 4; 6733 i++; 6734 } 6735 if (!i) /* no memory available */ 6736 return 0; 6737 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 6738 6739 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 6740 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 6741 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 6742 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 6743 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 6744 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 6745 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 6746 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 6747 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 6748 6749 /* the next few have explicit upper bounds */ 6750 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 6751 md->limit = md->base - 1 + 6752 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 6753 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 6754 md++; 6755 6756 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 6757 md->limit = md->base - 1 + 6758 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 6759 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 6760 md++; 6761 6762 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6763 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 6764 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 6765 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 6766 } else { 6767 md->base = 0; 6768 md->idx = nitems(region); /* hide it */ 6769 } 6770 md++; 6771 6772 #define ulp_region(reg) \ 6773 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 6774 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 6775 6776 ulp_region(RX_ISCSI); 6777 ulp_region(RX_TDDP); 6778 ulp_region(TX_TPT); 6779 ulp_region(RX_STAG); 6780 ulp_region(RX_RQ); 6781 ulp_region(RX_RQUDP); 6782 ulp_region(RX_PBL); 6783 ulp_region(TX_PBL); 6784 #undef ulp_region 6785 6786 md->base = 0; 6787 md->idx = nitems(region); 6788 if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) { 6789 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); 6790 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc, 6791 A_SGE_DBVFIFO_SIZE))) << 2) - 1; 6792 } 6793 md++; 6794 6795 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 6796 md->limit = md->base + sc->tids.ntids - 1; 6797 md++; 6798 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 6799 md->limit = md->base + sc->tids.ntids - 1; 6800 md++; 6801 6802 md->base = sc->vres.ocq.start; 6803 if (sc->vres.ocq.size) 6804 md->limit = md->base + sc->vres.ocq.size - 1; 6805 else 6806 md->idx = nitems(region); /* hide it */ 6807 md++; 6808 6809 /* add any address-space holes, there can be up to 3 */ 6810 for (n = 0; n < i - 1; n++) 6811 if (avail[n].limit < avail[n + 1].base) 6812 (md++)->base = avail[n].limit; 6813 if (avail[n].limit) 6814 (md++)->base = avail[n].limit; 6815 6816 n = md - mem; 6817 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 6818 6819 for (lo = 0; lo < i; lo++) 6820 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 6821 avail[lo].limit - 1); 6822 6823 sbuf_printf(sb, "\n"); 6824 for (i = 0; i < n; i++) { 6825 if (mem[i].idx >= nitems(region)) 6826 continue; /* skip holes */ 6827 if (!mem[i].limit) 6828 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 6829 mem_region_show(sb, region[mem[i].idx], mem[i].base, 6830 mem[i].limit); 6831 } 6832 6833 sbuf_printf(sb, "\n"); 6834 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 6835 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 6836 mem_region_show(sb, "uP RAM:", lo, hi); 6837 6838 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 6839 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 6840 mem_region_show(sb, "uP Extmem2:", lo, hi); 6841 6842 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 6843 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 6844 G_PMRXMAXPAGE(lo), 6845 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 6846 (lo & F_PMRXNUMCHN) ? 2 : 1); 6847 6848 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 6849 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 6850 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 6851 G_PMTXMAXPAGE(lo), 6852 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 6853 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 6854 sbuf_printf(sb, "%u p-structs\n", 6855 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 6856 6857 for (i = 0; i < 4; i++) { 6858 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 6859 if (is_t4(sc)) { 6860 used = G_USED(lo); 6861 alloc = G_ALLOC(lo); 6862 } else { 6863 used = G_T5_USED(lo); 6864 alloc = G_T5_ALLOC(lo); 6865 } 6866 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 6867 i, used, alloc); 6868 } 6869 for (i = 0; i < 4; i++) { 6870 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 6871 if (is_t4(sc)) { 6872 used = G_USED(lo); 6873 alloc = G_ALLOC(lo); 6874 } else { 6875 used = G_T5_USED(lo); 6876 alloc = G_T5_ALLOC(lo); 6877 } 6878 sbuf_printf(sb, 6879 "\nLoopback %d using %u pages out of %u allocated", 6880 i, used, alloc); 6881 } 6882 6883 rc = sbuf_finish(sb); 6884 sbuf_delete(sb); 6885 6886 return (rc); 6887 } 6888 6889 static inline void 6890 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) 6891 { 6892 *mask = x | y; 6893 y = htobe64(y); 6894 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); 6895 } 6896 6897 static int 6898 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) 6899 { 6900 struct adapter *sc = arg1; 6901 struct sbuf *sb; 6902 int rc, i, n; 6903 6904 rc = sysctl_wire_old_buffer(req, 0); 6905 if (rc != 0) 6906 return (rc); 6907 6908 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6909 if (sb == NULL) 6910 return (ENOMEM); 6911 6912 sbuf_printf(sb, 6913 "Idx Ethernet address Mask Vld Ports PF" 6914 " VF Replication P0 P1 P2 P3 ML"); 6915 n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES : 6916 NUM_MPS_T5_CLS_SRAM_L_INSTANCES; 6917 for (i = 0; i < n; i++) { 6918 uint64_t tcamx, tcamy, mask; 6919 uint32_t cls_lo, cls_hi; 6920 uint8_t addr[ETHER_ADDR_LEN]; 6921 6922 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); 6923 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); 6924 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 6925 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 6926 6927 if (tcamx & tcamy) 6928 continue; 6929 6930 tcamxy2valmask(tcamx, tcamy, addr, &mask); 6931 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" 6932 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], 6933 addr[3], addr[4], addr[5], (uintmax_t)mask, 6934 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', 6935 G_PORTMAP(cls_hi), G_PF(cls_lo), 6936 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); 6937 6938 if (cls_lo & F_REPLICATE) { 6939 struct fw_ldst_cmd ldst_cmd; 6940 6941 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 6942 ldst_cmd.op_to_addrspace = 6943 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 6944 F_FW_CMD_REQUEST | F_FW_CMD_READ | 6945 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 6946 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 6947 ldst_cmd.u.mps.rplc.fid_idx = 6948 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 6949 V_FW_LDST_CMD_IDX(i)); 6950 6951 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 6952 "t4mps"); 6953 if (rc) 6954 break; 6955 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 6956 sizeof(ldst_cmd), &ldst_cmd); 6957 end_synchronized_op(sc, 0); 6958 6959 if (rc != 0) { 6960 sbuf_printf(sb, 6961 " ------------ error %3u ------------", rc); 6962 rc = 0; 6963 } else { 6964 sbuf_printf(sb, " %08x %08x %08x %08x", 6965 be32toh(ldst_cmd.u.mps.rplc.rplc127_96), 6966 be32toh(ldst_cmd.u.mps.rplc.rplc95_64), 6967 be32toh(ldst_cmd.u.mps.rplc.rplc63_32), 6968 be32toh(ldst_cmd.u.mps.rplc.rplc31_0)); 6969 } 6970 } else 6971 sbuf_printf(sb, "%36s", ""); 6972 6973 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), 6974 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), 6975 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); 6976 } 6977 6978 if (rc) 6979 (void) sbuf_finish(sb); 6980 else 6981 rc = sbuf_finish(sb); 6982 sbuf_delete(sb); 6983 6984 return (rc); 6985 } 6986 6987 static int 6988 sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 6989 { 6990 struct adapter *sc = arg1; 6991 struct sbuf *sb; 6992 int rc; 6993 uint16_t mtus[NMTUS]; 6994 6995 rc = sysctl_wire_old_buffer(req, 0); 6996 if (rc != 0) 6997 return (rc); 6998 6999 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7000 if (sb == NULL) 7001 return (ENOMEM); 7002 7003 t4_read_mtu_tbl(sc, mtus, NULL); 7004 7005 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 7006 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 7007 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 7008 mtus[14], mtus[15]); 7009 7010 rc = sbuf_finish(sb); 7011 sbuf_delete(sb); 7012 7013 return (rc); 7014 } 7015 7016 static int 7017 sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 7018 { 7019 struct adapter *sc = arg1; 7020 struct sbuf *sb; 7021 int rc, i; 7022 uint32_t cnt[PM_NSTATS]; 7023 uint64_t cyc[PM_NSTATS]; 7024 static const char *rx_stats[] = { 7025 "Read:", "Write bypass:", "Write mem:", "Flush:" 7026 }; 7027 static const char *tx_stats[] = { 7028 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:" 7029 }; 7030 7031 rc = sysctl_wire_old_buffer(req, 0); 7032 if (rc != 0) 7033 return (rc); 7034 7035 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7036 if (sb == NULL) 7037 return (ENOMEM); 7038 7039 t4_pmtx_get_stats(sc, cnt, cyc); 7040 sbuf_printf(sb, " Tx pcmds Tx bytes"); 7041 for (i = 0; i < ARRAY_SIZE(tx_stats); i++) 7042 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i], 7043 cyc[i]); 7044 7045 t4_pmrx_get_stats(sc, cnt, cyc); 7046 sbuf_printf(sb, "\n Rx pcmds Rx bytes"); 7047 for (i = 0; i < ARRAY_SIZE(rx_stats); i++) 7048 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i], 7049 cyc[i]); 7050 7051 rc = sbuf_finish(sb); 7052 sbuf_delete(sb); 7053 7054 return (rc); 7055 } 7056 7057 static int 7058 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 7059 { 7060 struct adapter *sc = arg1; 7061 struct sbuf *sb; 7062 int rc; 7063 struct tp_rdma_stats stats; 7064 7065 rc = sysctl_wire_old_buffer(req, 0); 7066 if (rc != 0) 7067 return (rc); 7068 7069 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7070 if (sb == NULL) 7071 return (ENOMEM); 7072 7073 t4_tp_get_rdma_stats(sc, &stats); 7074 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 7075 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 7076 7077 rc = sbuf_finish(sb); 7078 sbuf_delete(sb); 7079 7080 return (rc); 7081 } 7082 7083 static int 7084 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 7085 { 7086 struct adapter *sc = arg1; 7087 struct sbuf *sb; 7088 int rc; 7089 struct tp_tcp_stats v4, v6; 7090 7091 rc = sysctl_wire_old_buffer(req, 0); 7092 if (rc != 0) 7093 return (rc); 7094 7095 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7096 if (sb == NULL) 7097 return (ENOMEM); 7098 7099 t4_tp_get_tcp_stats(sc, &v4, &v6); 7100 sbuf_printf(sb, 7101 " IP IPv6\n"); 7102 sbuf_printf(sb, "OutRsts: %20u %20u\n", 7103 v4.tcp_out_rsts, v6.tcp_out_rsts); 7104 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 7105 v4.tcp_in_segs, v6.tcp_in_segs); 7106 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 7107 v4.tcp_out_segs, v6.tcp_out_segs); 7108 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 7109 v4.tcp_retrans_segs, v6.tcp_retrans_segs); 7110 7111 rc = sbuf_finish(sb); 7112 sbuf_delete(sb); 7113 7114 return (rc); 7115 } 7116 7117 static int 7118 sysctl_tids(SYSCTL_HANDLER_ARGS) 7119 { 7120 struct adapter *sc = arg1; 7121 struct sbuf *sb; 7122 int rc; 7123 struct tid_info *t = &sc->tids; 7124 7125 rc = sysctl_wire_old_buffer(req, 0); 7126 if (rc != 0) 7127 return (rc); 7128 7129 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7130 if (sb == NULL) 7131 return (ENOMEM); 7132 7133 if (t->natids) { 7134 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 7135 t->atids_in_use); 7136 } 7137 7138 if (t->ntids) { 7139 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 7140 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 7141 7142 if (b) { 7143 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 7144 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 7145 t->ntids - 1); 7146 } else { 7147 sbuf_printf(sb, "TID range: %u-%u", 7148 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 7149 t->ntids - 1); 7150 } 7151 } else 7152 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 7153 sbuf_printf(sb, ", in use: %u\n", 7154 atomic_load_acq_int(&t->tids_in_use)); 7155 } 7156 7157 if (t->nstids) { 7158 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 7159 t->stid_base + t->nstids - 1, t->stids_in_use); 7160 } 7161 7162 if (t->nftids) { 7163 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 7164 t->ftid_base + t->nftids - 1); 7165 } 7166 7167 if (t->netids) { 7168 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base, 7169 t->etid_base + t->netids - 1); 7170 } 7171 7172 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 7173 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 7174 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 7175 7176 rc = sbuf_finish(sb); 7177 sbuf_delete(sb); 7178 7179 return (rc); 7180 } 7181 7182 static int 7183 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 7184 { 7185 struct adapter *sc = arg1; 7186 struct sbuf *sb; 7187 int rc; 7188 struct tp_err_stats stats; 7189 7190 rc = sysctl_wire_old_buffer(req, 0); 7191 if (rc != 0) 7192 return (rc); 7193 7194 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7195 if (sb == NULL) 7196 return (ENOMEM); 7197 7198 t4_tp_get_err_stats(sc, &stats); 7199 7200 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 7201 "channel 3\n"); 7202 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 7203 stats.mac_in_errs[0], stats.mac_in_errs[1], stats.mac_in_errs[2], 7204 stats.mac_in_errs[3]); 7205 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 7206 stats.hdr_in_errs[0], stats.hdr_in_errs[1], stats.hdr_in_errs[2], 7207 stats.hdr_in_errs[3]); 7208 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 7209 stats.tcp_in_errs[0], stats.tcp_in_errs[1], stats.tcp_in_errs[2], 7210 stats.tcp_in_errs[3]); 7211 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 7212 stats.tcp6_in_errs[0], stats.tcp6_in_errs[1], stats.tcp6_in_errs[2], 7213 stats.tcp6_in_errs[3]); 7214 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 7215 stats.tnl_cong_drops[0], stats.tnl_cong_drops[1], 7216 stats.tnl_cong_drops[2], stats.tnl_cong_drops[3]); 7217 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 7218 stats.tnl_tx_drops[0], stats.tnl_tx_drops[1], stats.tnl_tx_drops[2], 7219 stats.tnl_tx_drops[3]); 7220 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 7221 stats.ofld_vlan_drops[0], stats.ofld_vlan_drops[1], 7222 stats.ofld_vlan_drops[2], stats.ofld_vlan_drops[3]); 7223 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 7224 stats.ofld_chan_drops[0], stats.ofld_chan_drops[1], 7225 stats.ofld_chan_drops[2], stats.ofld_chan_drops[3]); 7226 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 7227 stats.ofld_no_neigh, stats.ofld_cong_defer); 7228 7229 rc = sbuf_finish(sb); 7230 sbuf_delete(sb); 7231 7232 return (rc); 7233 } 7234 7235 struct field_desc { 7236 const char *name; 7237 u_int start; 7238 u_int width; 7239 }; 7240 7241 static void 7242 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) 7243 { 7244 char buf[32]; 7245 int line_size = 0; 7246 7247 while (f->name) { 7248 uint64_t mask = (1ULL << f->width) - 1; 7249 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, 7250 ((uintmax_t)v >> f->start) & mask); 7251 7252 if (line_size + len >= 79) { 7253 line_size = 8; 7254 sbuf_printf(sb, "\n "); 7255 } 7256 sbuf_printf(sb, "%s ", buf); 7257 line_size += len + 1; 7258 f++; 7259 } 7260 sbuf_printf(sb, "\n"); 7261 } 7262 7263 static struct field_desc tp_la0[] = { 7264 { "RcfOpCodeOut", 60, 4 }, 7265 { "State", 56, 4 }, 7266 { "WcfState", 52, 4 }, 7267 { "RcfOpcSrcOut", 50, 2 }, 7268 { "CRxError", 49, 1 }, 7269 { "ERxError", 48, 1 }, 7270 { "SanityFailed", 47, 1 }, 7271 { "SpuriousMsg", 46, 1 }, 7272 { "FlushInputMsg", 45, 1 }, 7273 { "FlushInputCpl", 44, 1 }, 7274 { "RssUpBit", 43, 1 }, 7275 { "RssFilterHit", 42, 1 }, 7276 { "Tid", 32, 10 }, 7277 { "InitTcb", 31, 1 }, 7278 { "LineNumber", 24, 7 }, 7279 { "Emsg", 23, 1 }, 7280 { "EdataOut", 22, 1 }, 7281 { "Cmsg", 21, 1 }, 7282 { "CdataOut", 20, 1 }, 7283 { "EreadPdu", 19, 1 }, 7284 { "CreadPdu", 18, 1 }, 7285 { "TunnelPkt", 17, 1 }, 7286 { "RcfPeerFin", 16, 1 }, 7287 { "RcfReasonOut", 12, 4 }, 7288 { "TxCchannel", 10, 2 }, 7289 { "RcfTxChannel", 8, 2 }, 7290 { "RxEchannel", 6, 2 }, 7291 { "RcfRxChannel", 5, 1 }, 7292 { "RcfDataOutSrdy", 4, 1 }, 7293 { "RxDvld", 3, 1 }, 7294 { "RxOoDvld", 2, 1 }, 7295 { "RxCongestion", 1, 1 }, 7296 { "TxCongestion", 0, 1 }, 7297 { NULL } 7298 }; 7299 7300 static struct field_desc tp_la1[] = { 7301 { "CplCmdIn", 56, 8 }, 7302 { "CplCmdOut", 48, 8 }, 7303 { "ESynOut", 47, 1 }, 7304 { "EAckOut", 46, 1 }, 7305 { "EFinOut", 45, 1 }, 7306 { "ERstOut", 44, 1 }, 7307 { "SynIn", 43, 1 }, 7308 { "AckIn", 42, 1 }, 7309 { "FinIn", 41, 1 }, 7310 { "RstIn", 40, 1 }, 7311 { "DataIn", 39, 1 }, 7312 { "DataInVld", 38, 1 }, 7313 { "PadIn", 37, 1 }, 7314 { "RxBufEmpty", 36, 1 }, 7315 { "RxDdp", 35, 1 }, 7316 { "RxFbCongestion", 34, 1 }, 7317 { "TxFbCongestion", 33, 1 }, 7318 { "TxPktSumSrdy", 32, 1 }, 7319 { "RcfUlpType", 28, 4 }, 7320 { "Eread", 27, 1 }, 7321 { "Ebypass", 26, 1 }, 7322 { "Esave", 25, 1 }, 7323 { "Static0", 24, 1 }, 7324 { "Cread", 23, 1 }, 7325 { "Cbypass", 22, 1 }, 7326 { "Csave", 21, 1 }, 7327 { "CPktOut", 20, 1 }, 7328 { "RxPagePoolFull", 18, 2 }, 7329 { "RxLpbkPkt", 17, 1 }, 7330 { "TxLpbkPkt", 16, 1 }, 7331 { "RxVfValid", 15, 1 }, 7332 { "SynLearned", 14, 1 }, 7333 { "SetDelEntry", 13, 1 }, 7334 { "SetInvEntry", 12, 1 }, 7335 { "CpcmdDvld", 11, 1 }, 7336 { "CpcmdSave", 10, 1 }, 7337 { "RxPstructsFull", 8, 2 }, 7338 { "EpcmdDvld", 7, 1 }, 7339 { "EpcmdFlush", 6, 1 }, 7340 { "EpcmdTrimPrefix", 5, 1 }, 7341 { "EpcmdTrimPostfix", 4, 1 }, 7342 { "ERssIp4Pkt", 3, 1 }, 7343 { "ERssIp6Pkt", 2, 1 }, 7344 { "ERssTcpUdpPkt", 1, 1 }, 7345 { "ERssFceFipPkt", 0, 1 }, 7346 { NULL } 7347 }; 7348 7349 static struct field_desc tp_la2[] = { 7350 { "CplCmdIn", 56, 8 }, 7351 { "MpsVfVld", 55, 1 }, 7352 { "MpsPf", 52, 3 }, 7353 { "MpsVf", 44, 8 }, 7354 { "SynIn", 43, 1 }, 7355 { "AckIn", 42, 1 }, 7356 { "FinIn", 41, 1 }, 7357 { "RstIn", 40, 1 }, 7358 { "DataIn", 39, 1 }, 7359 { "DataInVld", 38, 1 }, 7360 { "PadIn", 37, 1 }, 7361 { "RxBufEmpty", 36, 1 }, 7362 { "RxDdp", 35, 1 }, 7363 { "RxFbCongestion", 34, 1 }, 7364 { "TxFbCongestion", 33, 1 }, 7365 { "TxPktSumSrdy", 32, 1 }, 7366 { "RcfUlpType", 28, 4 }, 7367 { "Eread", 27, 1 }, 7368 { "Ebypass", 26, 1 }, 7369 { "Esave", 25, 1 }, 7370 { "Static0", 24, 1 }, 7371 { "Cread", 23, 1 }, 7372 { "Cbypass", 22, 1 }, 7373 { "Csave", 21, 1 }, 7374 { "CPktOut", 20, 1 }, 7375 { "RxPagePoolFull", 18, 2 }, 7376 { "RxLpbkPkt", 17, 1 }, 7377 { "TxLpbkPkt", 16, 1 }, 7378 { "RxVfValid", 15, 1 }, 7379 { "SynLearned", 14, 1 }, 7380 { "SetDelEntry", 13, 1 }, 7381 { "SetInvEntry", 12, 1 }, 7382 { "CpcmdDvld", 11, 1 }, 7383 { "CpcmdSave", 10, 1 }, 7384 { "RxPstructsFull", 8, 2 }, 7385 { "EpcmdDvld", 7, 1 }, 7386 { "EpcmdFlush", 6, 1 }, 7387 { "EpcmdTrimPrefix", 5, 1 }, 7388 { "EpcmdTrimPostfix", 4, 1 }, 7389 { "ERssIp4Pkt", 3, 1 }, 7390 { "ERssIp6Pkt", 2, 1 }, 7391 { "ERssTcpUdpPkt", 1, 1 }, 7392 { "ERssFceFipPkt", 0, 1 }, 7393 { NULL } 7394 }; 7395 7396 static void 7397 tp_la_show(struct sbuf *sb, uint64_t *p, int idx) 7398 { 7399 7400 field_desc_show(sb, *p, tp_la0); 7401 } 7402 7403 static void 7404 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) 7405 { 7406 7407 if (idx) 7408 sbuf_printf(sb, "\n"); 7409 field_desc_show(sb, p[0], tp_la0); 7410 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 7411 field_desc_show(sb, p[1], tp_la0); 7412 } 7413 7414 static void 7415 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) 7416 { 7417 7418 if (idx) 7419 sbuf_printf(sb, "\n"); 7420 field_desc_show(sb, p[0], tp_la0); 7421 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 7422 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); 7423 } 7424 7425 static int 7426 sysctl_tp_la(SYSCTL_HANDLER_ARGS) 7427 { 7428 struct adapter *sc = arg1; 7429 struct sbuf *sb; 7430 uint64_t *buf, *p; 7431 int rc; 7432 u_int i, inc; 7433 void (*show_func)(struct sbuf *, uint64_t *, int); 7434 7435 rc = sysctl_wire_old_buffer(req, 0); 7436 if (rc != 0) 7437 return (rc); 7438 7439 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7440 if (sb == NULL) 7441 return (ENOMEM); 7442 7443 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); 7444 7445 t4_tp_read_la(sc, buf, NULL); 7446 p = buf; 7447 7448 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { 7449 case 2: 7450 inc = 2; 7451 show_func = tp_la_show2; 7452 break; 7453 case 3: 7454 inc = 2; 7455 show_func = tp_la_show3; 7456 break; 7457 default: 7458 inc = 1; 7459 show_func = tp_la_show; 7460 } 7461 7462 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) 7463 (*show_func)(sb, p, i); 7464 7465 rc = sbuf_finish(sb); 7466 sbuf_delete(sb); 7467 free(buf, M_CXGBE); 7468 return (rc); 7469 } 7470 7471 static int 7472 sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 7473 { 7474 struct adapter *sc = arg1; 7475 struct sbuf *sb; 7476 int rc; 7477 u64 nrate[NCHAN], orate[NCHAN]; 7478 7479 rc = sysctl_wire_old_buffer(req, 0); 7480 if (rc != 0) 7481 return (rc); 7482 7483 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7484 if (sb == NULL) 7485 return (ENOMEM); 7486 7487 t4_get_chan_txrate(sc, nrate, orate); 7488 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 7489 "channel 3\n"); 7490 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 7491 nrate[0], nrate[1], nrate[2], nrate[3]); 7492 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 7493 orate[0], orate[1], orate[2], orate[3]); 7494 7495 rc = sbuf_finish(sb); 7496 sbuf_delete(sb); 7497 7498 return (rc); 7499 } 7500 7501 static int 7502 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) 7503 { 7504 struct adapter *sc = arg1; 7505 struct sbuf *sb; 7506 uint32_t *buf, *p; 7507 int rc, i; 7508 7509 rc = sysctl_wire_old_buffer(req, 0); 7510 if (rc != 0) 7511 return (rc); 7512 7513 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7514 if (sb == NULL) 7515 return (ENOMEM); 7516 7517 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, 7518 M_ZERO | M_WAITOK); 7519 7520 t4_ulprx_read_la(sc, buf); 7521 p = buf; 7522 7523 sbuf_printf(sb, " Pcmd Type Message" 7524 " Data"); 7525 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { 7526 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", 7527 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); 7528 } 7529 7530 rc = sbuf_finish(sb); 7531 sbuf_delete(sb); 7532 free(buf, M_CXGBE); 7533 return (rc); 7534 } 7535 7536 static int 7537 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 7538 { 7539 struct adapter *sc = arg1; 7540 struct sbuf *sb; 7541 int rc, v; 7542 7543 rc = sysctl_wire_old_buffer(req, 0); 7544 if (rc != 0) 7545 return (rc); 7546 7547 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7548 if (sb == NULL) 7549 return (ENOMEM); 7550 7551 v = t4_read_reg(sc, A_SGE_STAT_CFG); 7552 if (G_STATSOURCE_T5(v) == 7) { 7553 if (G_STATMODE(v) == 0) { 7554 sbuf_printf(sb, "total %d, incomplete %d", 7555 t4_read_reg(sc, A_SGE_STAT_TOTAL), 7556 t4_read_reg(sc, A_SGE_STAT_MATCH)); 7557 } else if (G_STATMODE(v) == 1) { 7558 sbuf_printf(sb, "total %d, data overflow %d", 7559 t4_read_reg(sc, A_SGE_STAT_TOTAL), 7560 t4_read_reg(sc, A_SGE_STAT_MATCH)); 7561 } 7562 } 7563 rc = sbuf_finish(sb); 7564 sbuf_delete(sb); 7565 7566 return (rc); 7567 } 7568 #endif 7569 7570 static uint32_t 7571 fconf_to_mode(uint32_t fconf) 7572 { 7573 uint32_t mode; 7574 7575 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 7576 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 7577 7578 if (fconf & F_FRAGMENTATION) 7579 mode |= T4_FILTER_IP_FRAGMENT; 7580 7581 if (fconf & F_MPSHITTYPE) 7582 mode |= T4_FILTER_MPS_HIT_TYPE; 7583 7584 if (fconf & F_MACMATCH) 7585 mode |= T4_FILTER_MAC_IDX; 7586 7587 if (fconf & F_ETHERTYPE) 7588 mode |= T4_FILTER_ETH_TYPE; 7589 7590 if (fconf & F_PROTOCOL) 7591 mode |= T4_FILTER_IP_PROTO; 7592 7593 if (fconf & F_TOS) 7594 mode |= T4_FILTER_IP_TOS; 7595 7596 if (fconf & F_VLAN) 7597 mode |= T4_FILTER_VLAN; 7598 7599 if (fconf & F_VNIC_ID) 7600 mode |= T4_FILTER_VNIC; 7601 7602 if (fconf & F_PORT) 7603 mode |= T4_FILTER_PORT; 7604 7605 if (fconf & F_FCOE) 7606 mode |= T4_FILTER_FCoE; 7607 7608 return (mode); 7609 } 7610 7611 static uint32_t 7612 mode_to_fconf(uint32_t mode) 7613 { 7614 uint32_t fconf = 0; 7615 7616 if (mode & T4_FILTER_IP_FRAGMENT) 7617 fconf |= F_FRAGMENTATION; 7618 7619 if (mode & T4_FILTER_MPS_HIT_TYPE) 7620 fconf |= F_MPSHITTYPE; 7621 7622 if (mode & T4_FILTER_MAC_IDX) 7623 fconf |= F_MACMATCH; 7624 7625 if (mode & T4_FILTER_ETH_TYPE) 7626 fconf |= F_ETHERTYPE; 7627 7628 if (mode & T4_FILTER_IP_PROTO) 7629 fconf |= F_PROTOCOL; 7630 7631 if (mode & T4_FILTER_IP_TOS) 7632 fconf |= F_TOS; 7633 7634 if (mode & T4_FILTER_VLAN) 7635 fconf |= F_VLAN; 7636 7637 if (mode & T4_FILTER_VNIC) 7638 fconf |= F_VNIC_ID; 7639 7640 if (mode & T4_FILTER_PORT) 7641 fconf |= F_PORT; 7642 7643 if (mode & T4_FILTER_FCoE) 7644 fconf |= F_FCOE; 7645 7646 return (fconf); 7647 } 7648 7649 static uint32_t 7650 fspec_to_fconf(struct t4_filter_specification *fs) 7651 { 7652 uint32_t fconf = 0; 7653 7654 if (fs->val.frag || fs->mask.frag) 7655 fconf |= F_FRAGMENTATION; 7656 7657 if (fs->val.matchtype || fs->mask.matchtype) 7658 fconf |= F_MPSHITTYPE; 7659 7660 if (fs->val.macidx || fs->mask.macidx) 7661 fconf |= F_MACMATCH; 7662 7663 if (fs->val.ethtype || fs->mask.ethtype) 7664 fconf |= F_ETHERTYPE; 7665 7666 if (fs->val.proto || fs->mask.proto) 7667 fconf |= F_PROTOCOL; 7668 7669 if (fs->val.tos || fs->mask.tos) 7670 fconf |= F_TOS; 7671 7672 if (fs->val.vlan_vld || fs->mask.vlan_vld) 7673 fconf |= F_VLAN; 7674 7675 if (fs->val.vnic_vld || fs->mask.vnic_vld) 7676 fconf |= F_VNIC_ID; 7677 7678 if (fs->val.iport || fs->mask.iport) 7679 fconf |= F_PORT; 7680 7681 if (fs->val.fcoe || fs->mask.fcoe) 7682 fconf |= F_FCOE; 7683 7684 return (fconf); 7685 } 7686 7687 static int 7688 get_filter_mode(struct adapter *sc, uint32_t *mode) 7689 { 7690 int rc; 7691 uint32_t fconf; 7692 7693 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7694 "t4getfm"); 7695 if (rc) 7696 return (rc); 7697 7698 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 7699 A_TP_VLAN_PRI_MAP); 7700 7701 if (sc->params.tp.vlan_pri_map != fconf) { 7702 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 7703 device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map, 7704 fconf); 7705 } 7706 7707 *mode = fconf_to_mode(fconf); 7708 7709 end_synchronized_op(sc, LOCK_HELD); 7710 return (0); 7711 } 7712 7713 static int 7714 set_filter_mode(struct adapter *sc, uint32_t mode) 7715 { 7716 uint32_t fconf; 7717 int rc; 7718 7719 fconf = mode_to_fconf(mode); 7720 7721 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7722 "t4setfm"); 7723 if (rc) 7724 return (rc); 7725 7726 if (sc->tids.ftids_in_use > 0) { 7727 rc = EBUSY; 7728 goto done; 7729 } 7730 7731 #ifdef TCP_OFFLOAD 7732 if (uld_active(sc, ULD_TOM)) { 7733 rc = EBUSY; 7734 goto done; 7735 } 7736 #endif 7737 7738 rc = -t4_set_filter_mode(sc, fconf); 7739 done: 7740 end_synchronized_op(sc, LOCK_HELD); 7741 return (rc); 7742 } 7743 7744 static inline uint64_t 7745 get_filter_hits(struct adapter *sc, uint32_t fid) 7746 { 7747 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 7748 uint64_t hits; 7749 7750 memwin_info(sc, 0, &mw_base, NULL); 7751 off = position_memwin(sc, 0, 7752 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 7753 if (is_t4(sc)) { 7754 hits = t4_read_reg64(sc, mw_base + off + 16); 7755 hits = be64toh(hits); 7756 } else { 7757 hits = t4_read_reg(sc, mw_base + off + 24); 7758 hits = be32toh(hits); 7759 } 7760 7761 return (hits); 7762 } 7763 7764 static int 7765 get_filter(struct adapter *sc, struct t4_filter *t) 7766 { 7767 int i, rc, nfilters = sc->tids.nftids; 7768 struct filter_entry *f; 7769 7770 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7771 "t4getf"); 7772 if (rc) 7773 return (rc); 7774 7775 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 7776 t->idx >= nfilters) { 7777 t->idx = 0xffffffff; 7778 goto done; 7779 } 7780 7781 f = &sc->tids.ftid_tab[t->idx]; 7782 for (i = t->idx; i < nfilters; i++, f++) { 7783 if (f->valid) { 7784 t->idx = i; 7785 t->l2tidx = f->l2t ? f->l2t->idx : 0; 7786 t->smtidx = f->smtidx; 7787 if (f->fs.hitcnts) 7788 t->hits = get_filter_hits(sc, t->idx); 7789 else 7790 t->hits = UINT64_MAX; 7791 t->fs = f->fs; 7792 7793 goto done; 7794 } 7795 } 7796 7797 t->idx = 0xffffffff; 7798 done: 7799 end_synchronized_op(sc, LOCK_HELD); 7800 return (0); 7801 } 7802 7803 static int 7804 set_filter(struct adapter *sc, struct t4_filter *t) 7805 { 7806 unsigned int nfilters, nports; 7807 struct filter_entry *f; 7808 int i, rc; 7809 7810 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 7811 if (rc) 7812 return (rc); 7813 7814 nfilters = sc->tids.nftids; 7815 nports = sc->params.nports; 7816 7817 if (nfilters == 0) { 7818 rc = ENOTSUP; 7819 goto done; 7820 } 7821 7822 if (!(sc->flags & FULL_INIT_DONE)) { 7823 rc = EAGAIN; 7824 goto done; 7825 } 7826 7827 if (t->idx >= nfilters) { 7828 rc = EINVAL; 7829 goto done; 7830 } 7831 7832 /* Validate against the global filter mode */ 7833 if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) != 7834 sc->params.tp.vlan_pri_map) { 7835 rc = E2BIG; 7836 goto done; 7837 } 7838 7839 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 7840 rc = EINVAL; 7841 goto done; 7842 } 7843 7844 if (t->fs.val.iport >= nports) { 7845 rc = EINVAL; 7846 goto done; 7847 } 7848 7849 /* Can't specify an iq if not steering to it */ 7850 if (!t->fs.dirsteer && t->fs.iq) { 7851 rc = EINVAL; 7852 goto done; 7853 } 7854 7855 /* IPv6 filter idx must be 4 aligned */ 7856 if (t->fs.type == 1 && 7857 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 7858 rc = EINVAL; 7859 goto done; 7860 } 7861 7862 if (sc->tids.ftid_tab == NULL) { 7863 KASSERT(sc->tids.ftids_in_use == 0, 7864 ("%s: no memory allocated but filters_in_use > 0", 7865 __func__)); 7866 7867 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 7868 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 7869 if (sc->tids.ftid_tab == NULL) { 7870 rc = ENOMEM; 7871 goto done; 7872 } 7873 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 7874 } 7875 7876 for (i = 0; i < 4; i++) { 7877 f = &sc->tids.ftid_tab[t->idx + i]; 7878 7879 if (f->pending || f->valid) { 7880 rc = EBUSY; 7881 goto done; 7882 } 7883 if (f->locked) { 7884 rc = EPERM; 7885 goto done; 7886 } 7887 7888 if (t->fs.type == 0) 7889 break; 7890 } 7891 7892 f = &sc->tids.ftid_tab[t->idx]; 7893 f->fs = t->fs; 7894 7895 rc = set_filter_wr(sc, t->idx); 7896 done: 7897 end_synchronized_op(sc, 0); 7898 7899 if (rc == 0) { 7900 mtx_lock(&sc->tids.ftid_lock); 7901 for (;;) { 7902 if (f->pending == 0) { 7903 rc = f->valid ? 0 : EIO; 7904 break; 7905 } 7906 7907 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7908 PCATCH, "t4setfw", 0)) { 7909 rc = EINPROGRESS; 7910 break; 7911 } 7912 } 7913 mtx_unlock(&sc->tids.ftid_lock); 7914 } 7915 return (rc); 7916 } 7917 7918 static int 7919 del_filter(struct adapter *sc, struct t4_filter *t) 7920 { 7921 unsigned int nfilters; 7922 struct filter_entry *f; 7923 int rc; 7924 7925 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 7926 if (rc) 7927 return (rc); 7928 7929 nfilters = sc->tids.nftids; 7930 7931 if (nfilters == 0) { 7932 rc = ENOTSUP; 7933 goto done; 7934 } 7935 7936 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 7937 t->idx >= nfilters) { 7938 rc = EINVAL; 7939 goto done; 7940 } 7941 7942 if (!(sc->flags & FULL_INIT_DONE)) { 7943 rc = EAGAIN; 7944 goto done; 7945 } 7946 7947 f = &sc->tids.ftid_tab[t->idx]; 7948 7949 if (f->pending) { 7950 rc = EBUSY; 7951 goto done; 7952 } 7953 if (f->locked) { 7954 rc = EPERM; 7955 goto done; 7956 } 7957 7958 if (f->valid) { 7959 t->fs = f->fs; /* extra info for the caller */ 7960 rc = del_filter_wr(sc, t->idx); 7961 } 7962 7963 done: 7964 end_synchronized_op(sc, 0); 7965 7966 if (rc == 0) { 7967 mtx_lock(&sc->tids.ftid_lock); 7968 for (;;) { 7969 if (f->pending == 0) { 7970 rc = f->valid ? EIO : 0; 7971 break; 7972 } 7973 7974 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7975 PCATCH, "t4delfw", 0)) { 7976 rc = EINPROGRESS; 7977 break; 7978 } 7979 } 7980 mtx_unlock(&sc->tids.ftid_lock); 7981 } 7982 7983 return (rc); 7984 } 7985 7986 static void 7987 clear_filter(struct filter_entry *f) 7988 { 7989 if (f->l2t) 7990 t4_l2t_release(f->l2t); 7991 7992 bzero(f, sizeof (*f)); 7993 } 7994 7995 static int 7996 set_filter_wr(struct adapter *sc, int fidx) 7997 { 7998 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7999 struct fw_filter_wr *fwr; 8000 unsigned int ftid; 8001 struct wrq_cookie cookie; 8002 8003 ASSERT_SYNCHRONIZED_OP(sc); 8004 8005 if (f->fs.newdmac || f->fs.newvlan) { 8006 /* This filter needs an L2T entry; allocate one. */ 8007 f->l2t = t4_l2t_alloc_switching(sc->l2t); 8008 if (f->l2t == NULL) 8009 return (EAGAIN); 8010 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 8011 f->fs.dmac)) { 8012 t4_l2t_release(f->l2t); 8013 f->l2t = NULL; 8014 return (ENOMEM); 8015 } 8016 } 8017 8018 ftid = sc->tids.ftid_base + fidx; 8019 8020 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 8021 if (fwr == NULL) 8022 return (ENOMEM); 8023 bzero(fwr, sizeof(*fwr)); 8024 8025 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 8026 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 8027 fwr->tid_to_iq = 8028 htobe32(V_FW_FILTER_WR_TID(ftid) | 8029 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 8030 V_FW_FILTER_WR_NOREPLY(0) | 8031 V_FW_FILTER_WR_IQ(f->fs.iq)); 8032 fwr->del_filter_to_l2tix = 8033 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 8034 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 8035 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 8036 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 8037 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 8038 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 8039 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 8040 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 8041 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 8042 f->fs.newvlan == VLAN_REWRITE) | 8043 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 8044 f->fs.newvlan == VLAN_REWRITE) | 8045 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 8046 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 8047 V_FW_FILTER_WR_PRIO(f->fs.prio) | 8048 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 8049 fwr->ethtype = htobe16(f->fs.val.ethtype); 8050 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 8051 fwr->frag_to_ovlan_vldm = 8052 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 8053 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 8054 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 8055 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 8056 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 8057 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 8058 fwr->smac_sel = 0; 8059 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 8060 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 8061 fwr->maci_to_matchtypem = 8062 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 8063 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 8064 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 8065 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 8066 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 8067 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 8068 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 8069 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 8070 fwr->ptcl = f->fs.val.proto; 8071 fwr->ptclm = f->fs.mask.proto; 8072 fwr->ttyp = f->fs.val.tos; 8073 fwr->ttypm = f->fs.mask.tos; 8074 fwr->ivlan = htobe16(f->fs.val.vlan); 8075 fwr->ivlanm = htobe16(f->fs.mask.vlan); 8076 fwr->ovlan = htobe16(f->fs.val.vnic); 8077 fwr->ovlanm = htobe16(f->fs.mask.vnic); 8078 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 8079 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 8080 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 8081 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 8082 fwr->lp = htobe16(f->fs.val.dport); 8083 fwr->lpm = htobe16(f->fs.mask.dport); 8084 fwr->fp = htobe16(f->fs.val.sport); 8085 fwr->fpm = htobe16(f->fs.mask.sport); 8086 if (f->fs.newsmac) 8087 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 8088 8089 f->pending = 1; 8090 sc->tids.ftids_in_use++; 8091 8092 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 8093 return (0); 8094 } 8095 8096 static int 8097 del_filter_wr(struct adapter *sc, int fidx) 8098 { 8099 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 8100 struct fw_filter_wr *fwr; 8101 unsigned int ftid; 8102 struct wrq_cookie cookie; 8103 8104 ftid = sc->tids.ftid_base + fidx; 8105 8106 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 8107 if (fwr == NULL) 8108 return (ENOMEM); 8109 bzero(fwr, sizeof (*fwr)); 8110 8111 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 8112 8113 f->pending = 1; 8114 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 8115 return (0); 8116 } 8117 8118 int 8119 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 8120 { 8121 struct adapter *sc = iq->adapter; 8122 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 8123 unsigned int idx = GET_TID(rpl); 8124 unsigned int rc; 8125 struct filter_entry *f; 8126 8127 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 8128 rss->opcode)); 8129 8130 if (is_ftid(sc, idx)) { 8131 8132 idx -= sc->tids.ftid_base; 8133 f = &sc->tids.ftid_tab[idx]; 8134 rc = G_COOKIE(rpl->cookie); 8135 8136 mtx_lock(&sc->tids.ftid_lock); 8137 if (rc == FW_FILTER_WR_FLT_ADDED) { 8138 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 8139 __func__, idx)); 8140 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 8141 f->pending = 0; /* asynchronous setup completed */ 8142 f->valid = 1; 8143 } else { 8144 if (rc != FW_FILTER_WR_FLT_DELETED) { 8145 /* Add or delete failed, display an error */ 8146 log(LOG_ERR, 8147 "filter %u setup failed with error %u\n", 8148 idx, rc); 8149 } 8150 8151 clear_filter(f); 8152 sc->tids.ftids_in_use--; 8153 } 8154 wakeup(&sc->tids.ftid_tab); 8155 mtx_unlock(&sc->tids.ftid_lock); 8156 } 8157 8158 return (0); 8159 } 8160 8161 static int 8162 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 8163 { 8164 int rc; 8165 8166 if (cntxt->cid > M_CTXTQID) 8167 return (EINVAL); 8168 8169 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 8170 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 8171 return (EINVAL); 8172 8173 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 8174 if (rc) 8175 return (rc); 8176 8177 if (sc->flags & FW_OK) { 8178 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 8179 &cntxt->data[0]); 8180 if (rc == 0) 8181 goto done; 8182 } 8183 8184 /* 8185 * Read via firmware failed or wasn't even attempted. Read directly via 8186 * the backdoor. 8187 */ 8188 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 8189 done: 8190 end_synchronized_op(sc, 0); 8191 return (rc); 8192 } 8193 8194 static int 8195 load_fw(struct adapter *sc, struct t4_data *fw) 8196 { 8197 int rc; 8198 uint8_t *fw_data; 8199 8200 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 8201 if (rc) 8202 return (rc); 8203 8204 if (sc->flags & FULL_INIT_DONE) { 8205 rc = EBUSY; 8206 goto done; 8207 } 8208 8209 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 8210 if (fw_data == NULL) { 8211 rc = ENOMEM; 8212 goto done; 8213 } 8214 8215 rc = copyin(fw->data, fw_data, fw->len); 8216 if (rc == 0) 8217 rc = -t4_load_fw(sc, fw_data, fw->len); 8218 8219 free(fw_data, M_CXGBE); 8220 done: 8221 end_synchronized_op(sc, 0); 8222 return (rc); 8223 } 8224 8225 static int 8226 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 8227 { 8228 uint32_t addr, off, remaining, i, n; 8229 uint32_t *buf, *b; 8230 uint32_t mw_base, mw_aperture; 8231 int rc; 8232 uint8_t *dst; 8233 8234 rc = validate_mem_range(sc, mr->addr, mr->len); 8235 if (rc != 0) 8236 return (rc); 8237 8238 memwin_info(sc, win, &mw_base, &mw_aperture); 8239 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 8240 addr = mr->addr; 8241 remaining = mr->len; 8242 dst = (void *)mr->data; 8243 8244 while (remaining) { 8245 off = position_memwin(sc, win, addr); 8246 8247 /* number of bytes that we'll copy in the inner loop */ 8248 n = min(remaining, mw_aperture - off); 8249 for (i = 0; i < n; i += 4) 8250 *b++ = t4_read_reg(sc, mw_base + off + i); 8251 8252 rc = copyout(buf, dst, n); 8253 if (rc != 0) 8254 break; 8255 8256 b = buf; 8257 dst += n; 8258 remaining -= n; 8259 addr += n; 8260 } 8261 8262 free(buf, M_CXGBE); 8263 return (rc); 8264 } 8265 8266 static int 8267 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 8268 { 8269 int rc; 8270 8271 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 8272 return (EINVAL); 8273 8274 if (i2cd->len > sizeof(i2cd->data)) 8275 return (EFBIG); 8276 8277 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 8278 if (rc) 8279 return (rc); 8280 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 8281 i2cd->offset, i2cd->len, &i2cd->data[0]); 8282 end_synchronized_op(sc, 0); 8283 8284 return (rc); 8285 } 8286 8287 static int 8288 in_range(int val, int lo, int hi) 8289 { 8290 8291 return (val < 0 || (val <= hi && val >= lo)); 8292 } 8293 8294 static int 8295 set_sched_class(struct adapter *sc, struct t4_sched_params *p) 8296 { 8297 int fw_subcmd, fw_type, rc; 8298 8299 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc"); 8300 if (rc) 8301 return (rc); 8302 8303 if (!(sc->flags & FULL_INIT_DONE)) { 8304 rc = EAGAIN; 8305 goto done; 8306 } 8307 8308 /* 8309 * Translate the cxgbetool parameters into T4 firmware parameters. (The 8310 * sub-command and type are in common locations.) 8311 */ 8312 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG) 8313 fw_subcmd = FW_SCHED_SC_CONFIG; 8314 else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS) 8315 fw_subcmd = FW_SCHED_SC_PARAMS; 8316 else { 8317 rc = EINVAL; 8318 goto done; 8319 } 8320 if (p->type == SCHED_CLASS_TYPE_PACKET) 8321 fw_type = FW_SCHED_TYPE_PKTSCHED; 8322 else { 8323 rc = EINVAL; 8324 goto done; 8325 } 8326 8327 if (fw_subcmd == FW_SCHED_SC_CONFIG) { 8328 /* Vet our parameters ..*/ 8329 if (p->u.config.minmax < 0) { 8330 rc = EINVAL; 8331 goto done; 8332 } 8333 8334 /* And pass the request to the firmware ...*/ 8335 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1); 8336 goto done; 8337 } 8338 8339 if (fw_subcmd == FW_SCHED_SC_PARAMS) { 8340 int fw_level; 8341 int fw_mode; 8342 int fw_rateunit; 8343 int fw_ratemode; 8344 8345 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL) 8346 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL; 8347 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) 8348 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR; 8349 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) 8350 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL; 8351 else { 8352 rc = EINVAL; 8353 goto done; 8354 } 8355 8356 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS) 8357 fw_mode = FW_SCHED_PARAMS_MODE_CLASS; 8358 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW) 8359 fw_mode = FW_SCHED_PARAMS_MODE_FLOW; 8360 else { 8361 rc = EINVAL; 8362 goto done; 8363 } 8364 8365 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS) 8366 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 8367 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS) 8368 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE; 8369 else { 8370 rc = EINVAL; 8371 goto done; 8372 } 8373 8374 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL) 8375 fw_ratemode = FW_SCHED_PARAMS_RATE_REL; 8376 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS) 8377 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS; 8378 else { 8379 rc = EINVAL; 8380 goto done; 8381 } 8382 8383 /* Vet our parameters ... */ 8384 if (!in_range(p->u.params.channel, 0, 3) || 8385 !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) || 8386 !in_range(p->u.params.minrate, 0, 10000000) || 8387 !in_range(p->u.params.maxrate, 0, 10000000) || 8388 !in_range(p->u.params.weight, 0, 100)) { 8389 rc = ERANGE; 8390 goto done; 8391 } 8392 8393 /* 8394 * Translate any unset parameters into the firmware's 8395 * nomenclature and/or fail the call if the parameters 8396 * are required ... 8397 */ 8398 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 || 8399 p->u.params.channel < 0 || p->u.params.cl < 0) { 8400 rc = EINVAL; 8401 goto done; 8402 } 8403 if (p->u.params.minrate < 0) 8404 p->u.params.minrate = 0; 8405 if (p->u.params.maxrate < 0) { 8406 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 8407 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 8408 rc = EINVAL; 8409 goto done; 8410 } else 8411 p->u.params.maxrate = 0; 8412 } 8413 if (p->u.params.weight < 0) { 8414 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) { 8415 rc = EINVAL; 8416 goto done; 8417 } else 8418 p->u.params.weight = 0; 8419 } 8420 if (p->u.params.pktsize < 0) { 8421 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 8422 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 8423 rc = EINVAL; 8424 goto done; 8425 } else 8426 p->u.params.pktsize = 0; 8427 } 8428 8429 /* See what the firmware thinks of the request ... */ 8430 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode, 8431 fw_rateunit, fw_ratemode, p->u.params.channel, 8432 p->u.params.cl, p->u.params.minrate, p->u.params.maxrate, 8433 p->u.params.weight, p->u.params.pktsize, 1); 8434 goto done; 8435 } 8436 8437 rc = EINVAL; 8438 done: 8439 end_synchronized_op(sc, 0); 8440 return (rc); 8441 } 8442 8443 static int 8444 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p) 8445 { 8446 struct port_info *pi = NULL; 8447 struct vi_info *vi; 8448 struct sge_txq *txq; 8449 uint32_t fw_mnem, fw_queue, fw_class; 8450 int i, rc; 8451 8452 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq"); 8453 if (rc) 8454 return (rc); 8455 8456 if (!(sc->flags & FULL_INIT_DONE)) { 8457 rc = EAGAIN; 8458 goto done; 8459 } 8460 8461 if (p->port >= sc->params.nports) { 8462 rc = EINVAL; 8463 goto done; 8464 } 8465 8466 /* XXX: Only supported for the main VI. */ 8467 pi = sc->port[p->port]; 8468 vi = &pi->vi[0]; 8469 if (!in_range(p->queue, 0, vi->ntxq - 1) || !in_range(p->cl, 0, 7)) { 8470 rc = EINVAL; 8471 goto done; 8472 } 8473 8474 /* 8475 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX 8476 * Scheduling Class in this case). 8477 */ 8478 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 8479 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH)); 8480 fw_class = p->cl < 0 ? 0xffffffff : p->cl; 8481 8482 /* 8483 * If op.queue is non-negative, then we're only changing the scheduling 8484 * on a single specified TX queue. 8485 */ 8486 if (p->queue >= 0) { 8487 txq = &sc->sge.txq[vi->first_txq + p->queue]; 8488 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 8489 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 8490 &fw_class); 8491 goto done; 8492 } 8493 8494 /* 8495 * Change the scheduling on all the TX queues for the 8496 * interface. 8497 */ 8498 for_each_txq(vi, i, txq) { 8499 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 8500 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 8501 &fw_class); 8502 if (rc) 8503 goto done; 8504 } 8505 8506 rc = 0; 8507 done: 8508 end_synchronized_op(sc, 0); 8509 return (rc); 8510 } 8511 8512 int 8513 t4_os_find_pci_capability(struct adapter *sc, int cap) 8514 { 8515 int i; 8516 8517 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 8518 } 8519 8520 int 8521 t4_os_pci_save_state(struct adapter *sc) 8522 { 8523 device_t dev; 8524 struct pci_devinfo *dinfo; 8525 8526 dev = sc->dev; 8527 dinfo = device_get_ivars(dev); 8528 8529 pci_cfg_save(dev, dinfo, 0); 8530 return (0); 8531 } 8532 8533 int 8534 t4_os_pci_restore_state(struct adapter *sc) 8535 { 8536 device_t dev; 8537 struct pci_devinfo *dinfo; 8538 8539 dev = sc->dev; 8540 dinfo = device_get_ivars(dev); 8541 8542 pci_cfg_restore(dev, dinfo); 8543 return (0); 8544 } 8545 8546 void 8547 t4_os_portmod_changed(const struct adapter *sc, int idx) 8548 { 8549 struct port_info *pi = sc->port[idx]; 8550 struct vi_info *vi; 8551 struct ifnet *ifp; 8552 int v; 8553 static const char *mod_str[] = { 8554 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 8555 }; 8556 8557 for_each_vi(pi, v, vi) { 8558 build_medialist(pi, &vi->media); 8559 } 8560 8561 ifp = pi->vi[0].ifp; 8562 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 8563 if_printf(ifp, "transceiver unplugged.\n"); 8564 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 8565 if_printf(ifp, "unknown transceiver inserted.\n"); 8566 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 8567 if_printf(ifp, "unsupported transceiver inserted.\n"); 8568 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 8569 if_printf(ifp, "%s transceiver inserted.\n", 8570 mod_str[pi->mod_type]); 8571 } else { 8572 if_printf(ifp, "transceiver (type %d) inserted.\n", 8573 pi->mod_type); 8574 } 8575 } 8576 8577 void 8578 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason) 8579 { 8580 struct port_info *pi = sc->port[idx]; 8581 struct vi_info *vi; 8582 struct ifnet *ifp; 8583 int v; 8584 8585 if (link_stat) 8586 pi->linkdnrc = -1; 8587 else { 8588 if (reason >= 0) 8589 pi->linkdnrc = reason; 8590 } 8591 for_each_vi(pi, v, vi) { 8592 ifp = vi->ifp; 8593 if (ifp == NULL) 8594 continue; 8595 8596 if (link_stat) { 8597 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 8598 if_link_state_change(ifp, LINK_STATE_UP); 8599 } else { 8600 if_link_state_change(ifp, LINK_STATE_DOWN); 8601 } 8602 } 8603 } 8604 8605 void 8606 t4_iterate(void (*func)(struct adapter *, void *), void *arg) 8607 { 8608 struct adapter *sc; 8609 8610 sx_slock(&t4_list_lock); 8611 SLIST_FOREACH(sc, &t4_list, link) { 8612 /* 8613 * func should not make any assumptions about what state sc is 8614 * in - the only guarantee is that sc->sc_lock is a valid lock. 8615 */ 8616 func(sc, arg); 8617 } 8618 sx_sunlock(&t4_list_lock); 8619 } 8620 8621 static int 8622 t4_open(struct cdev *dev, int flags, int type, struct thread *td) 8623 { 8624 return (0); 8625 } 8626 8627 static int 8628 t4_close(struct cdev *dev, int flags, int type, struct thread *td) 8629 { 8630 return (0); 8631 } 8632 8633 static int 8634 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 8635 struct thread *td) 8636 { 8637 int rc; 8638 struct adapter *sc = dev->si_drv1; 8639 8640 rc = priv_check(td, PRIV_DRIVER); 8641 if (rc != 0) 8642 return (rc); 8643 8644 switch (cmd) { 8645 case CHELSIO_T4_GETREG: { 8646 struct t4_reg *edata = (struct t4_reg *)data; 8647 8648 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8649 return (EFAULT); 8650 8651 if (edata->size == 4) 8652 edata->val = t4_read_reg(sc, edata->addr); 8653 else if (edata->size == 8) 8654 edata->val = t4_read_reg64(sc, edata->addr); 8655 else 8656 return (EINVAL); 8657 8658 break; 8659 } 8660 case CHELSIO_T4_SETREG: { 8661 struct t4_reg *edata = (struct t4_reg *)data; 8662 8663 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8664 return (EFAULT); 8665 8666 if (edata->size == 4) { 8667 if (edata->val & 0xffffffff00000000) 8668 return (EINVAL); 8669 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 8670 } else if (edata->size == 8) 8671 t4_write_reg64(sc, edata->addr, edata->val); 8672 else 8673 return (EINVAL); 8674 break; 8675 } 8676 case CHELSIO_T4_REGDUMP: { 8677 struct t4_regdump *regs = (struct t4_regdump *)data; 8678 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 8679 uint8_t *buf; 8680 8681 if (regs->len < reglen) { 8682 regs->len = reglen; /* hint to the caller */ 8683 return (ENOBUFS); 8684 } 8685 8686 regs->len = reglen; 8687 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 8688 t4_get_regs(sc, regs, buf); 8689 rc = copyout(buf, regs->data, reglen); 8690 free(buf, M_CXGBE); 8691 break; 8692 } 8693 case CHELSIO_T4_GET_FILTER_MODE: 8694 rc = get_filter_mode(sc, (uint32_t *)data); 8695 break; 8696 case CHELSIO_T4_SET_FILTER_MODE: 8697 rc = set_filter_mode(sc, *(uint32_t *)data); 8698 break; 8699 case CHELSIO_T4_GET_FILTER: 8700 rc = get_filter(sc, (struct t4_filter *)data); 8701 break; 8702 case CHELSIO_T4_SET_FILTER: 8703 rc = set_filter(sc, (struct t4_filter *)data); 8704 break; 8705 case CHELSIO_T4_DEL_FILTER: 8706 rc = del_filter(sc, (struct t4_filter *)data); 8707 break; 8708 case CHELSIO_T4_GET_SGE_CONTEXT: 8709 rc = get_sge_context(sc, (struct t4_sge_context *)data); 8710 break; 8711 case CHELSIO_T4_LOAD_FW: 8712 rc = load_fw(sc, (struct t4_data *)data); 8713 break; 8714 case CHELSIO_T4_GET_MEM: 8715 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 8716 break; 8717 case CHELSIO_T4_GET_I2C: 8718 rc = read_i2c(sc, (struct t4_i2c_data *)data); 8719 break; 8720 case CHELSIO_T4_CLEAR_STATS: { 8721 int i, v; 8722 u_int port_id = *(uint32_t *)data; 8723 struct port_info *pi; 8724 struct vi_info *vi; 8725 8726 if (port_id >= sc->params.nports) 8727 return (EINVAL); 8728 pi = sc->port[port_id]; 8729 8730 /* MAC stats */ 8731 t4_clr_port_stats(sc, pi->tx_chan); 8732 pi->tx_parse_error = 0; 8733 mtx_lock(&sc->regwin_lock); 8734 for_each_vi(pi, v, vi) { 8735 if (vi->flags & VI_INIT_DONE) 8736 t4_clr_vi_stats(sc, vi->viid); 8737 } 8738 mtx_unlock(&sc->regwin_lock); 8739 8740 /* 8741 * Since this command accepts a port, clear stats for 8742 * all VIs on this port. 8743 */ 8744 for_each_vi(pi, v, vi) { 8745 if (vi->flags & VI_INIT_DONE) { 8746 struct sge_rxq *rxq; 8747 struct sge_txq *txq; 8748 struct sge_wrq *wrq; 8749 8750 if (vi->flags & VI_NETMAP) 8751 continue; 8752 8753 for_each_rxq(vi, i, rxq) { 8754 #if defined(INET) || defined(INET6) 8755 rxq->lro.lro_queued = 0; 8756 rxq->lro.lro_flushed = 0; 8757 #endif 8758 rxq->rxcsum = 0; 8759 rxq->vlan_extraction = 0; 8760 } 8761 8762 for_each_txq(vi, i, txq) { 8763 txq->txcsum = 0; 8764 txq->tso_wrs = 0; 8765 txq->vlan_insertion = 0; 8766 txq->imm_wrs = 0; 8767 txq->sgl_wrs = 0; 8768 txq->txpkt_wrs = 0; 8769 txq->txpkts0_wrs = 0; 8770 txq->txpkts1_wrs = 0; 8771 txq->txpkts0_pkts = 0; 8772 txq->txpkts1_pkts = 0; 8773 mp_ring_reset_stats(txq->r); 8774 } 8775 8776 #ifdef TCP_OFFLOAD 8777 /* nothing to clear for each ofld_rxq */ 8778 8779 for_each_ofld_txq(vi, i, wrq) { 8780 wrq->tx_wrs_direct = 0; 8781 wrq->tx_wrs_copied = 0; 8782 } 8783 #endif 8784 8785 if (IS_MAIN_VI(vi)) { 8786 wrq = &sc->sge.ctrlq[pi->port_id]; 8787 wrq->tx_wrs_direct = 0; 8788 wrq->tx_wrs_copied = 0; 8789 } 8790 } 8791 } 8792 break; 8793 } 8794 case CHELSIO_T4_SCHED_CLASS: 8795 rc = set_sched_class(sc, (struct t4_sched_params *)data); 8796 break; 8797 case CHELSIO_T4_SCHED_QUEUE: 8798 rc = set_sched_queue(sc, (struct t4_sched_queue *)data); 8799 break; 8800 case CHELSIO_T4_GET_TRACER: 8801 rc = t4_get_tracer(sc, (struct t4_tracer *)data); 8802 break; 8803 case CHELSIO_T4_SET_TRACER: 8804 rc = t4_set_tracer(sc, (struct t4_tracer *)data); 8805 break; 8806 default: 8807 rc = EINVAL; 8808 } 8809 8810 return (rc); 8811 } 8812 8813 #ifdef TCP_OFFLOAD 8814 void 8815 t4_iscsi_init(struct adapter *sc, u_int tag_mask, const u_int *pgsz_order) 8816 { 8817 8818 t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask); 8819 t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) | 8820 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) | 8821 V_HPZ3(pgsz_order[3])); 8822 } 8823 8824 static int 8825 toe_capability(struct vi_info *vi, int enable) 8826 { 8827 int rc; 8828 struct port_info *pi = vi->pi; 8829 struct adapter *sc = pi->adapter; 8830 8831 ASSERT_SYNCHRONIZED_OP(sc); 8832 8833 if (!is_offload(sc)) 8834 return (ENODEV); 8835 8836 if (enable) { 8837 if ((vi->ifp->if_capenable & IFCAP_TOE) != 0) { 8838 /* TOE is already enabled. */ 8839 return (0); 8840 } 8841 8842 /* 8843 * We need the port's queues around so that we're able to send 8844 * and receive CPLs to/from the TOE even if the ifnet for this 8845 * port has never been UP'd administratively. 8846 */ 8847 if (!(vi->flags & VI_INIT_DONE)) { 8848 rc = cxgbe_init_synchronized(vi); 8849 if (rc) 8850 return (rc); 8851 } 8852 if (!(pi->vi[0].flags & VI_INIT_DONE)) { 8853 rc = cxgbe_init_synchronized(&pi->vi[0]); 8854 if (rc) 8855 return (rc); 8856 } 8857 8858 if (isset(&sc->offload_map, pi->port_id)) { 8859 /* TOE is enabled on another VI of this port. */ 8860 pi->uld_vis++; 8861 return (0); 8862 } 8863 8864 if (!uld_active(sc, ULD_TOM)) { 8865 rc = t4_activate_uld(sc, ULD_TOM); 8866 if (rc == EAGAIN) { 8867 log(LOG_WARNING, 8868 "You must kldload t4_tom.ko before trying " 8869 "to enable TOE on a cxgbe interface.\n"); 8870 } 8871 if (rc != 0) 8872 return (rc); 8873 KASSERT(sc->tom_softc != NULL, 8874 ("%s: TOM activated but softc NULL", __func__)); 8875 KASSERT(uld_active(sc, ULD_TOM), 8876 ("%s: TOM activated but flag not set", __func__)); 8877 } 8878 8879 /* Activate iWARP and iSCSI too, if the modules are loaded. */ 8880 if (!uld_active(sc, ULD_IWARP)) 8881 (void) t4_activate_uld(sc, ULD_IWARP); 8882 if (!uld_active(sc, ULD_ISCSI)) 8883 (void) t4_activate_uld(sc, ULD_ISCSI); 8884 8885 pi->uld_vis++; 8886 setbit(&sc->offload_map, pi->port_id); 8887 } else { 8888 pi->uld_vis--; 8889 8890 if (!isset(&sc->offload_map, pi->port_id) || pi->uld_vis > 0) 8891 return (0); 8892 8893 KASSERT(uld_active(sc, ULD_TOM), 8894 ("%s: TOM never initialized?", __func__)); 8895 clrbit(&sc->offload_map, pi->port_id); 8896 } 8897 8898 return (0); 8899 } 8900 8901 /* 8902 * Add an upper layer driver to the global list. 8903 */ 8904 int 8905 t4_register_uld(struct uld_info *ui) 8906 { 8907 int rc = 0; 8908 struct uld_info *u; 8909 8910 sx_xlock(&t4_uld_list_lock); 8911 SLIST_FOREACH(u, &t4_uld_list, link) { 8912 if (u->uld_id == ui->uld_id) { 8913 rc = EEXIST; 8914 goto done; 8915 } 8916 } 8917 8918 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 8919 ui->refcount = 0; 8920 done: 8921 sx_xunlock(&t4_uld_list_lock); 8922 return (rc); 8923 } 8924 8925 int 8926 t4_unregister_uld(struct uld_info *ui) 8927 { 8928 int rc = EINVAL; 8929 struct uld_info *u; 8930 8931 sx_xlock(&t4_uld_list_lock); 8932 8933 SLIST_FOREACH(u, &t4_uld_list, link) { 8934 if (u == ui) { 8935 if (ui->refcount > 0) { 8936 rc = EBUSY; 8937 goto done; 8938 } 8939 8940 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 8941 rc = 0; 8942 goto done; 8943 } 8944 } 8945 done: 8946 sx_xunlock(&t4_uld_list_lock); 8947 return (rc); 8948 } 8949 8950 int 8951 t4_activate_uld(struct adapter *sc, int id) 8952 { 8953 int rc; 8954 struct uld_info *ui; 8955 8956 ASSERT_SYNCHRONIZED_OP(sc); 8957 8958 if (id < 0 || id > ULD_MAX) 8959 return (EINVAL); 8960 rc = EAGAIN; /* kldoad the module with this ULD and try again. */ 8961 8962 sx_slock(&t4_uld_list_lock); 8963 8964 SLIST_FOREACH(ui, &t4_uld_list, link) { 8965 if (ui->uld_id == id) { 8966 if (!(sc->flags & FULL_INIT_DONE)) { 8967 rc = adapter_full_init(sc); 8968 if (rc != 0) 8969 break; 8970 } 8971 8972 rc = ui->activate(sc); 8973 if (rc == 0) { 8974 setbit(&sc->active_ulds, id); 8975 ui->refcount++; 8976 } 8977 break; 8978 } 8979 } 8980 8981 sx_sunlock(&t4_uld_list_lock); 8982 8983 return (rc); 8984 } 8985 8986 int 8987 t4_deactivate_uld(struct adapter *sc, int id) 8988 { 8989 int rc; 8990 struct uld_info *ui; 8991 8992 ASSERT_SYNCHRONIZED_OP(sc); 8993 8994 if (id < 0 || id > ULD_MAX) 8995 return (EINVAL); 8996 rc = ENXIO; 8997 8998 sx_slock(&t4_uld_list_lock); 8999 9000 SLIST_FOREACH(ui, &t4_uld_list, link) { 9001 if (ui->uld_id == id) { 9002 rc = ui->deactivate(sc); 9003 if (rc == 0) { 9004 clrbit(&sc->active_ulds, id); 9005 ui->refcount--; 9006 } 9007 break; 9008 } 9009 } 9010 9011 sx_sunlock(&t4_uld_list_lock); 9012 9013 return (rc); 9014 } 9015 9016 int 9017 uld_active(struct adapter *sc, int uld_id) 9018 { 9019 9020 MPASS(uld_id >= 0 && uld_id <= ULD_MAX); 9021 9022 return (isset(&sc->active_ulds, uld_id)); 9023 } 9024 #endif 9025 9026 /* 9027 * Come up with reasonable defaults for some of the tunables, provided they're 9028 * not set by the user (in which case we'll use the values as is). 9029 */ 9030 static void 9031 tweak_tunables(void) 9032 { 9033 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 9034 9035 if (t4_ntxq10g < 1) { 9036 #ifdef RSS 9037 t4_ntxq10g = rss_getnumbuckets(); 9038 #else 9039 t4_ntxq10g = min(nc, NTXQ_10G); 9040 #endif 9041 } 9042 9043 if (t4_ntxq1g < 1) { 9044 #ifdef RSS 9045 /* XXX: way too many for 1GbE? */ 9046 t4_ntxq1g = rss_getnumbuckets(); 9047 #else 9048 t4_ntxq1g = min(nc, NTXQ_1G); 9049 #endif 9050 } 9051 9052 if (t4_nrxq10g < 1) { 9053 #ifdef RSS 9054 t4_nrxq10g = rss_getnumbuckets(); 9055 #else 9056 t4_nrxq10g = min(nc, NRXQ_10G); 9057 #endif 9058 } 9059 9060 if (t4_nrxq1g < 1) { 9061 #ifdef RSS 9062 /* XXX: way too many for 1GbE? */ 9063 t4_nrxq1g = rss_getnumbuckets(); 9064 #else 9065 t4_nrxq1g = min(nc, NRXQ_1G); 9066 #endif 9067 } 9068 9069 #ifdef TCP_OFFLOAD 9070 if (t4_nofldtxq10g < 1) 9071 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 9072 9073 if (t4_nofldtxq1g < 1) 9074 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 9075 9076 if (t4_nofldrxq10g < 1) 9077 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 9078 9079 if (t4_nofldrxq1g < 1) 9080 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 9081 9082 if (t4_toecaps_allowed == -1) 9083 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 9084 #else 9085 if (t4_toecaps_allowed == -1) 9086 t4_toecaps_allowed = 0; 9087 #endif 9088 9089 #ifdef DEV_NETMAP 9090 if (t4_nnmtxq10g < 1) 9091 t4_nnmtxq10g = min(nc, NNMTXQ_10G); 9092 9093 if (t4_nnmtxq1g < 1) 9094 t4_nnmtxq1g = min(nc, NNMTXQ_1G); 9095 9096 if (t4_nnmrxq10g < 1) 9097 t4_nnmrxq10g = min(nc, NNMRXQ_10G); 9098 9099 if (t4_nnmrxq1g < 1) 9100 t4_nnmrxq1g = min(nc, NNMRXQ_1G); 9101 #endif 9102 9103 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 9104 t4_tmr_idx_10g = TMR_IDX_10G; 9105 9106 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 9107 t4_pktc_idx_10g = PKTC_IDX_10G; 9108 9109 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 9110 t4_tmr_idx_1g = TMR_IDX_1G; 9111 9112 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 9113 t4_pktc_idx_1g = PKTC_IDX_1G; 9114 9115 if (t4_qsize_txq < 128) 9116 t4_qsize_txq = 128; 9117 9118 if (t4_qsize_rxq < 128) 9119 t4_qsize_rxq = 128; 9120 while (t4_qsize_rxq & 7) 9121 t4_qsize_rxq++; 9122 9123 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 9124 } 9125 9126 static struct sx mlu; /* mod load unload */ 9127 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload"); 9128 9129 static int 9130 mod_event(module_t mod, int cmd, void *arg) 9131 { 9132 int rc = 0; 9133 static int loaded = 0; 9134 9135 switch (cmd) { 9136 case MOD_LOAD: 9137 sx_xlock(&mlu); 9138 if (loaded++ == 0) { 9139 t4_sge_modload(); 9140 sx_init(&t4_list_lock, "T4/T5 adapters"); 9141 SLIST_INIT(&t4_list); 9142 #ifdef TCP_OFFLOAD 9143 sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); 9144 SLIST_INIT(&t4_uld_list); 9145 #endif 9146 t4_tracer_modload(); 9147 tweak_tunables(); 9148 } 9149 sx_xunlock(&mlu); 9150 break; 9151 9152 case MOD_UNLOAD: 9153 sx_xlock(&mlu); 9154 if (--loaded == 0) { 9155 int tries; 9156 9157 sx_slock(&t4_list_lock); 9158 if (!SLIST_EMPTY(&t4_list)) { 9159 rc = EBUSY; 9160 sx_sunlock(&t4_list_lock); 9161 goto done_unload; 9162 } 9163 #ifdef TCP_OFFLOAD 9164 sx_slock(&t4_uld_list_lock); 9165 if (!SLIST_EMPTY(&t4_uld_list)) { 9166 rc = EBUSY; 9167 sx_sunlock(&t4_uld_list_lock); 9168 sx_sunlock(&t4_list_lock); 9169 goto done_unload; 9170 } 9171 #endif 9172 tries = 0; 9173 while (tries++ < 5 && t4_sge_extfree_refs() != 0) { 9174 uprintf("%ju clusters with custom free routine " 9175 "still is use.\n", t4_sge_extfree_refs()); 9176 pause("t4unload", 2 * hz); 9177 } 9178 #ifdef TCP_OFFLOAD 9179 sx_sunlock(&t4_uld_list_lock); 9180 #endif 9181 sx_sunlock(&t4_list_lock); 9182 9183 if (t4_sge_extfree_refs() == 0) { 9184 t4_tracer_modunload(); 9185 #ifdef TCP_OFFLOAD 9186 sx_destroy(&t4_uld_list_lock); 9187 #endif 9188 sx_destroy(&t4_list_lock); 9189 t4_sge_modunload(); 9190 loaded = 0; 9191 } else { 9192 rc = EBUSY; 9193 loaded++; /* undo earlier decrement */ 9194 } 9195 } 9196 done_unload: 9197 sx_xunlock(&mlu); 9198 break; 9199 } 9200 9201 return (rc); 9202 } 9203 9204 static devclass_t t4_devclass, t5_devclass; 9205 static devclass_t cxgbe_devclass, cxl_devclass; 9206 static devclass_t vcxgbe_devclass, vcxl_devclass; 9207 9208 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 9209 MODULE_VERSION(t4nex, 1); 9210 MODULE_DEPEND(t4nex, firmware, 1, 1, 1); 9211 #ifdef DEV_NETMAP 9212 MODULE_DEPEND(t4nex, netmap, 1, 1, 1); 9213 #endif /* DEV_NETMAP */ 9214 9215 9216 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 9217 MODULE_VERSION(t5nex, 1); 9218 MODULE_DEPEND(t5nex, firmware, 1, 1, 1); 9219 #ifdef DEV_NETMAP 9220 MODULE_DEPEND(t5nex, netmap, 1, 1, 1); 9221 #endif /* DEV_NETMAP */ 9222 9223 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 9224 MODULE_VERSION(cxgbe, 1); 9225 9226 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 9227 MODULE_VERSION(cxl, 1); 9228 9229 DRIVER_MODULE(vcxgbe, cxgbe, vcxgbe_driver, vcxgbe_devclass, 0, 0); 9230 MODULE_VERSION(vcxgbe, 1); 9231 9232 DRIVER_MODULE(vcxl, cxl, vcxl_driver, vcxl_devclass, 0, 0); 9233 MODULE_VERSION(vcxl, 1); 9234