1 /*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_inet.h" 32 #include "opt_inet6.h" 33 #include "opt_rss.h" 34 35 #include <sys/param.h> 36 #include <sys/conf.h> 37 #include <sys/priv.h> 38 #include <sys/kernel.h> 39 #include <sys/bus.h> 40 #include <sys/module.h> 41 #include <sys/malloc.h> 42 #include <sys/queue.h> 43 #include <sys/taskqueue.h> 44 #include <sys/pciio.h> 45 #include <dev/pci/pcireg.h> 46 #include <dev/pci/pcivar.h> 47 #include <dev/pci/pci_private.h> 48 #include <sys/firmware.h> 49 #include <sys/sbuf.h> 50 #include <sys/smp.h> 51 #include <sys/socket.h> 52 #include <sys/sockio.h> 53 #include <sys/sysctl.h> 54 #include <net/ethernet.h> 55 #include <net/if.h> 56 #include <net/if_types.h> 57 #include <net/if_dl.h> 58 #include <net/if_vlan_var.h> 59 #ifdef RSS 60 #include <net/rss_config.h> 61 #endif 62 #if defined(__i386__) || defined(__amd64__) 63 #include <vm/vm.h> 64 #include <vm/pmap.h> 65 #endif 66 67 #include "common/common.h" 68 #include "common/t4_msg.h" 69 #include "common/t4_regs.h" 70 #include "common/t4_regs_values.h" 71 #include "t4_ioctl.h" 72 #include "t4_l2t.h" 73 #include "t4_mp_ring.h" 74 75 /* T4 bus driver interface */ 76 static int t4_probe(device_t); 77 static int t4_attach(device_t); 78 static int t4_detach(device_t); 79 static device_method_t t4_methods[] = { 80 DEVMETHOD(device_probe, t4_probe), 81 DEVMETHOD(device_attach, t4_attach), 82 DEVMETHOD(device_detach, t4_detach), 83 84 DEVMETHOD_END 85 }; 86 static driver_t t4_driver = { 87 "t4nex", 88 t4_methods, 89 sizeof(struct adapter) 90 }; 91 92 93 /* T4 port (cxgbe) interface */ 94 static int cxgbe_probe(device_t); 95 static int cxgbe_attach(device_t); 96 static int cxgbe_detach(device_t); 97 static device_method_t cxgbe_methods[] = { 98 DEVMETHOD(device_probe, cxgbe_probe), 99 DEVMETHOD(device_attach, cxgbe_attach), 100 DEVMETHOD(device_detach, cxgbe_detach), 101 { 0, 0 } 102 }; 103 static driver_t cxgbe_driver = { 104 "cxgbe", 105 cxgbe_methods, 106 sizeof(struct port_info) 107 }; 108 109 static d_ioctl_t t4_ioctl; 110 static d_open_t t4_open; 111 static d_close_t t4_close; 112 113 static struct cdevsw t4_cdevsw = { 114 .d_version = D_VERSION, 115 .d_flags = 0, 116 .d_open = t4_open, 117 .d_close = t4_close, 118 .d_ioctl = t4_ioctl, 119 .d_name = "t4nex", 120 }; 121 122 /* T5 bus driver interface */ 123 static int t5_probe(device_t); 124 static device_method_t t5_methods[] = { 125 DEVMETHOD(device_probe, t5_probe), 126 DEVMETHOD(device_attach, t4_attach), 127 DEVMETHOD(device_detach, t4_detach), 128 129 DEVMETHOD_END 130 }; 131 static driver_t t5_driver = { 132 "t5nex", 133 t5_methods, 134 sizeof(struct adapter) 135 }; 136 137 138 /* T5 port (cxl) interface */ 139 static driver_t cxl_driver = { 140 "cxl", 141 cxgbe_methods, 142 sizeof(struct port_info) 143 }; 144 145 static struct cdevsw t5_cdevsw = { 146 .d_version = D_VERSION, 147 .d_flags = 0, 148 .d_open = t4_open, 149 .d_close = t4_close, 150 .d_ioctl = t4_ioctl, 151 .d_name = "t5nex", 152 }; 153 154 /* ifnet + media interface */ 155 static void cxgbe_init(void *); 156 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 157 static int cxgbe_transmit(struct ifnet *, struct mbuf *); 158 static void cxgbe_qflush(struct ifnet *); 159 static uint64_t cxgbe_get_counter(struct ifnet *, ift_counter); 160 static int cxgbe_media_change(struct ifnet *); 161 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 162 163 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 164 165 /* 166 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 167 * then ADAPTER_LOCK, then t4_uld_list_lock. 168 */ 169 static struct sx t4_list_lock; 170 SLIST_HEAD(, adapter) t4_list; 171 #ifdef TCP_OFFLOAD 172 static struct sx t4_uld_list_lock; 173 SLIST_HEAD(, uld_info) t4_uld_list; 174 #endif 175 176 /* 177 * Tunables. See tweak_tunables() too. 178 * 179 * Each tunable is set to a default value here if it's known at compile-time. 180 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 181 * provide a reasonable default when the driver is loaded. 182 * 183 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 184 * T5 are under hw.cxl. 185 */ 186 187 /* 188 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 189 */ 190 #define NTXQ_10G 16 191 static int t4_ntxq10g = -1; 192 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 193 194 #define NRXQ_10G 8 195 static int t4_nrxq10g = -1; 196 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 197 198 #define NTXQ_1G 4 199 static int t4_ntxq1g = -1; 200 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 201 202 #define NRXQ_1G 2 203 static int t4_nrxq1g = -1; 204 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 205 206 static int t4_rsrv_noflowq = 0; 207 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq); 208 209 #ifdef TCP_OFFLOAD 210 #define NOFLDTXQ_10G 8 211 static int t4_nofldtxq10g = -1; 212 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 213 214 #define NOFLDRXQ_10G 2 215 static int t4_nofldrxq10g = -1; 216 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 217 218 #define NOFLDTXQ_1G 2 219 static int t4_nofldtxq1g = -1; 220 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 221 222 #define NOFLDRXQ_1G 1 223 static int t4_nofldrxq1g = -1; 224 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 225 #endif 226 227 #ifdef DEV_NETMAP 228 #define NNMTXQ_10G 2 229 static int t4_nnmtxq10g = -1; 230 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g); 231 232 #define NNMRXQ_10G 2 233 static int t4_nnmrxq10g = -1; 234 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g); 235 236 #define NNMTXQ_1G 1 237 static int t4_nnmtxq1g = -1; 238 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g); 239 240 #define NNMRXQ_1G 1 241 static int t4_nnmrxq1g = -1; 242 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g); 243 #endif 244 245 /* 246 * Holdoff parameters for 10G and 1G ports. 247 */ 248 #define TMR_IDX_10G 1 249 static int t4_tmr_idx_10g = TMR_IDX_10G; 250 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 251 252 #define PKTC_IDX_10G (-1) 253 static int t4_pktc_idx_10g = PKTC_IDX_10G; 254 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 255 256 #define TMR_IDX_1G 1 257 static int t4_tmr_idx_1g = TMR_IDX_1G; 258 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 259 260 #define PKTC_IDX_1G (-1) 261 static int t4_pktc_idx_1g = PKTC_IDX_1G; 262 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 263 264 /* 265 * Size (# of entries) of each tx and rx queue. 266 */ 267 static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 268 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 269 270 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 271 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 272 273 /* 274 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 275 */ 276 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 277 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 278 279 /* 280 * Configuration file. 281 */ 282 #define DEFAULT_CF "default" 283 #define FLASH_CF "flash" 284 #define UWIRE_CF "uwire" 285 #define FPGA_CF "fpga" 286 static char t4_cfg_file[32] = DEFAULT_CF; 287 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 288 289 /* 290 * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively). 291 * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them. 292 * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water 293 * mark or when signalled to do so, 0 to never emit PAUSE. 294 */ 295 static int t4_pause_settings = PAUSE_TX | PAUSE_RX; 296 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings); 297 298 /* 299 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 300 * encouraged respectively). 301 */ 302 static unsigned int t4_fw_install = 1; 303 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 304 305 /* 306 * ASIC features that will be used. Disable the ones you don't want so that the 307 * chip resources aren't wasted on features that will not be used. 308 */ 309 static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 310 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 311 312 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 313 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 314 315 static int t4_toecaps_allowed = -1; 316 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 317 318 static int t4_rdmacaps_allowed = 0; 319 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 320 321 static int t4_iscsicaps_allowed = 0; 322 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 323 324 static int t4_fcoecaps_allowed = 0; 325 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 326 327 static int t5_write_combine = 0; 328 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 329 330 struct intrs_and_queues { 331 uint16_t intr_type; /* INTx, MSI, or MSI-X */ 332 uint16_t nirq; /* Total # of vectors */ 333 uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */ 334 uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */ 335 uint16_t ntxq10g; /* # of NIC txq's for each 10G port */ 336 uint16_t nrxq10g; /* # of NIC rxq's for each 10G port */ 337 uint16_t ntxq1g; /* # of NIC txq's for each 1G port */ 338 uint16_t nrxq1g; /* # of NIC rxq's for each 1G port */ 339 uint16_t rsrv_noflowq; /* Flag whether to reserve queue 0 */ 340 #ifdef TCP_OFFLOAD 341 uint16_t nofldtxq10g; /* # of TOE txq's for each 10G port */ 342 uint16_t nofldrxq10g; /* # of TOE rxq's for each 10G port */ 343 uint16_t nofldtxq1g; /* # of TOE txq's for each 1G port */ 344 uint16_t nofldrxq1g; /* # of TOE rxq's for each 1G port */ 345 #endif 346 #ifdef DEV_NETMAP 347 uint16_t nnmtxq10g; /* # of netmap txq's for each 10G port */ 348 uint16_t nnmrxq10g; /* # of netmap rxq's for each 10G port */ 349 uint16_t nnmtxq1g; /* # of netmap txq's for each 1G port */ 350 uint16_t nnmrxq1g; /* # of netmap rxq's for each 1G port */ 351 #endif 352 }; 353 354 struct filter_entry { 355 uint32_t valid:1; /* filter allocated and valid */ 356 uint32_t locked:1; /* filter is administratively locked */ 357 uint32_t pending:1; /* filter action is pending firmware reply */ 358 uint32_t smtidx:8; /* Source MAC Table index for smac */ 359 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 360 361 struct t4_filter_specification fs; 362 }; 363 364 static int map_bars_0_and_4(struct adapter *); 365 static int map_bar_2(struct adapter *); 366 static void setup_memwin(struct adapter *); 367 static int validate_mem_range(struct adapter *, uint32_t, int); 368 static int fwmtype_to_hwmtype(int); 369 static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 370 uint32_t *); 371 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 372 static uint32_t position_memwin(struct adapter *, int, uint32_t); 373 static int cfg_itype_and_nqueues(struct adapter *, int, int, 374 struct intrs_and_queues *); 375 static int prep_firmware(struct adapter *); 376 static int partition_resources(struct adapter *, const struct firmware *, 377 const char *); 378 static int get_params__pre_init(struct adapter *); 379 static int get_params__post_init(struct adapter *); 380 static int set_params__post_init(struct adapter *); 381 static void t4_set_desc(struct adapter *); 382 static void build_medialist(struct port_info *, struct ifmedia *); 383 static int cxgbe_init_synchronized(struct port_info *); 384 static int cxgbe_uninit_synchronized(struct port_info *); 385 static int setup_intr_handlers(struct adapter *); 386 static void quiesce_txq(struct adapter *, struct sge_txq *); 387 static void quiesce_wrq(struct adapter *, struct sge_wrq *); 388 static void quiesce_iq(struct adapter *, struct sge_iq *); 389 static void quiesce_fl(struct adapter *, struct sge_fl *); 390 static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 391 driver_intr_t *, void *, char *); 392 static int t4_free_irq(struct adapter *, struct irq *); 393 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 394 unsigned int); 395 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 396 static void cxgbe_refresh_stats(struct adapter *, struct port_info *); 397 static void cxgbe_tick(void *); 398 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 399 static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 400 struct mbuf *); 401 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 402 static int fw_msg_not_handled(struct adapter *, const __be64 *); 403 static int t4_sysctls(struct adapter *); 404 static int cxgbe_sysctls(struct port_info *); 405 static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 406 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 407 static int sysctl_btphy(SYSCTL_HANDLER_ARGS); 408 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); 409 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 410 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 411 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 412 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 413 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS); 414 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 415 static int sysctl_temperature(SYSCTL_HANDLER_ARGS); 416 #ifdef SBUF_DRAIN 417 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 418 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 419 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 420 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); 421 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); 422 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 423 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 424 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 425 static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 426 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 427 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 428 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 429 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); 430 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 431 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); 432 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 433 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 434 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 435 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 436 static int sysctl_tids(SYSCTL_HANDLER_ARGS); 437 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 438 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 439 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 440 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 441 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 442 #endif 443 static uint32_t fconf_to_mode(uint32_t); 444 static uint32_t mode_to_fconf(uint32_t); 445 static uint32_t fspec_to_fconf(struct t4_filter_specification *); 446 static int get_filter_mode(struct adapter *, uint32_t *); 447 static int set_filter_mode(struct adapter *, uint32_t); 448 static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 449 static int get_filter(struct adapter *, struct t4_filter *); 450 static int set_filter(struct adapter *, struct t4_filter *); 451 static int del_filter(struct adapter *, struct t4_filter *); 452 static void clear_filter(struct filter_entry *); 453 static int set_filter_wr(struct adapter *, int); 454 static int del_filter_wr(struct adapter *, int); 455 static int get_sge_context(struct adapter *, struct t4_sge_context *); 456 static int load_fw(struct adapter *, struct t4_data *); 457 static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 458 static int read_i2c(struct adapter *, struct t4_i2c_data *); 459 static int set_sched_class(struct adapter *, struct t4_sched_params *); 460 static int set_sched_queue(struct adapter *, struct t4_sched_queue *); 461 #ifdef TCP_OFFLOAD 462 static int toe_capability(struct port_info *, int); 463 #endif 464 static int mod_event(module_t, int, void *); 465 466 struct { 467 uint16_t device; 468 char *desc; 469 } t4_pciids[] = { 470 {0xa000, "Chelsio Terminator 4 FPGA"}, 471 {0x4400, "Chelsio T440-dbg"}, 472 {0x4401, "Chelsio T420-CR"}, 473 {0x4402, "Chelsio T422-CR"}, 474 {0x4403, "Chelsio T440-CR"}, 475 {0x4404, "Chelsio T420-BCH"}, 476 {0x4405, "Chelsio T440-BCH"}, 477 {0x4406, "Chelsio T440-CH"}, 478 {0x4407, "Chelsio T420-SO"}, 479 {0x4408, "Chelsio T420-CX"}, 480 {0x4409, "Chelsio T420-BT"}, 481 {0x440a, "Chelsio T404-BT"}, 482 {0x440e, "Chelsio T440-LP-CR"}, 483 }, t5_pciids[] = { 484 {0xb000, "Chelsio Terminator 5 FPGA"}, 485 {0x5400, "Chelsio T580-dbg"}, 486 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ 487 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ 488 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ 489 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ 490 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ 491 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ 492 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ 493 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ 494 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 495 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ 496 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ 497 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ 498 {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */ 499 #ifdef notyet 500 {0x5404, "Chelsio T520-BCH"}, 501 {0x5405, "Chelsio T540-BCH"}, 502 {0x5406, "Chelsio T540-CH"}, 503 {0x5408, "Chelsio T520-CX"}, 504 {0x540b, "Chelsio B520-SR"}, 505 {0x540c, "Chelsio B504-BT"}, 506 {0x540f, "Chelsio Amsterdam"}, 507 {0x5413, "Chelsio T580-CHR"}, 508 #endif 509 }; 510 511 #ifdef TCP_OFFLOAD 512 /* 513 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 514 * exactly the same for both rxq and ofld_rxq. 515 */ 516 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 517 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 518 #endif 519 520 /* No easy way to include t4_msg.h before adapter.h so we check this way */ 521 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 522 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 523 524 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); 525 526 static int 527 t4_probe(device_t dev) 528 { 529 int i; 530 uint16_t v = pci_get_vendor(dev); 531 uint16_t d = pci_get_device(dev); 532 uint8_t f = pci_get_function(dev); 533 534 if (v != PCI_VENDOR_ID_CHELSIO) 535 return (ENXIO); 536 537 /* Attach only to PF0 of the FPGA */ 538 if (d == 0xa000 && f != 0) 539 return (ENXIO); 540 541 for (i = 0; i < nitems(t4_pciids); i++) { 542 if (d == t4_pciids[i].device) { 543 device_set_desc(dev, t4_pciids[i].desc); 544 return (BUS_PROBE_DEFAULT); 545 } 546 } 547 548 return (ENXIO); 549 } 550 551 static int 552 t5_probe(device_t dev) 553 { 554 int i; 555 uint16_t v = pci_get_vendor(dev); 556 uint16_t d = pci_get_device(dev); 557 uint8_t f = pci_get_function(dev); 558 559 if (v != PCI_VENDOR_ID_CHELSIO) 560 return (ENXIO); 561 562 /* Attach only to PF0 of the FPGA */ 563 if (d == 0xb000 && f != 0) 564 return (ENXIO); 565 566 for (i = 0; i < nitems(t5_pciids); i++) { 567 if (d == t5_pciids[i].device) { 568 device_set_desc(dev, t5_pciids[i].desc); 569 return (BUS_PROBE_DEFAULT); 570 } 571 } 572 573 return (ENXIO); 574 } 575 576 static int 577 t4_attach(device_t dev) 578 { 579 struct adapter *sc; 580 int rc = 0, i, n10g, n1g, rqidx, tqidx; 581 struct intrs_and_queues iaq; 582 struct sge *s; 583 #ifdef TCP_OFFLOAD 584 int ofld_rqidx, ofld_tqidx; 585 #endif 586 #ifdef DEV_NETMAP 587 int nm_rqidx, nm_tqidx; 588 #endif 589 590 sc = device_get_softc(dev); 591 sc->dev = dev; 592 TUNABLE_INT_FETCH("hw.cxgbe.debug_flags", &sc->debug_flags); 593 594 pci_enable_busmaster(dev); 595 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 596 uint32_t v; 597 598 pci_set_max_read_req(dev, 4096); 599 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 600 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 601 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 602 603 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5); 604 } 605 606 sc->traceq = -1; 607 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); 608 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", 609 device_get_nameunit(dev)); 610 611 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 612 device_get_nameunit(dev)); 613 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 614 sx_xlock(&t4_list_lock); 615 SLIST_INSERT_HEAD(&t4_list, sc, link); 616 sx_xunlock(&t4_list_lock); 617 618 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 619 TAILQ_INIT(&sc->sfl); 620 callout_init(&sc->sfl_callout, 1); 621 622 mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF); 623 624 rc = map_bars_0_and_4(sc); 625 if (rc != 0) 626 goto done; /* error message displayed already */ 627 628 /* 629 * This is the real PF# to which we're attaching. Works from within PCI 630 * passthrough environments too, where pci_get_function() could return a 631 * different PF# depending on the passthrough configuration. We need to 632 * use the real PF# in all our communication with the firmware. 633 */ 634 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 635 sc->mbox = sc->pf; 636 637 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 638 sc->an_handler = an_not_handled; 639 for (i = 0; i < nitems(sc->cpl_handler); i++) 640 sc->cpl_handler[i] = cpl_not_handled; 641 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 642 sc->fw_msg_handler[i] = fw_msg_not_handled; 643 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 644 t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt); 645 t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt); 646 t4_init_sge_cpl_handlers(sc); 647 648 /* Prepare the adapter for operation */ 649 rc = -t4_prep_adapter(sc); 650 if (rc != 0) { 651 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 652 goto done; 653 } 654 655 /* 656 * Do this really early, with the memory windows set up even before the 657 * character device. The userland tool's register i/o and mem read 658 * will work even in "recovery mode". 659 */ 660 setup_memwin(sc); 661 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 662 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 663 device_get_nameunit(dev)); 664 if (sc->cdev == NULL) 665 device_printf(dev, "failed to create nexus char device.\n"); 666 else 667 sc->cdev->si_drv1 = sc; 668 669 /* Go no further if recovery mode has been requested. */ 670 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 671 device_printf(dev, "recovery mode.\n"); 672 goto done; 673 } 674 675 #if defined(__i386__) 676 if ((cpu_feature & CPUID_CX8) == 0) { 677 device_printf(dev, "64 bit atomics not available.\n"); 678 rc = ENOTSUP; 679 goto done; 680 } 681 #endif 682 683 /* Prepare the firmware for operation */ 684 rc = prep_firmware(sc); 685 if (rc != 0) 686 goto done; /* error message displayed already */ 687 688 rc = get_params__post_init(sc); 689 if (rc != 0) 690 goto done; /* error message displayed already */ 691 692 rc = set_params__post_init(sc); 693 if (rc != 0) 694 goto done; /* error message displayed already */ 695 696 rc = map_bar_2(sc); 697 if (rc != 0) 698 goto done; /* error message displayed already */ 699 700 rc = t4_create_dma_tag(sc); 701 if (rc != 0) 702 goto done; /* error message displayed already */ 703 704 /* 705 * First pass over all the ports - allocate VIs and initialize some 706 * basic parameters like mac address, port type, etc. We also figure 707 * out whether a port is 10G or 1G and use that information when 708 * calculating how many interrupts to attempt to allocate. 709 */ 710 n10g = n1g = 0; 711 for_each_port(sc, i) { 712 struct port_info *pi; 713 714 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 715 sc->port[i] = pi; 716 717 /* These must be set before t4_port_init */ 718 pi->adapter = sc; 719 pi->port_id = i; 720 721 /* Allocate the vi and initialize parameters like mac addr */ 722 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 723 if (rc != 0) { 724 device_printf(dev, "unable to initialize port %d: %d\n", 725 i, rc); 726 free(pi, M_CXGBE); 727 sc->port[i] = NULL; 728 goto done; 729 } 730 731 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX); 732 pi->link_cfg.requested_fc |= t4_pause_settings; 733 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX); 734 pi->link_cfg.fc |= t4_pause_settings; 735 736 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 737 if (rc != 0) { 738 device_printf(dev, "port %d l1cfg failed: %d\n", i, rc); 739 free(pi, M_CXGBE); 740 sc->port[i] = NULL; 741 goto done; 742 } 743 744 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 745 device_get_nameunit(dev), i); 746 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 747 sc->chan_map[pi->tx_chan] = i; 748 749 if (is_10G_port(pi) || is_40G_port(pi)) { 750 n10g++; 751 pi->tmr_idx = t4_tmr_idx_10g; 752 pi->pktc_idx = t4_pktc_idx_10g; 753 } else { 754 n1g++; 755 pi->tmr_idx = t4_tmr_idx_1g; 756 pi->pktc_idx = t4_pktc_idx_1g; 757 } 758 759 pi->xact_addr_filt = -1; 760 pi->linkdnrc = -1; 761 762 pi->qsize_rxq = t4_qsize_rxq; 763 pi->qsize_txq = t4_qsize_txq; 764 765 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 766 if (pi->dev == NULL) { 767 device_printf(dev, 768 "failed to add device for port %d.\n", i); 769 rc = ENXIO; 770 goto done; 771 } 772 device_set_softc(pi->dev, pi); 773 } 774 775 /* 776 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 777 */ 778 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 779 if (rc != 0) 780 goto done; /* error message displayed already */ 781 782 sc->intr_type = iaq.intr_type; 783 sc->intr_count = iaq.nirq; 784 785 s = &sc->sge; 786 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 787 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 788 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 789 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 790 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 791 #ifdef TCP_OFFLOAD 792 if (is_offload(sc)) { 793 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 794 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 795 s->neq += s->nofldtxq + s->nofldrxq; 796 s->niq += s->nofldrxq; 797 798 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 799 M_CXGBE, M_ZERO | M_WAITOK); 800 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 801 M_CXGBE, M_ZERO | M_WAITOK); 802 } 803 #endif 804 #ifdef DEV_NETMAP 805 s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g; 806 s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g; 807 s->neq += s->nnmtxq + s->nnmrxq; 808 s->niq += s->nnmrxq; 809 810 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq), 811 M_CXGBE, M_ZERO | M_WAITOK); 812 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq), 813 M_CXGBE, M_ZERO | M_WAITOK); 814 #endif 815 816 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 817 M_ZERO | M_WAITOK); 818 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 819 M_ZERO | M_WAITOK); 820 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 821 M_ZERO | M_WAITOK); 822 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 823 M_ZERO | M_WAITOK); 824 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 825 M_ZERO | M_WAITOK); 826 827 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 828 M_ZERO | M_WAITOK); 829 830 t4_init_l2t(sc, M_WAITOK); 831 832 /* 833 * Second pass over the ports. This time we know the number of rx and 834 * tx queues that each port should get. 835 */ 836 rqidx = tqidx = 0; 837 #ifdef TCP_OFFLOAD 838 ofld_rqidx = ofld_tqidx = 0; 839 #endif 840 #ifdef DEV_NETMAP 841 nm_rqidx = nm_tqidx = 0; 842 #endif 843 for_each_port(sc, i) { 844 struct port_info *pi = sc->port[i]; 845 846 if (pi == NULL) 847 continue; 848 849 pi->first_rxq = rqidx; 850 pi->first_txq = tqidx; 851 if (is_10G_port(pi) || is_40G_port(pi)) { 852 pi->flags |= iaq.intr_flags_10g; 853 pi->nrxq = iaq.nrxq10g; 854 pi->ntxq = iaq.ntxq10g; 855 } else { 856 pi->flags |= iaq.intr_flags_1g; 857 pi->nrxq = iaq.nrxq1g; 858 pi->ntxq = iaq.ntxq1g; 859 } 860 861 if (pi->ntxq > 1) 862 pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0; 863 else 864 pi->rsrv_noflowq = 0; 865 866 rqidx += pi->nrxq; 867 tqidx += pi->ntxq; 868 #ifdef TCP_OFFLOAD 869 if (is_offload(sc)) { 870 pi->first_ofld_rxq = ofld_rqidx; 871 pi->first_ofld_txq = ofld_tqidx; 872 if (is_10G_port(pi) || is_40G_port(pi)) { 873 pi->nofldrxq = iaq.nofldrxq10g; 874 pi->nofldtxq = iaq.nofldtxq10g; 875 } else { 876 pi->nofldrxq = iaq.nofldrxq1g; 877 pi->nofldtxq = iaq.nofldtxq1g; 878 } 879 ofld_rqidx += pi->nofldrxq; 880 ofld_tqidx += pi->nofldtxq; 881 } 882 #endif 883 #ifdef DEV_NETMAP 884 pi->first_nm_rxq = nm_rqidx; 885 pi->first_nm_txq = nm_tqidx; 886 if (is_10G_port(pi) || is_40G_port(pi)) { 887 pi->nnmrxq = iaq.nnmrxq10g; 888 pi->nnmtxq = iaq.nnmtxq10g; 889 } else { 890 pi->nnmrxq = iaq.nnmrxq1g; 891 pi->nnmtxq = iaq.nnmtxq1g; 892 } 893 nm_rqidx += pi->nnmrxq; 894 nm_tqidx += pi->nnmtxq; 895 #endif 896 } 897 898 rc = setup_intr_handlers(sc); 899 if (rc != 0) { 900 device_printf(dev, 901 "failed to setup interrupt handlers: %d\n", rc); 902 goto done; 903 } 904 905 rc = bus_generic_attach(dev); 906 if (rc != 0) { 907 device_printf(dev, 908 "failed to attach all child ports: %d\n", rc); 909 goto done; 910 } 911 912 device_printf(dev, 913 "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 914 sc->params.pci.speed, sc->params.pci.width, sc->params.nports, 915 sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" : 916 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 917 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 918 919 t4_set_desc(sc); 920 921 done: 922 if (rc != 0 && sc->cdev) { 923 /* cdev was created and so cxgbetool works; recover that way. */ 924 device_printf(dev, 925 "error during attach, adapter is now in recovery mode.\n"); 926 rc = 0; 927 } 928 929 if (rc != 0) 930 t4_detach(dev); 931 else 932 t4_sysctls(sc); 933 934 return (rc); 935 } 936 937 /* 938 * Idempotent 939 */ 940 static int 941 t4_detach(device_t dev) 942 { 943 struct adapter *sc; 944 struct port_info *pi; 945 int i, rc; 946 947 sc = device_get_softc(dev); 948 949 if (sc->flags & FULL_INIT_DONE) 950 t4_intr_disable(sc); 951 952 if (sc->cdev) { 953 destroy_dev(sc->cdev); 954 sc->cdev = NULL; 955 } 956 957 rc = bus_generic_detach(dev); 958 if (rc) { 959 device_printf(dev, 960 "failed to detach child devices: %d\n", rc); 961 return (rc); 962 } 963 964 for (i = 0; i < sc->intr_count; i++) 965 t4_free_irq(sc, &sc->irq[i]); 966 967 for (i = 0; i < MAX_NPORTS; i++) { 968 pi = sc->port[i]; 969 if (pi) { 970 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid); 971 if (pi->dev) 972 device_delete_child(dev, pi->dev); 973 974 mtx_destroy(&pi->pi_lock); 975 free(pi, M_CXGBE); 976 } 977 } 978 979 if (sc->flags & FULL_INIT_DONE) 980 adapter_full_uninit(sc); 981 982 if (sc->flags & FW_OK) 983 t4_fw_bye(sc, sc->mbox); 984 985 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 986 pci_release_msi(dev); 987 988 if (sc->regs_res) 989 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 990 sc->regs_res); 991 992 if (sc->udbs_res) 993 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 994 sc->udbs_res); 995 996 if (sc->msix_res) 997 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 998 sc->msix_res); 999 1000 if (sc->l2t) 1001 t4_free_l2t(sc->l2t); 1002 1003 #ifdef TCP_OFFLOAD 1004 free(sc->sge.ofld_rxq, M_CXGBE); 1005 free(sc->sge.ofld_txq, M_CXGBE); 1006 #endif 1007 #ifdef DEV_NETMAP 1008 free(sc->sge.nm_rxq, M_CXGBE); 1009 free(sc->sge.nm_txq, M_CXGBE); 1010 #endif 1011 free(sc->irq, M_CXGBE); 1012 free(sc->sge.rxq, M_CXGBE); 1013 free(sc->sge.txq, M_CXGBE); 1014 free(sc->sge.ctrlq, M_CXGBE); 1015 free(sc->sge.iqmap, M_CXGBE); 1016 free(sc->sge.eqmap, M_CXGBE); 1017 free(sc->tids.ftid_tab, M_CXGBE); 1018 t4_destroy_dma_tag(sc); 1019 if (mtx_initialized(&sc->sc_lock)) { 1020 sx_xlock(&t4_list_lock); 1021 SLIST_REMOVE(&t4_list, sc, adapter, link); 1022 sx_xunlock(&t4_list_lock); 1023 mtx_destroy(&sc->sc_lock); 1024 } 1025 1026 if (mtx_initialized(&sc->tids.ftid_lock)) 1027 mtx_destroy(&sc->tids.ftid_lock); 1028 if (mtx_initialized(&sc->sfl_lock)) 1029 mtx_destroy(&sc->sfl_lock); 1030 if (mtx_initialized(&sc->ifp_lock)) 1031 mtx_destroy(&sc->ifp_lock); 1032 if (mtx_initialized(&sc->regwin_lock)) 1033 mtx_destroy(&sc->regwin_lock); 1034 1035 bzero(sc, sizeof(*sc)); 1036 1037 return (0); 1038 } 1039 1040 static int 1041 cxgbe_probe(device_t dev) 1042 { 1043 char buf[128]; 1044 struct port_info *pi = device_get_softc(dev); 1045 1046 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 1047 device_set_desc_copy(dev, buf); 1048 1049 return (BUS_PROBE_DEFAULT); 1050 } 1051 1052 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 1053 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 1054 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS) 1055 #define T4_CAP_ENABLE (T4_CAP) 1056 1057 static int 1058 cxgbe_attach(device_t dev) 1059 { 1060 struct port_info *pi = device_get_softc(dev); 1061 struct ifnet *ifp; 1062 char *s; 1063 int n, o; 1064 1065 /* Allocate an ifnet and set it up */ 1066 ifp = if_alloc(IFT_ETHER); 1067 if (ifp == NULL) { 1068 device_printf(dev, "Cannot allocate ifnet\n"); 1069 return (ENOMEM); 1070 } 1071 pi->ifp = ifp; 1072 ifp->if_softc = pi; 1073 1074 callout_init(&pi->tick, 1); 1075 1076 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1077 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1078 1079 ifp->if_init = cxgbe_init; 1080 ifp->if_ioctl = cxgbe_ioctl; 1081 ifp->if_transmit = cxgbe_transmit; 1082 ifp->if_qflush = cxgbe_qflush; 1083 ifp->if_get_counter = cxgbe_get_counter; 1084 1085 ifp->if_capabilities = T4_CAP; 1086 #ifdef TCP_OFFLOAD 1087 if (is_offload(pi->adapter)) 1088 ifp->if_capabilities |= IFCAP_TOE; 1089 #endif 1090 ifp->if_capenable = T4_CAP_ENABLE; 1091 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1092 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1093 1094 ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1095 ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS; 1096 ifp->if_hw_tsomaxsegsize = 65536; 1097 1098 /* Initialize ifmedia for this port */ 1099 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 1100 cxgbe_media_status); 1101 build_medialist(pi, &pi->media); 1102 1103 pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 1104 EVENTHANDLER_PRI_ANY); 1105 1106 ether_ifattach(ifp, pi->hw_addr); 1107 1108 n = 128; 1109 s = malloc(n, M_CXGBE, M_WAITOK); 1110 o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq); 1111 MPASS(n > o); 1112 #ifdef TCP_OFFLOAD 1113 if (is_offload(pi->adapter)) { 1114 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)", 1115 pi->nofldtxq, pi->nofldrxq); 1116 MPASS(n > o); 1117 } 1118 #endif 1119 #ifdef DEV_NETMAP 1120 o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq, 1121 pi->nnmrxq); 1122 MPASS(n > o); 1123 #endif 1124 device_printf(dev, "%s\n", s); 1125 free(s, M_CXGBE); 1126 1127 #ifdef DEV_NETMAP 1128 /* nm_media handled here to keep implementation private to this file */ 1129 ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change, 1130 cxgbe_media_status); 1131 build_medialist(pi, &pi->nm_media); 1132 create_netmap_ifnet(pi); /* logs errors it something fails */ 1133 #endif 1134 cxgbe_sysctls(pi); 1135 1136 return (0); 1137 } 1138 1139 static int 1140 cxgbe_detach(device_t dev) 1141 { 1142 struct port_info *pi = device_get_softc(dev); 1143 struct adapter *sc = pi->adapter; 1144 struct ifnet *ifp = pi->ifp; 1145 1146 /* Tell if_ioctl and if_init that the port is going away */ 1147 ADAPTER_LOCK(sc); 1148 SET_DOOMED(pi); 1149 wakeup(&sc->flags); 1150 while (IS_BUSY(sc)) 1151 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 1152 SET_BUSY(sc); 1153 #ifdef INVARIANTS 1154 sc->last_op = "t4detach"; 1155 sc->last_op_thr = curthread; 1156 sc->last_op_flags = 0; 1157 #endif 1158 ADAPTER_UNLOCK(sc); 1159 1160 if (pi->flags & HAS_TRACEQ) { 1161 sc->traceq = -1; /* cloner should not create ifnet */ 1162 t4_tracer_port_detach(sc); 1163 } 1164 1165 if (pi->vlan_c) 1166 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c); 1167 1168 PORT_LOCK(pi); 1169 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1170 callout_stop(&pi->tick); 1171 PORT_UNLOCK(pi); 1172 callout_drain(&pi->tick); 1173 1174 /* Let detach proceed even if these fail. */ 1175 cxgbe_uninit_synchronized(pi); 1176 port_full_uninit(pi); 1177 1178 ifmedia_removeall(&pi->media); 1179 ether_ifdetach(pi->ifp); 1180 if_free(pi->ifp); 1181 1182 #ifdef DEV_NETMAP 1183 /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */ 1184 destroy_netmap_ifnet(pi); 1185 #endif 1186 1187 ADAPTER_LOCK(sc); 1188 CLR_BUSY(sc); 1189 wakeup(&sc->flags); 1190 ADAPTER_UNLOCK(sc); 1191 1192 return (0); 1193 } 1194 1195 static void 1196 cxgbe_init(void *arg) 1197 { 1198 struct port_info *pi = arg; 1199 struct adapter *sc = pi->adapter; 1200 1201 if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0) 1202 return; 1203 cxgbe_init_synchronized(pi); 1204 end_synchronized_op(sc, 0); 1205 } 1206 1207 static int 1208 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1209 { 1210 int rc = 0, mtu, flags, can_sleep; 1211 struct port_info *pi = ifp->if_softc; 1212 struct adapter *sc = pi->adapter; 1213 struct ifreq *ifr = (struct ifreq *)data; 1214 uint32_t mask; 1215 1216 switch (cmd) { 1217 case SIOCSIFMTU: 1218 mtu = ifr->ifr_mtu; 1219 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1220 return (EINVAL); 1221 1222 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu"); 1223 if (rc) 1224 return (rc); 1225 ifp->if_mtu = mtu; 1226 if (pi->flags & PORT_INIT_DONE) { 1227 t4_update_fl_bufsize(ifp); 1228 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1229 rc = update_mac_settings(ifp, XGMAC_MTU); 1230 } 1231 end_synchronized_op(sc, 0); 1232 break; 1233 1234 case SIOCSIFFLAGS: 1235 can_sleep = 0; 1236 redo_sifflags: 1237 rc = begin_synchronized_op(sc, pi, 1238 can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg"); 1239 if (rc) 1240 return (rc); 1241 1242 if (ifp->if_flags & IFF_UP) { 1243 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1244 flags = pi->if_flags; 1245 if ((ifp->if_flags ^ flags) & 1246 (IFF_PROMISC | IFF_ALLMULTI)) { 1247 if (can_sleep == 1) { 1248 end_synchronized_op(sc, 0); 1249 can_sleep = 0; 1250 goto redo_sifflags; 1251 } 1252 rc = update_mac_settings(ifp, 1253 XGMAC_PROMISC | XGMAC_ALLMULTI); 1254 } 1255 } else { 1256 if (can_sleep == 0) { 1257 end_synchronized_op(sc, LOCK_HELD); 1258 can_sleep = 1; 1259 goto redo_sifflags; 1260 } 1261 rc = cxgbe_init_synchronized(pi); 1262 } 1263 pi->if_flags = ifp->if_flags; 1264 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1265 if (can_sleep == 0) { 1266 end_synchronized_op(sc, LOCK_HELD); 1267 can_sleep = 1; 1268 goto redo_sifflags; 1269 } 1270 rc = cxgbe_uninit_synchronized(pi); 1271 } 1272 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD); 1273 break; 1274 1275 case SIOCADDMULTI: 1276 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1277 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi"); 1278 if (rc) 1279 return (rc); 1280 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1281 rc = update_mac_settings(ifp, XGMAC_MCADDRS); 1282 end_synchronized_op(sc, LOCK_HELD); 1283 break; 1284 1285 case SIOCSIFCAP: 1286 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap"); 1287 if (rc) 1288 return (rc); 1289 1290 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1291 if (mask & IFCAP_TXCSUM) { 1292 ifp->if_capenable ^= IFCAP_TXCSUM; 1293 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1294 1295 if (IFCAP_TSO4 & ifp->if_capenable && 1296 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1297 ifp->if_capenable &= ~IFCAP_TSO4; 1298 if_printf(ifp, 1299 "tso4 disabled due to -txcsum.\n"); 1300 } 1301 } 1302 if (mask & IFCAP_TXCSUM_IPV6) { 1303 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1304 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1305 1306 if (IFCAP_TSO6 & ifp->if_capenable && 1307 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1308 ifp->if_capenable &= ~IFCAP_TSO6; 1309 if_printf(ifp, 1310 "tso6 disabled due to -txcsum6.\n"); 1311 } 1312 } 1313 if (mask & IFCAP_RXCSUM) 1314 ifp->if_capenable ^= IFCAP_RXCSUM; 1315 if (mask & IFCAP_RXCSUM_IPV6) 1316 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1317 1318 /* 1319 * Note that we leave CSUM_TSO alone (it is always set). The 1320 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1321 * sending a TSO request our way, so it's sufficient to toggle 1322 * IFCAP_TSOx only. 1323 */ 1324 if (mask & IFCAP_TSO4) { 1325 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1326 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1327 if_printf(ifp, "enable txcsum first.\n"); 1328 rc = EAGAIN; 1329 goto fail; 1330 } 1331 ifp->if_capenable ^= IFCAP_TSO4; 1332 } 1333 if (mask & IFCAP_TSO6) { 1334 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1335 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1336 if_printf(ifp, "enable txcsum6 first.\n"); 1337 rc = EAGAIN; 1338 goto fail; 1339 } 1340 ifp->if_capenable ^= IFCAP_TSO6; 1341 } 1342 if (mask & IFCAP_LRO) { 1343 #if defined(INET) || defined(INET6) 1344 int i; 1345 struct sge_rxq *rxq; 1346 1347 ifp->if_capenable ^= IFCAP_LRO; 1348 for_each_rxq(pi, i, rxq) { 1349 if (ifp->if_capenable & IFCAP_LRO) 1350 rxq->iq.flags |= IQ_LRO_ENABLED; 1351 else 1352 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1353 } 1354 #endif 1355 } 1356 #ifdef TCP_OFFLOAD 1357 if (mask & IFCAP_TOE) { 1358 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1359 1360 rc = toe_capability(pi, enable); 1361 if (rc != 0) 1362 goto fail; 1363 1364 ifp->if_capenable ^= mask; 1365 } 1366 #endif 1367 if (mask & IFCAP_VLAN_HWTAGGING) { 1368 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1369 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1370 rc = update_mac_settings(ifp, XGMAC_VLANEX); 1371 } 1372 if (mask & IFCAP_VLAN_MTU) { 1373 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1374 1375 /* Need to find out how to disable auto-mtu-inflation */ 1376 } 1377 if (mask & IFCAP_VLAN_HWTSO) 1378 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1379 if (mask & IFCAP_VLAN_HWCSUM) 1380 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1381 1382 #ifdef VLAN_CAPABILITIES 1383 VLAN_CAPABILITIES(ifp); 1384 #endif 1385 fail: 1386 end_synchronized_op(sc, 0); 1387 break; 1388 1389 case SIOCSIFMEDIA: 1390 case SIOCGIFMEDIA: 1391 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1392 break; 1393 1394 case SIOCGI2C: { 1395 struct ifi2creq i2c; 1396 1397 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); 1398 if (rc != 0) 1399 break; 1400 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) { 1401 rc = EPERM; 1402 break; 1403 } 1404 if (i2c.len > sizeof(i2c.data)) { 1405 rc = EINVAL; 1406 break; 1407 } 1408 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4i2c"); 1409 if (rc) 1410 return (rc); 1411 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr, 1412 i2c.offset, i2c.len, &i2c.data[0]); 1413 end_synchronized_op(sc, 0); 1414 if (rc == 0) 1415 rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); 1416 break; 1417 } 1418 1419 default: 1420 rc = ether_ioctl(ifp, cmd, data); 1421 } 1422 1423 return (rc); 1424 } 1425 1426 static int 1427 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1428 { 1429 struct port_info *pi = ifp->if_softc; 1430 struct adapter *sc = pi->adapter; 1431 struct sge_txq *txq; 1432 void *items[1]; 1433 int rc; 1434 1435 M_ASSERTPKTHDR(m); 1436 MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */ 1437 1438 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1439 m_freem(m); 1440 return (ENETDOWN); 1441 } 1442 1443 rc = parse_pkt(&m); 1444 if (__predict_false(rc != 0)) { 1445 MPASS(m == NULL); /* was freed already */ 1446 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */ 1447 return (rc); 1448 } 1449 1450 /* Select a txq. */ 1451 txq = &sc->sge.txq[pi->first_txq]; 1452 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 1453 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) + 1454 pi->rsrv_noflowq); 1455 1456 items[0] = m; 1457 rc = mp_ring_enqueue(txq->r, items, 1, 4096); 1458 if (__predict_false(rc != 0)) 1459 m_freem(m); 1460 1461 return (rc); 1462 } 1463 1464 static void 1465 cxgbe_qflush(struct ifnet *ifp) 1466 { 1467 struct port_info *pi = ifp->if_softc; 1468 struct sge_txq *txq; 1469 int i; 1470 1471 /* queues do not exist if !PORT_INIT_DONE. */ 1472 if (pi->flags & PORT_INIT_DONE) { 1473 for_each_txq(pi, i, txq) { 1474 TXQ_LOCK(txq); 1475 txq->eq.flags &= ~EQ_ENABLED; 1476 TXQ_UNLOCK(txq); 1477 while (!mp_ring_is_idle(txq->r)) { 1478 mp_ring_check_drainage(txq->r, 0); 1479 pause("qflush", 1); 1480 } 1481 } 1482 } 1483 if_qflush(ifp); 1484 } 1485 1486 static uint64_t 1487 cxgbe_get_counter(struct ifnet *ifp, ift_counter c) 1488 { 1489 struct port_info *pi = ifp->if_softc; 1490 struct adapter *sc = pi->adapter; 1491 struct port_stats *s = &pi->stats; 1492 1493 cxgbe_refresh_stats(sc, pi); 1494 1495 switch (c) { 1496 case IFCOUNTER_IPACKETS: 1497 return (s->rx_frames - s->rx_pause); 1498 1499 case IFCOUNTER_IERRORS: 1500 return (s->rx_jabber + s->rx_runt + s->rx_too_long + 1501 s->rx_fcs_err + s->rx_len_err); 1502 1503 case IFCOUNTER_OPACKETS: 1504 return (s->tx_frames - s->tx_pause); 1505 1506 case IFCOUNTER_OERRORS: 1507 return (s->tx_error_frames); 1508 1509 case IFCOUNTER_IBYTES: 1510 return (s->rx_octets - s->rx_pause * 64); 1511 1512 case IFCOUNTER_OBYTES: 1513 return (s->tx_octets - s->tx_pause * 64); 1514 1515 case IFCOUNTER_IMCASTS: 1516 return (s->rx_mcast_frames - s->rx_pause); 1517 1518 case IFCOUNTER_OMCASTS: 1519 return (s->tx_mcast_frames - s->tx_pause); 1520 1521 case IFCOUNTER_IQDROPS: 1522 return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 1523 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 1524 s->rx_trunc3 + pi->tnl_cong_drops); 1525 1526 case IFCOUNTER_OQDROPS: { 1527 uint64_t drops; 1528 1529 drops = s->tx_drop; 1530 if (pi->flags & PORT_INIT_DONE) { 1531 int i; 1532 struct sge_txq *txq; 1533 1534 for_each_txq(pi, i, txq) 1535 drops += counter_u64_fetch(txq->r->drops); 1536 } 1537 1538 return (drops); 1539 1540 } 1541 1542 default: 1543 return (if_get_counter_default(ifp, c)); 1544 } 1545 } 1546 1547 static int 1548 cxgbe_media_change(struct ifnet *ifp) 1549 { 1550 struct port_info *pi = ifp->if_softc; 1551 1552 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1553 1554 return (EOPNOTSUPP); 1555 } 1556 1557 static void 1558 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1559 { 1560 struct port_info *pi = ifp->if_softc; 1561 struct ifmedia *media = NULL; 1562 struct ifmedia_entry *cur; 1563 int speed = pi->link_cfg.speed; 1564 1565 if (ifp == pi->ifp) 1566 media = &pi->media; 1567 #ifdef DEV_NETMAP 1568 else if (ifp == pi->nm_ifp) 1569 media = &pi->nm_media; 1570 #endif 1571 MPASS(media != NULL); 1572 1573 cur = media->ifm_cur; 1574 1575 ifmr->ifm_status = IFM_AVALID; 1576 if (!pi->link_cfg.link_ok) 1577 return; 1578 1579 ifmr->ifm_status |= IFM_ACTIVE; 1580 1581 /* active and current will differ iff current media is autoselect. */ 1582 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1583 return; 1584 1585 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1586 if (speed == SPEED_10000) 1587 ifmr->ifm_active |= IFM_10G_T; 1588 else if (speed == SPEED_1000) 1589 ifmr->ifm_active |= IFM_1000_T; 1590 else if (speed == SPEED_100) 1591 ifmr->ifm_active |= IFM_100_TX; 1592 else if (speed == SPEED_10) 1593 ifmr->ifm_active |= IFM_10_T; 1594 else 1595 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1596 speed)); 1597 } 1598 1599 void 1600 t4_fatal_err(struct adapter *sc) 1601 { 1602 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1603 t4_intr_disable(sc); 1604 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1605 device_get_nameunit(sc->dev)); 1606 } 1607 1608 static int 1609 map_bars_0_and_4(struct adapter *sc) 1610 { 1611 sc->regs_rid = PCIR_BAR(0); 1612 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1613 &sc->regs_rid, RF_ACTIVE); 1614 if (sc->regs_res == NULL) { 1615 device_printf(sc->dev, "cannot map registers.\n"); 1616 return (ENXIO); 1617 } 1618 sc->bt = rman_get_bustag(sc->regs_res); 1619 sc->bh = rman_get_bushandle(sc->regs_res); 1620 sc->mmio_len = rman_get_size(sc->regs_res); 1621 setbit(&sc->doorbells, DOORBELL_KDB); 1622 1623 sc->msix_rid = PCIR_BAR(4); 1624 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1625 &sc->msix_rid, RF_ACTIVE); 1626 if (sc->msix_res == NULL) { 1627 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1628 return (ENXIO); 1629 } 1630 1631 return (0); 1632 } 1633 1634 static int 1635 map_bar_2(struct adapter *sc) 1636 { 1637 1638 /* 1639 * T4: only iWARP driver uses the userspace doorbells. There is no need 1640 * to map it if RDMA is disabled. 1641 */ 1642 if (is_t4(sc) && sc->rdmacaps == 0) 1643 return (0); 1644 1645 sc->udbs_rid = PCIR_BAR(2); 1646 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1647 &sc->udbs_rid, RF_ACTIVE); 1648 if (sc->udbs_res == NULL) { 1649 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1650 return (ENXIO); 1651 } 1652 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1653 1654 if (is_t5(sc)) { 1655 setbit(&sc->doorbells, DOORBELL_UDB); 1656 #if defined(__i386__) || defined(__amd64__) 1657 if (t5_write_combine) { 1658 int rc; 1659 1660 /* 1661 * Enable write combining on BAR2. This is the 1662 * userspace doorbell BAR and is split into 128B 1663 * (UDBS_SEG_SIZE) doorbell regions, each associated 1664 * with an egress queue. The first 64B has the doorbell 1665 * and the second 64B can be used to submit a tx work 1666 * request with an implicit doorbell. 1667 */ 1668 1669 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1670 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1671 if (rc == 0) { 1672 clrbit(&sc->doorbells, DOORBELL_UDB); 1673 setbit(&sc->doorbells, DOORBELL_WCWR); 1674 setbit(&sc->doorbells, DOORBELL_UDBWC); 1675 } else { 1676 device_printf(sc->dev, 1677 "couldn't enable write combining: %d\n", 1678 rc); 1679 } 1680 1681 t4_write_reg(sc, A_SGE_STAT_CFG, 1682 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1683 } 1684 #endif 1685 } 1686 1687 return (0); 1688 } 1689 1690 static const struct memwin t4_memwin[] = { 1691 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1692 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1693 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1694 }; 1695 1696 static const struct memwin t5_memwin[] = { 1697 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1698 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1699 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1700 }; 1701 1702 static void 1703 setup_memwin(struct adapter *sc) 1704 { 1705 const struct memwin *mw; 1706 int i, n; 1707 uint32_t bar0; 1708 1709 if (is_t4(sc)) { 1710 /* 1711 * Read low 32b of bar0 indirectly via the hardware backdoor 1712 * mechanism. Works from within PCI passthrough environments 1713 * too, where rman_get_start() can return a different value. We 1714 * need to program the T4 memory window decoders with the actual 1715 * addresses that will be coming across the PCIe link. 1716 */ 1717 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1718 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1719 1720 mw = &t4_memwin[0]; 1721 n = nitems(t4_memwin); 1722 } else { 1723 /* T5 uses the relative offset inside the PCIe BAR */ 1724 bar0 = 0; 1725 1726 mw = &t5_memwin[0]; 1727 n = nitems(t5_memwin); 1728 } 1729 1730 for (i = 0; i < n; i++, mw++) { 1731 t4_write_reg(sc, 1732 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 1733 (mw->base + bar0) | V_BIR(0) | 1734 V_WINDOW(ilog2(mw->aperture) - 10)); 1735 } 1736 1737 /* flush */ 1738 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1739 } 1740 1741 /* 1742 * Verify that the memory range specified by the addr/len pair is valid and lies 1743 * entirely within a single region (EDCx or MCx). 1744 */ 1745 static int 1746 validate_mem_range(struct adapter *sc, uint32_t addr, int len) 1747 { 1748 uint32_t em, addr_len, maddr, mlen; 1749 1750 /* Memory can only be accessed in naturally aligned 4 byte units */ 1751 if (addr & 3 || len & 3 || len == 0) 1752 return (EINVAL); 1753 1754 /* Enabled memories */ 1755 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1756 if (em & F_EDRAM0_ENABLE) { 1757 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1758 maddr = G_EDRAM0_BASE(addr_len) << 20; 1759 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1760 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1761 addr + len <= maddr + mlen) 1762 return (0); 1763 } 1764 if (em & F_EDRAM1_ENABLE) { 1765 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1766 maddr = G_EDRAM1_BASE(addr_len) << 20; 1767 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1768 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1769 addr + len <= maddr + mlen) 1770 return (0); 1771 } 1772 if (em & F_EXT_MEM_ENABLE) { 1773 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1774 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1775 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1776 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1777 addr + len <= maddr + mlen) 1778 return (0); 1779 } 1780 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 1781 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1782 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1783 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1784 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1785 addr + len <= maddr + mlen) 1786 return (0); 1787 } 1788 1789 return (EFAULT); 1790 } 1791 1792 static int 1793 fwmtype_to_hwmtype(int mtype) 1794 { 1795 1796 switch (mtype) { 1797 case FW_MEMTYPE_EDC0: 1798 return (MEM_EDC0); 1799 case FW_MEMTYPE_EDC1: 1800 return (MEM_EDC1); 1801 case FW_MEMTYPE_EXTMEM: 1802 return (MEM_MC0); 1803 case FW_MEMTYPE_EXTMEM1: 1804 return (MEM_MC1); 1805 default: 1806 panic("%s: cannot translate fw mtype %d.", __func__, mtype); 1807 } 1808 } 1809 1810 /* 1811 * Verify that the memory range specified by the memtype/offset/len pair is 1812 * valid and lies entirely within the memtype specified. The global address of 1813 * the start of the range is returned in addr. 1814 */ 1815 static int 1816 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 1817 uint32_t *addr) 1818 { 1819 uint32_t em, addr_len, maddr, mlen; 1820 1821 /* Memory can only be accessed in naturally aligned 4 byte units */ 1822 if (off & 3 || len & 3 || len == 0) 1823 return (EINVAL); 1824 1825 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1826 switch (fwmtype_to_hwmtype(mtype)) { 1827 case MEM_EDC0: 1828 if (!(em & F_EDRAM0_ENABLE)) 1829 return (EINVAL); 1830 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1831 maddr = G_EDRAM0_BASE(addr_len) << 20; 1832 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1833 break; 1834 case MEM_EDC1: 1835 if (!(em & F_EDRAM1_ENABLE)) 1836 return (EINVAL); 1837 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1838 maddr = G_EDRAM1_BASE(addr_len) << 20; 1839 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1840 break; 1841 case MEM_MC: 1842 if (!(em & F_EXT_MEM_ENABLE)) 1843 return (EINVAL); 1844 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1845 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1846 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1847 break; 1848 case MEM_MC1: 1849 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 1850 return (EINVAL); 1851 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1852 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1853 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1854 break; 1855 default: 1856 return (EINVAL); 1857 } 1858 1859 if (mlen > 0 && off < mlen && off + len <= mlen) { 1860 *addr = maddr + off; /* global address */ 1861 return (0); 1862 } 1863 1864 return (EFAULT); 1865 } 1866 1867 static void 1868 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 1869 { 1870 const struct memwin *mw; 1871 1872 if (is_t4(sc)) { 1873 KASSERT(win >= 0 && win < nitems(t4_memwin), 1874 ("%s: incorrect memwin# (%d)", __func__, win)); 1875 mw = &t4_memwin[win]; 1876 } else { 1877 KASSERT(win >= 0 && win < nitems(t5_memwin), 1878 ("%s: incorrect memwin# (%d)", __func__, win)); 1879 mw = &t5_memwin[win]; 1880 } 1881 1882 if (base != NULL) 1883 *base = mw->base; 1884 if (aperture != NULL) 1885 *aperture = mw->aperture; 1886 } 1887 1888 /* 1889 * Positions the memory window such that it can be used to access the specified 1890 * address in the chip's address space. The return value is the offset of addr 1891 * from the start of the window. 1892 */ 1893 static uint32_t 1894 position_memwin(struct adapter *sc, int n, uint32_t addr) 1895 { 1896 uint32_t start, pf; 1897 uint32_t reg; 1898 1899 KASSERT(n >= 0 && n <= 3, 1900 ("%s: invalid window %d.", __func__, n)); 1901 KASSERT((addr & 3) == 0, 1902 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 1903 1904 if (is_t4(sc)) { 1905 pf = 0; 1906 start = addr & ~0xf; /* start must be 16B aligned */ 1907 } else { 1908 pf = V_PFNUM(sc->pf); 1909 start = addr & ~0x7f; /* start must be 128B aligned */ 1910 } 1911 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 1912 1913 t4_write_reg(sc, reg, start | pf); 1914 t4_read_reg(sc, reg); 1915 1916 return (addr - start); 1917 } 1918 1919 static int 1920 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1921 struct intrs_and_queues *iaq) 1922 { 1923 int rc, itype, navail, nrxq10g, nrxq1g, n; 1924 int nofldrxq10g = 0, nofldrxq1g = 0; 1925 int nnmrxq10g = 0, nnmrxq1g = 0; 1926 1927 bzero(iaq, sizeof(*iaq)); 1928 1929 iaq->ntxq10g = t4_ntxq10g; 1930 iaq->ntxq1g = t4_ntxq1g; 1931 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1932 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1933 iaq->rsrv_noflowq = t4_rsrv_noflowq; 1934 #ifdef TCP_OFFLOAD 1935 if (is_offload(sc)) { 1936 iaq->nofldtxq10g = t4_nofldtxq10g; 1937 iaq->nofldtxq1g = t4_nofldtxq1g; 1938 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1939 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1940 } 1941 #endif 1942 #ifdef DEV_NETMAP 1943 iaq->nnmtxq10g = t4_nnmtxq10g; 1944 iaq->nnmtxq1g = t4_nnmtxq1g; 1945 iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g; 1946 iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g; 1947 #endif 1948 1949 for (itype = INTR_MSIX; itype; itype >>= 1) { 1950 1951 if ((itype & t4_intr_types) == 0) 1952 continue; /* not allowed */ 1953 1954 if (itype == INTR_MSIX) 1955 navail = pci_msix_count(sc->dev); 1956 else if (itype == INTR_MSI) 1957 navail = pci_msi_count(sc->dev); 1958 else 1959 navail = 1; 1960 restart: 1961 if (navail == 0) 1962 continue; 1963 1964 iaq->intr_type = itype; 1965 iaq->intr_flags_10g = 0; 1966 iaq->intr_flags_1g = 0; 1967 1968 /* 1969 * Best option: an interrupt vector for errors, one for the 1970 * firmware event queue, and one for every rxq (NIC, TOE, and 1971 * netmap). 1972 */ 1973 iaq->nirq = T4_EXTRA_INTR; 1974 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g); 1975 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g); 1976 if (iaq->nirq <= navail && 1977 (itype != INTR_MSI || powerof2(iaq->nirq))) { 1978 iaq->intr_flags_10g = INTR_ALL; 1979 iaq->intr_flags_1g = INTR_ALL; 1980 goto allocate; 1981 } 1982 1983 /* 1984 * Second best option: a vector for errors, one for the firmware 1985 * event queue, and vectors for either all the NIC rx queues or 1986 * all the TOE rx queues. The queues that don't get vectors 1987 * will forward their interrupts to those that do. 1988 * 1989 * Note: netmap rx queues cannot be created early and so they 1990 * can't be setup to receive forwarded interrupts for others. 1991 */ 1992 iaq->nirq = T4_EXTRA_INTR; 1993 if (nrxq10g >= nofldrxq10g) { 1994 iaq->intr_flags_10g = INTR_RXQ; 1995 iaq->nirq += n10g * nrxq10g; 1996 #ifdef DEV_NETMAP 1997 iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g); 1998 #endif 1999 } else { 2000 iaq->intr_flags_10g = INTR_OFLD_RXQ; 2001 iaq->nirq += n10g * nofldrxq10g; 2002 #ifdef DEV_NETMAP 2003 iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g); 2004 #endif 2005 } 2006 if (nrxq1g >= nofldrxq1g) { 2007 iaq->intr_flags_1g = INTR_RXQ; 2008 iaq->nirq += n1g * nrxq1g; 2009 #ifdef DEV_NETMAP 2010 iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g); 2011 #endif 2012 } else { 2013 iaq->intr_flags_1g = INTR_OFLD_RXQ; 2014 iaq->nirq += n1g * nofldrxq1g; 2015 #ifdef DEV_NETMAP 2016 iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g); 2017 #endif 2018 } 2019 if (iaq->nirq <= navail && 2020 (itype != INTR_MSI || powerof2(iaq->nirq))) 2021 goto allocate; 2022 2023 /* 2024 * Next best option: an interrupt vector for errors, one for the 2025 * firmware event queue, and at least one per port. At this 2026 * point we know we'll have to downsize nrxq and/or nofldrxq 2027 * and/or nnmrxq to fit what's available to us. 2028 */ 2029 iaq->nirq = T4_EXTRA_INTR; 2030 iaq->nirq += n10g + n1g; 2031 if (iaq->nirq <= navail) { 2032 int leftover = navail - iaq->nirq; 2033 2034 if (n10g > 0) { 2035 int target = max(nrxq10g, nofldrxq10g); 2036 2037 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ? 2038 INTR_RXQ : INTR_OFLD_RXQ; 2039 2040 n = 1; 2041 while (n < target && leftover >= n10g) { 2042 leftover -= n10g; 2043 iaq->nirq += n10g; 2044 n++; 2045 } 2046 iaq->nrxq10g = min(n, nrxq10g); 2047 #ifdef TCP_OFFLOAD 2048 iaq->nofldrxq10g = min(n, nofldrxq10g); 2049 #endif 2050 #ifdef DEV_NETMAP 2051 iaq->nnmrxq10g = min(n, nnmrxq10g); 2052 #endif 2053 } 2054 2055 if (n1g > 0) { 2056 int target = max(nrxq1g, nofldrxq1g); 2057 2058 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ? 2059 INTR_RXQ : INTR_OFLD_RXQ; 2060 2061 n = 1; 2062 while (n < target && leftover >= n1g) { 2063 leftover -= n1g; 2064 iaq->nirq += n1g; 2065 n++; 2066 } 2067 iaq->nrxq1g = min(n, nrxq1g); 2068 #ifdef TCP_OFFLOAD 2069 iaq->nofldrxq1g = min(n, nofldrxq1g); 2070 #endif 2071 #ifdef DEV_NETMAP 2072 iaq->nnmrxq1g = min(n, nnmrxq1g); 2073 #endif 2074 } 2075 2076 if (itype != INTR_MSI || powerof2(iaq->nirq)) 2077 goto allocate; 2078 } 2079 2080 /* 2081 * Least desirable option: one interrupt vector for everything. 2082 */ 2083 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 2084 iaq->intr_flags_10g = iaq->intr_flags_1g = 0; 2085 #ifdef TCP_OFFLOAD 2086 if (is_offload(sc)) 2087 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 2088 #endif 2089 #ifdef DEV_NETMAP 2090 iaq->nnmrxq10g = iaq->nnmrxq1g = 1; 2091 #endif 2092 2093 allocate: 2094 navail = iaq->nirq; 2095 rc = 0; 2096 if (itype == INTR_MSIX) 2097 rc = pci_alloc_msix(sc->dev, &navail); 2098 else if (itype == INTR_MSI) 2099 rc = pci_alloc_msi(sc->dev, &navail); 2100 2101 if (rc == 0) { 2102 if (navail == iaq->nirq) 2103 return (0); 2104 2105 /* 2106 * Didn't get the number requested. Use whatever number 2107 * the kernel is willing to allocate (it's in navail). 2108 */ 2109 device_printf(sc->dev, "fewer vectors than requested, " 2110 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 2111 itype, iaq->nirq, navail); 2112 pci_release_msi(sc->dev); 2113 goto restart; 2114 } 2115 2116 device_printf(sc->dev, 2117 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 2118 itype, rc, iaq->nirq, navail); 2119 } 2120 2121 device_printf(sc->dev, 2122 "failed to find a usable interrupt type. " 2123 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 2124 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 2125 2126 return (ENXIO); 2127 } 2128 2129 #define FW_VERSION(chip) ( \ 2130 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ 2131 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ 2132 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ 2133 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) 2134 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) 2135 2136 struct fw_info { 2137 uint8_t chip; 2138 char *kld_name; 2139 char *fw_mod_name; 2140 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 2141 } fw_info[] = { 2142 { 2143 .chip = CHELSIO_T4, 2144 .kld_name = "t4fw_cfg", 2145 .fw_mod_name = "t4fw", 2146 .fw_hdr = { 2147 .chip = FW_HDR_CHIP_T4, 2148 .fw_ver = htobe32_const(FW_VERSION(T4)), 2149 .intfver_nic = FW_INTFVER(T4, NIC), 2150 .intfver_vnic = FW_INTFVER(T4, VNIC), 2151 .intfver_ofld = FW_INTFVER(T4, OFLD), 2152 .intfver_ri = FW_INTFVER(T4, RI), 2153 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 2154 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 2155 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 2156 .intfver_fcoe = FW_INTFVER(T4, FCOE), 2157 }, 2158 }, { 2159 .chip = CHELSIO_T5, 2160 .kld_name = "t5fw_cfg", 2161 .fw_mod_name = "t5fw", 2162 .fw_hdr = { 2163 .chip = FW_HDR_CHIP_T5, 2164 .fw_ver = htobe32_const(FW_VERSION(T5)), 2165 .intfver_nic = FW_INTFVER(T5, NIC), 2166 .intfver_vnic = FW_INTFVER(T5, VNIC), 2167 .intfver_ofld = FW_INTFVER(T5, OFLD), 2168 .intfver_ri = FW_INTFVER(T5, RI), 2169 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 2170 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 2171 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 2172 .intfver_fcoe = FW_INTFVER(T5, FCOE), 2173 }, 2174 } 2175 }; 2176 2177 static struct fw_info * 2178 find_fw_info(int chip) 2179 { 2180 int i; 2181 2182 for (i = 0; i < nitems(fw_info); i++) { 2183 if (fw_info[i].chip == chip) 2184 return (&fw_info[i]); 2185 } 2186 return (NULL); 2187 } 2188 2189 /* 2190 * Is the given firmware API compatible with the one the driver was compiled 2191 * with? 2192 */ 2193 static int 2194 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 2195 { 2196 2197 /* short circuit if it's the exact same firmware version */ 2198 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 2199 return (1); 2200 2201 /* 2202 * XXX: Is this too conservative? Perhaps I should limit this to the 2203 * features that are supported in the driver. 2204 */ 2205 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 2206 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 2207 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 2208 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 2209 return (1); 2210 #undef SAME_INTF 2211 2212 return (0); 2213 } 2214 2215 /* 2216 * The firmware in the KLD is usable, but should it be installed? This routine 2217 * explains itself in detail if it indicates the KLD firmware should be 2218 * installed. 2219 */ 2220 static int 2221 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 2222 { 2223 const char *reason; 2224 2225 if (!card_fw_usable) { 2226 reason = "incompatible or unusable"; 2227 goto install; 2228 } 2229 2230 if (k > c) { 2231 reason = "older than the version bundled with this driver"; 2232 goto install; 2233 } 2234 2235 if (t4_fw_install == 2 && k != c) { 2236 reason = "different than the version bundled with this driver"; 2237 goto install; 2238 } 2239 2240 return (0); 2241 2242 install: 2243 if (t4_fw_install == 0) { 2244 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2245 "but the driver is prohibited from installing a different " 2246 "firmware on the card.\n", 2247 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2248 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 2249 2250 return (0); 2251 } 2252 2253 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2254 "installing firmware %u.%u.%u.%u on card.\n", 2255 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2256 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 2257 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2258 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2259 2260 return (1); 2261 } 2262 /* 2263 * Establish contact with the firmware and determine if we are the master driver 2264 * or not, and whether we are responsible for chip initialization. 2265 */ 2266 static int 2267 prep_firmware(struct adapter *sc) 2268 { 2269 const struct firmware *fw = NULL, *default_cfg; 2270 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 2271 enum dev_state state; 2272 struct fw_info *fw_info; 2273 struct fw_hdr *card_fw; /* fw on the card */ 2274 const struct fw_hdr *kld_fw; /* fw in the KLD */ 2275 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 2276 against */ 2277 2278 /* Contact firmware. */ 2279 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 2280 if (rc < 0 || state == DEV_STATE_ERR) { 2281 rc = -rc; 2282 device_printf(sc->dev, 2283 "failed to connect to the firmware: %d, %d.\n", rc, state); 2284 return (rc); 2285 } 2286 pf = rc; 2287 if (pf == sc->mbox) 2288 sc->flags |= MASTER_PF; 2289 else if (state == DEV_STATE_UNINIT) { 2290 /* 2291 * We didn't get to be the master so we definitely won't be 2292 * configuring the chip. It's a bug if someone else hasn't 2293 * configured it already. 2294 */ 2295 device_printf(sc->dev, "couldn't be master(%d), " 2296 "device not already initialized either(%d).\n", rc, state); 2297 return (EDOOFUS); 2298 } 2299 2300 /* This is the firmware whose headers the driver was compiled against */ 2301 fw_info = find_fw_info(chip_id(sc)); 2302 if (fw_info == NULL) { 2303 device_printf(sc->dev, 2304 "unable to look up firmware information for chip %d.\n", 2305 chip_id(sc)); 2306 return (EINVAL); 2307 } 2308 drv_fw = &fw_info->fw_hdr; 2309 2310 /* 2311 * The firmware KLD contains many modules. The KLD name is also the 2312 * name of the module that contains the default config file. 2313 */ 2314 default_cfg = firmware_get(fw_info->kld_name); 2315 2316 /* Read the header of the firmware on the card */ 2317 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 2318 rc = -t4_read_flash(sc, FLASH_FW_START, 2319 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 2320 if (rc == 0) 2321 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 2322 else { 2323 device_printf(sc->dev, 2324 "Unable to read card's firmware header: %d\n", rc); 2325 card_fw_usable = 0; 2326 } 2327 2328 /* This is the firmware in the KLD */ 2329 fw = firmware_get(fw_info->fw_mod_name); 2330 if (fw != NULL) { 2331 kld_fw = (const void *)fw->data; 2332 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 2333 } else { 2334 kld_fw = NULL; 2335 kld_fw_usable = 0; 2336 } 2337 2338 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2339 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) { 2340 /* 2341 * Common case: the firmware on the card is an exact match and 2342 * the KLD is an exact match too, or the KLD is 2343 * absent/incompatible. Note that t4_fw_install = 2 is ignored 2344 * here -- use cxgbetool loadfw if you want to reinstall the 2345 * same firmware as the one on the card. 2346 */ 2347 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2348 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2349 be32toh(card_fw->fw_ver))) { 2350 2351 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 2352 if (rc != 0) { 2353 device_printf(sc->dev, 2354 "failed to install firmware: %d\n", rc); 2355 goto done; 2356 } 2357 2358 /* Installed successfully, update the cached header too. */ 2359 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2360 card_fw_usable = 1; 2361 need_fw_reset = 0; /* already reset as part of load_fw */ 2362 } 2363 2364 if (!card_fw_usable) { 2365 uint32_t d, c, k; 2366 2367 d = ntohl(drv_fw->fw_ver); 2368 c = ntohl(card_fw->fw_ver); 2369 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2370 2371 device_printf(sc->dev, "Cannot find a usable firmware: " 2372 "fw_install %d, chip state %d, " 2373 "driver compiled with %d.%d.%d.%d, " 2374 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2375 t4_fw_install, state, 2376 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2377 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2378 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2379 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2380 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2381 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2382 rc = EINVAL; 2383 goto done; 2384 } 2385 2386 /* We're using whatever's on the card and it's known to be good. */ 2387 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2388 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2389 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2390 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2391 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2392 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2393 t4_get_tp_version(sc, &sc->params.tp_vers); 2394 2395 /* Reset device */ 2396 if (need_fw_reset && 2397 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2398 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2399 if (rc != ETIMEDOUT && rc != EIO) 2400 t4_fw_bye(sc, sc->mbox); 2401 goto done; 2402 } 2403 sc->flags |= FW_OK; 2404 2405 rc = get_params__pre_init(sc); 2406 if (rc != 0) 2407 goto done; /* error message displayed already */ 2408 2409 /* Partition adapter resources as specified in the config file. */ 2410 if (state == DEV_STATE_UNINIT) { 2411 2412 KASSERT(sc->flags & MASTER_PF, 2413 ("%s: trying to change chip settings when not master.", 2414 __func__)); 2415 2416 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2417 if (rc != 0) 2418 goto done; /* error message displayed already */ 2419 2420 t4_tweak_chip_settings(sc); 2421 2422 /* get basic stuff going */ 2423 rc = -t4_fw_initialize(sc, sc->mbox); 2424 if (rc != 0) { 2425 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2426 goto done; 2427 } 2428 } else { 2429 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2430 sc->cfcsum = 0; 2431 } 2432 2433 done: 2434 free(card_fw, M_CXGBE); 2435 if (fw != NULL) 2436 firmware_put(fw, FIRMWARE_UNLOAD); 2437 if (default_cfg != NULL) 2438 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2439 2440 return (rc); 2441 } 2442 2443 #define FW_PARAM_DEV(param) \ 2444 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2445 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2446 #define FW_PARAM_PFVF(param) \ 2447 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2448 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2449 2450 /* 2451 * Partition chip resources for use between various PFs, VFs, etc. 2452 */ 2453 static int 2454 partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2455 const char *name_prefix) 2456 { 2457 const struct firmware *cfg = NULL; 2458 int rc = 0; 2459 struct fw_caps_config_cmd caps; 2460 uint32_t mtype, moff, finicsum, cfcsum; 2461 2462 /* 2463 * Figure out what configuration file to use. Pick the default config 2464 * file for the card if the user hasn't specified one explicitly. 2465 */ 2466 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2467 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2468 /* Card specific overrides go here. */ 2469 if (pci_get_device(sc->dev) == 0x440a) 2470 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2471 if (is_fpga(sc)) 2472 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2473 } 2474 2475 /* 2476 * We need to load another module if the profile is anything except 2477 * "default" or "flash". 2478 */ 2479 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2480 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2481 char s[32]; 2482 2483 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2484 cfg = firmware_get(s); 2485 if (cfg == NULL) { 2486 if (default_cfg != NULL) { 2487 device_printf(sc->dev, 2488 "unable to load module \"%s\" for " 2489 "configuration profile \"%s\", will use " 2490 "the default config file instead.\n", 2491 s, sc->cfg_file); 2492 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2493 "%s", DEFAULT_CF); 2494 } else { 2495 device_printf(sc->dev, 2496 "unable to load module \"%s\" for " 2497 "configuration profile \"%s\", will use " 2498 "the config file on the card's flash " 2499 "instead.\n", s, sc->cfg_file); 2500 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2501 "%s", FLASH_CF); 2502 } 2503 } 2504 } 2505 2506 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2507 default_cfg == NULL) { 2508 device_printf(sc->dev, 2509 "default config file not available, will use the config " 2510 "file on the card's flash instead.\n"); 2511 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2512 } 2513 2514 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2515 u_int cflen, i, n; 2516 const uint32_t *cfdata; 2517 uint32_t param, val, addr, off, mw_base, mw_aperture; 2518 2519 KASSERT(cfg != NULL || default_cfg != NULL, 2520 ("%s: no config to upload", __func__)); 2521 2522 /* 2523 * Ask the firmware where it wants us to upload the config file. 2524 */ 2525 param = FW_PARAM_DEV(CF); 2526 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2527 if (rc != 0) { 2528 /* No support for config file? Shouldn't happen. */ 2529 device_printf(sc->dev, 2530 "failed to query config file location: %d.\n", rc); 2531 goto done; 2532 } 2533 mtype = G_FW_PARAMS_PARAM_Y(val); 2534 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2535 2536 /* 2537 * XXX: sheer laziness. We deliberately added 4 bytes of 2538 * useless stuffing/comments at the end of the config file so 2539 * it's ok to simply throw away the last remaining bytes when 2540 * the config file is not an exact multiple of 4. This also 2541 * helps with the validate_mt_off_len check. 2542 */ 2543 if (cfg != NULL) { 2544 cflen = cfg->datasize & ~3; 2545 cfdata = cfg->data; 2546 } else { 2547 cflen = default_cfg->datasize & ~3; 2548 cfdata = default_cfg->data; 2549 } 2550 2551 if (cflen > FLASH_CFG_MAX_SIZE) { 2552 device_printf(sc->dev, 2553 "config file too long (%d, max allowed is %d). " 2554 "Will try to use the config on the card, if any.\n", 2555 cflen, FLASH_CFG_MAX_SIZE); 2556 goto use_config_on_flash; 2557 } 2558 2559 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2560 if (rc != 0) { 2561 device_printf(sc->dev, 2562 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2563 "Will try to use the config on the card, if any.\n", 2564 __func__, mtype, moff, cflen, rc); 2565 goto use_config_on_flash; 2566 } 2567 2568 memwin_info(sc, 2, &mw_base, &mw_aperture); 2569 while (cflen) { 2570 off = position_memwin(sc, 2, addr); 2571 n = min(cflen, mw_aperture - off); 2572 for (i = 0; i < n; i += 4) 2573 t4_write_reg(sc, mw_base + off + i, *cfdata++); 2574 cflen -= n; 2575 addr += n; 2576 } 2577 } else { 2578 use_config_on_flash: 2579 mtype = FW_MEMTYPE_FLASH; 2580 moff = t4_flash_cfg_addr(sc); 2581 } 2582 2583 bzero(&caps, sizeof(caps)); 2584 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2585 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2586 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2587 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2588 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2589 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2590 if (rc != 0) { 2591 device_printf(sc->dev, 2592 "failed to pre-process config file: %d " 2593 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2594 goto done; 2595 } 2596 2597 finicsum = be32toh(caps.finicsum); 2598 cfcsum = be32toh(caps.cfcsum); 2599 if (finicsum != cfcsum) { 2600 device_printf(sc->dev, 2601 "WARNING: config file checksum mismatch: %08x %08x\n", 2602 finicsum, cfcsum); 2603 } 2604 sc->cfcsum = cfcsum; 2605 2606 #define LIMIT_CAPS(x) do { \ 2607 caps.x &= htobe16(t4_##x##_allowed); \ 2608 } while (0) 2609 2610 /* 2611 * Let the firmware know what features will (not) be used so it can tune 2612 * things accordingly. 2613 */ 2614 LIMIT_CAPS(linkcaps); 2615 LIMIT_CAPS(niccaps); 2616 LIMIT_CAPS(toecaps); 2617 LIMIT_CAPS(rdmacaps); 2618 LIMIT_CAPS(iscsicaps); 2619 LIMIT_CAPS(fcoecaps); 2620 #undef LIMIT_CAPS 2621 2622 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2623 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2624 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2625 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2626 if (rc != 0) { 2627 device_printf(sc->dev, 2628 "failed to process config file: %d.\n", rc); 2629 } 2630 done: 2631 if (cfg != NULL) 2632 firmware_put(cfg, FIRMWARE_UNLOAD); 2633 return (rc); 2634 } 2635 2636 /* 2637 * Retrieve parameters that are needed (or nice to have) very early. 2638 */ 2639 static int 2640 get_params__pre_init(struct adapter *sc) 2641 { 2642 int rc; 2643 uint32_t param[2], val[2]; 2644 struct fw_devlog_cmd cmd; 2645 struct devlog_params *dlog = &sc->params.devlog; 2646 2647 param[0] = FW_PARAM_DEV(PORTVEC); 2648 param[1] = FW_PARAM_DEV(CCLK); 2649 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2650 if (rc != 0) { 2651 device_printf(sc->dev, 2652 "failed to query parameters (pre_init): %d.\n", rc); 2653 return (rc); 2654 } 2655 2656 sc->params.portvec = val[0]; 2657 sc->params.nports = bitcount32(val[0]); 2658 sc->params.vpd.cclk = val[1]; 2659 2660 /* Read device log parameters. */ 2661 bzero(&cmd, sizeof(cmd)); 2662 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2663 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2664 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2665 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2666 if (rc != 0) { 2667 device_printf(sc->dev, 2668 "failed to get devlog parameters: %d.\n", rc); 2669 bzero(dlog, sizeof (*dlog)); 2670 rc = 0; /* devlog isn't critical for device operation */ 2671 } else { 2672 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2673 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2674 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2675 dlog->size = be32toh(cmd.memsize_devlog); 2676 } 2677 2678 return (rc); 2679 } 2680 2681 /* 2682 * Retrieve various parameters that are of interest to the driver. The device 2683 * has been initialized by the firmware at this point. 2684 */ 2685 static int 2686 get_params__post_init(struct adapter *sc) 2687 { 2688 int rc; 2689 uint32_t param[7], val[7]; 2690 struct fw_caps_config_cmd caps; 2691 2692 param[0] = FW_PARAM_PFVF(IQFLINT_START); 2693 param[1] = FW_PARAM_PFVF(EQ_START); 2694 param[2] = FW_PARAM_PFVF(FILTER_START); 2695 param[3] = FW_PARAM_PFVF(FILTER_END); 2696 param[4] = FW_PARAM_PFVF(L2T_START); 2697 param[5] = FW_PARAM_PFVF(L2T_END); 2698 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2699 if (rc != 0) { 2700 device_printf(sc->dev, 2701 "failed to query parameters (post_init): %d.\n", rc); 2702 return (rc); 2703 } 2704 2705 sc->sge.iq_start = val[0]; 2706 sc->sge.eq_start = val[1]; 2707 sc->tids.ftid_base = val[2]; 2708 sc->tids.nftids = val[3] - val[2] + 1; 2709 sc->params.ftid_min = val[2]; 2710 sc->params.ftid_max = val[3]; 2711 sc->vres.l2t.start = val[4]; 2712 sc->vres.l2t.size = val[5] - val[4] + 1; 2713 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2714 ("%s: L2 table size (%u) larger than expected (%u)", 2715 __func__, sc->vres.l2t.size, L2T_SIZE)); 2716 2717 /* get capabilites */ 2718 bzero(&caps, sizeof(caps)); 2719 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2720 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2721 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2722 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2723 if (rc != 0) { 2724 device_printf(sc->dev, 2725 "failed to get card capabilities: %d.\n", rc); 2726 return (rc); 2727 } 2728 2729 #define READ_CAPS(x) do { \ 2730 sc->x = htobe16(caps.x); \ 2731 } while (0) 2732 READ_CAPS(linkcaps); 2733 READ_CAPS(niccaps); 2734 READ_CAPS(toecaps); 2735 READ_CAPS(rdmacaps); 2736 READ_CAPS(iscsicaps); 2737 READ_CAPS(fcoecaps); 2738 2739 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { 2740 param[0] = FW_PARAM_PFVF(ETHOFLD_START); 2741 param[1] = FW_PARAM_PFVF(ETHOFLD_END); 2742 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2743 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); 2744 if (rc != 0) { 2745 device_printf(sc->dev, 2746 "failed to query NIC parameters: %d.\n", rc); 2747 return (rc); 2748 } 2749 sc->tids.etid_base = val[0]; 2750 sc->params.etid_min = val[0]; 2751 sc->tids.netids = val[1] - val[0] + 1; 2752 sc->params.netids = sc->tids.netids; 2753 sc->params.eo_wr_cred = val[2]; 2754 sc->params.ethoffload = 1; 2755 } 2756 2757 if (sc->toecaps) { 2758 /* query offload-related parameters */ 2759 param[0] = FW_PARAM_DEV(NTID); 2760 param[1] = FW_PARAM_PFVF(SERVER_START); 2761 param[2] = FW_PARAM_PFVF(SERVER_END); 2762 param[3] = FW_PARAM_PFVF(TDDP_START); 2763 param[4] = FW_PARAM_PFVF(TDDP_END); 2764 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2765 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2766 if (rc != 0) { 2767 device_printf(sc->dev, 2768 "failed to query TOE parameters: %d.\n", rc); 2769 return (rc); 2770 } 2771 sc->tids.ntids = val[0]; 2772 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 2773 sc->tids.stid_base = val[1]; 2774 sc->tids.nstids = val[2] - val[1] + 1; 2775 sc->vres.ddp.start = val[3]; 2776 sc->vres.ddp.size = val[4] - val[3] + 1; 2777 sc->params.ofldq_wr_cred = val[5]; 2778 sc->params.offload = 1; 2779 } 2780 if (sc->rdmacaps) { 2781 param[0] = FW_PARAM_PFVF(STAG_START); 2782 param[1] = FW_PARAM_PFVF(STAG_END); 2783 param[2] = FW_PARAM_PFVF(RQ_START); 2784 param[3] = FW_PARAM_PFVF(RQ_END); 2785 param[4] = FW_PARAM_PFVF(PBL_START); 2786 param[5] = FW_PARAM_PFVF(PBL_END); 2787 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2788 if (rc != 0) { 2789 device_printf(sc->dev, 2790 "failed to query RDMA parameters(1): %d.\n", rc); 2791 return (rc); 2792 } 2793 sc->vres.stag.start = val[0]; 2794 sc->vres.stag.size = val[1] - val[0] + 1; 2795 sc->vres.rq.start = val[2]; 2796 sc->vres.rq.size = val[3] - val[2] + 1; 2797 sc->vres.pbl.start = val[4]; 2798 sc->vres.pbl.size = val[5] - val[4] + 1; 2799 2800 param[0] = FW_PARAM_PFVF(SQRQ_START); 2801 param[1] = FW_PARAM_PFVF(SQRQ_END); 2802 param[2] = FW_PARAM_PFVF(CQ_START); 2803 param[3] = FW_PARAM_PFVF(CQ_END); 2804 param[4] = FW_PARAM_PFVF(OCQ_START); 2805 param[5] = FW_PARAM_PFVF(OCQ_END); 2806 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2807 if (rc != 0) { 2808 device_printf(sc->dev, 2809 "failed to query RDMA parameters(2): %d.\n", rc); 2810 return (rc); 2811 } 2812 sc->vres.qp.start = val[0]; 2813 sc->vres.qp.size = val[1] - val[0] + 1; 2814 sc->vres.cq.start = val[2]; 2815 sc->vres.cq.size = val[3] - val[2] + 1; 2816 sc->vres.ocq.start = val[4]; 2817 sc->vres.ocq.size = val[5] - val[4] + 1; 2818 } 2819 if (sc->iscsicaps) { 2820 param[0] = FW_PARAM_PFVF(ISCSI_START); 2821 param[1] = FW_PARAM_PFVF(ISCSI_END); 2822 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2823 if (rc != 0) { 2824 device_printf(sc->dev, 2825 "failed to query iSCSI parameters: %d.\n", rc); 2826 return (rc); 2827 } 2828 sc->vres.iscsi.start = val[0]; 2829 sc->vres.iscsi.size = val[1] - val[0] + 1; 2830 } 2831 2832 /* 2833 * We've got the params we wanted to query via the firmware. Now grab 2834 * some others directly from the chip. 2835 */ 2836 rc = t4_read_chip_settings(sc); 2837 2838 return (rc); 2839 } 2840 2841 static int 2842 set_params__post_init(struct adapter *sc) 2843 { 2844 uint32_t param, val; 2845 2846 /* ask for encapsulated CPLs */ 2847 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 2848 val = 1; 2849 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2850 2851 return (0); 2852 } 2853 2854 #undef FW_PARAM_PFVF 2855 #undef FW_PARAM_DEV 2856 2857 static void 2858 t4_set_desc(struct adapter *sc) 2859 { 2860 char buf[128]; 2861 struct adapter_params *p = &sc->params; 2862 2863 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, " 2864 "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "", 2865 chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec); 2866 2867 device_set_desc_copy(sc->dev, buf); 2868 } 2869 2870 static void 2871 build_medialist(struct port_info *pi, struct ifmedia *media) 2872 { 2873 int m; 2874 2875 PORT_LOCK(pi); 2876 2877 ifmedia_removeall(media); 2878 2879 m = IFM_ETHER | IFM_FDX; 2880 2881 switch(pi->port_type) { 2882 case FW_PORT_TYPE_BT_XFI: 2883 case FW_PORT_TYPE_BT_XAUI: 2884 ifmedia_add(media, m | IFM_10G_T, 0, NULL); 2885 /* fall through */ 2886 2887 case FW_PORT_TYPE_BT_SGMII: 2888 ifmedia_add(media, m | IFM_1000_T, 0, NULL); 2889 ifmedia_add(media, m | IFM_100_TX, 0, NULL); 2890 ifmedia_add(media, IFM_ETHER | IFM_AUTO, 0, NULL); 2891 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2892 break; 2893 2894 case FW_PORT_TYPE_CX4: 2895 ifmedia_add(media, m | IFM_10G_CX4, 0, NULL); 2896 ifmedia_set(media, m | IFM_10G_CX4); 2897 break; 2898 2899 case FW_PORT_TYPE_QSFP_10G: 2900 case FW_PORT_TYPE_SFP: 2901 case FW_PORT_TYPE_FIBER_XFI: 2902 case FW_PORT_TYPE_FIBER_XAUI: 2903 switch (pi->mod_type) { 2904 2905 case FW_PORT_MOD_TYPE_LR: 2906 ifmedia_add(media, m | IFM_10G_LR, 0, NULL); 2907 ifmedia_set(media, m | IFM_10G_LR); 2908 break; 2909 2910 case FW_PORT_MOD_TYPE_SR: 2911 ifmedia_add(media, m | IFM_10G_SR, 0, NULL); 2912 ifmedia_set(media, m | IFM_10G_SR); 2913 break; 2914 2915 case FW_PORT_MOD_TYPE_LRM: 2916 ifmedia_add(media, m | IFM_10G_LRM, 0, NULL); 2917 ifmedia_set(media, m | IFM_10G_LRM); 2918 break; 2919 2920 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2921 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2922 ifmedia_add(media, m | IFM_10G_TWINAX, 0, NULL); 2923 ifmedia_set(media, m | IFM_10G_TWINAX); 2924 break; 2925 2926 case FW_PORT_MOD_TYPE_NONE: 2927 m &= ~IFM_FDX; 2928 ifmedia_add(media, m | IFM_NONE, 0, NULL); 2929 ifmedia_set(media, m | IFM_NONE); 2930 break; 2931 2932 case FW_PORT_MOD_TYPE_NA: 2933 case FW_PORT_MOD_TYPE_ER: 2934 default: 2935 device_printf(pi->dev, 2936 "unknown port_type (%d), mod_type (%d)\n", 2937 pi->port_type, pi->mod_type); 2938 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 2939 ifmedia_set(media, m | IFM_UNKNOWN); 2940 break; 2941 } 2942 break; 2943 2944 case FW_PORT_TYPE_QSFP: 2945 switch (pi->mod_type) { 2946 2947 case FW_PORT_MOD_TYPE_LR: 2948 ifmedia_add(media, m | IFM_40G_LR4, 0, NULL); 2949 ifmedia_set(media, m | IFM_40G_LR4); 2950 break; 2951 2952 case FW_PORT_MOD_TYPE_SR: 2953 ifmedia_add(media, m | IFM_40G_SR4, 0, NULL); 2954 ifmedia_set(media, m | IFM_40G_SR4); 2955 break; 2956 2957 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2958 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2959 ifmedia_add(media, m | IFM_40G_CR4, 0, NULL); 2960 ifmedia_set(media, m | IFM_40G_CR4); 2961 break; 2962 2963 case FW_PORT_MOD_TYPE_NONE: 2964 m &= ~IFM_FDX; 2965 ifmedia_add(media, m | IFM_NONE, 0, NULL); 2966 ifmedia_set(media, m | IFM_NONE); 2967 break; 2968 2969 default: 2970 device_printf(pi->dev, 2971 "unknown port_type (%d), mod_type (%d)\n", 2972 pi->port_type, pi->mod_type); 2973 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 2974 ifmedia_set(media, m | IFM_UNKNOWN); 2975 break; 2976 } 2977 break; 2978 2979 default: 2980 device_printf(pi->dev, 2981 "unknown port_type (%d), mod_type (%d)\n", pi->port_type, 2982 pi->mod_type); 2983 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 2984 ifmedia_set(media, m | IFM_UNKNOWN); 2985 break; 2986 } 2987 2988 PORT_UNLOCK(pi); 2989 } 2990 2991 #define FW_MAC_EXACT_CHUNK 7 2992 2993 /* 2994 * Program the port's XGMAC based on parameters in ifnet. The caller also 2995 * indicates which parameters should be programmed (the rest are left alone). 2996 */ 2997 int 2998 update_mac_settings(struct ifnet *ifp, int flags) 2999 { 3000 int rc = 0; 3001 struct port_info *pi = ifp->if_softc; 3002 struct adapter *sc = pi->adapter; 3003 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 3004 uint16_t viid = 0xffff; 3005 int16_t *xact_addr_filt = NULL; 3006 3007 ASSERT_SYNCHRONIZED_OP(sc); 3008 KASSERT(flags, ("%s: not told what to update.", __func__)); 3009 3010 if (ifp == pi->ifp) { 3011 viid = pi->viid; 3012 xact_addr_filt = &pi->xact_addr_filt; 3013 } 3014 #ifdef DEV_NETMAP 3015 else if (ifp == pi->nm_ifp) { 3016 viid = pi->nm_viid; 3017 xact_addr_filt = &pi->nm_xact_addr_filt; 3018 } 3019 #endif 3020 if (flags & XGMAC_MTU) 3021 mtu = ifp->if_mtu; 3022 3023 if (flags & XGMAC_PROMISC) 3024 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 3025 3026 if (flags & XGMAC_ALLMULTI) 3027 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 3028 3029 if (flags & XGMAC_VLANEX) 3030 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 3031 3032 if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) { 3033 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti, 3034 1, vlanex, false); 3035 if (rc) { 3036 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, 3037 rc); 3038 return (rc); 3039 } 3040 } 3041 3042 if (flags & XGMAC_UCADDR) { 3043 uint8_t ucaddr[ETHER_ADDR_LEN]; 3044 3045 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 3046 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr, 3047 true, true); 3048 if (rc < 0) { 3049 rc = -rc; 3050 if_printf(ifp, "change_mac failed: %d\n", rc); 3051 return (rc); 3052 } else { 3053 *xact_addr_filt = rc; 3054 rc = 0; 3055 } 3056 } 3057 3058 if (flags & XGMAC_MCADDRS) { 3059 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 3060 int del = 1; 3061 uint64_t hash = 0; 3062 struct ifmultiaddr *ifma; 3063 int i = 0, j; 3064 3065 if_maddr_rlock(ifp); 3066 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3067 if (ifma->ifma_addr->sa_family != AF_LINK) 3068 continue; 3069 mcaddr[i] = 3070 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 3071 MPASS(ETHER_IS_MULTICAST(mcaddr[i])); 3072 i++; 3073 3074 if (i == FW_MAC_EXACT_CHUNK) { 3075 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, 3076 i, mcaddr, NULL, &hash, 0); 3077 if (rc < 0) { 3078 rc = -rc; 3079 for (j = 0; j < i; j++) { 3080 if_printf(ifp, 3081 "failed to add mc address" 3082 " %02x:%02x:%02x:" 3083 "%02x:%02x:%02x rc=%d\n", 3084 mcaddr[j][0], mcaddr[j][1], 3085 mcaddr[j][2], mcaddr[j][3], 3086 mcaddr[j][4], mcaddr[j][5], 3087 rc); 3088 } 3089 goto mcfail; 3090 } 3091 del = 0; 3092 i = 0; 3093 } 3094 } 3095 if (i > 0) { 3096 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i, 3097 mcaddr, NULL, &hash, 0); 3098 if (rc < 0) { 3099 rc = -rc; 3100 for (j = 0; j < i; j++) { 3101 if_printf(ifp, 3102 "failed to add mc address" 3103 " %02x:%02x:%02x:" 3104 "%02x:%02x:%02x rc=%d\n", 3105 mcaddr[j][0], mcaddr[j][1], 3106 mcaddr[j][2], mcaddr[j][3], 3107 mcaddr[j][4], mcaddr[j][5], 3108 rc); 3109 } 3110 goto mcfail; 3111 } 3112 } 3113 3114 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0); 3115 if (rc != 0) 3116 if_printf(ifp, "failed to set mc address hash: %d", rc); 3117 mcfail: 3118 if_maddr_runlock(ifp); 3119 } 3120 3121 return (rc); 3122 } 3123 3124 /* 3125 * {begin|end}_synchronized_op must be called from the same thread. 3126 */ 3127 int 3128 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags, 3129 char *wmesg) 3130 { 3131 int rc, pri; 3132 3133 #ifdef WITNESS 3134 /* the caller thinks it's ok to sleep, but is it really? */ 3135 if (flags & SLEEP_OK) 3136 pause("t4slptst", 1); 3137 #endif 3138 3139 if (INTR_OK) 3140 pri = PCATCH; 3141 else 3142 pri = 0; 3143 3144 ADAPTER_LOCK(sc); 3145 for (;;) { 3146 3147 if (pi && IS_DOOMED(pi)) { 3148 rc = ENXIO; 3149 goto done; 3150 } 3151 3152 if (!IS_BUSY(sc)) { 3153 rc = 0; 3154 break; 3155 } 3156 3157 if (!(flags & SLEEP_OK)) { 3158 rc = EBUSY; 3159 goto done; 3160 } 3161 3162 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 3163 rc = EINTR; 3164 goto done; 3165 } 3166 } 3167 3168 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 3169 SET_BUSY(sc); 3170 #ifdef INVARIANTS 3171 sc->last_op = wmesg; 3172 sc->last_op_thr = curthread; 3173 sc->last_op_flags = flags; 3174 #endif 3175 3176 done: 3177 if (!(flags & HOLD_LOCK) || rc) 3178 ADAPTER_UNLOCK(sc); 3179 3180 return (rc); 3181 } 3182 3183 /* 3184 * {begin|end}_synchronized_op must be called from the same thread. 3185 */ 3186 void 3187 end_synchronized_op(struct adapter *sc, int flags) 3188 { 3189 3190 if (flags & LOCK_HELD) 3191 ADAPTER_LOCK_ASSERT_OWNED(sc); 3192 else 3193 ADAPTER_LOCK(sc); 3194 3195 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 3196 CLR_BUSY(sc); 3197 wakeup(&sc->flags); 3198 ADAPTER_UNLOCK(sc); 3199 } 3200 3201 static int 3202 cxgbe_init_synchronized(struct port_info *pi) 3203 { 3204 struct adapter *sc = pi->adapter; 3205 struct ifnet *ifp = pi->ifp; 3206 int rc = 0, i; 3207 struct sge_txq *txq; 3208 3209 ASSERT_SYNCHRONIZED_OP(sc); 3210 3211 if (isset(&sc->open_device_map, pi->port_id)) { 3212 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 3213 ("mismatch between open_device_map and if_drv_flags")); 3214 return (0); /* already running */ 3215 } 3216 3217 if (!(sc->flags & FULL_INIT_DONE) && 3218 ((rc = adapter_full_init(sc)) != 0)) 3219 return (rc); /* error message displayed already */ 3220 3221 if (!(pi->flags & PORT_INIT_DONE) && 3222 ((rc = port_full_init(pi)) != 0)) 3223 return (rc); /* error message displayed already */ 3224 3225 rc = update_mac_settings(ifp, XGMAC_ALL); 3226 if (rc) 3227 goto done; /* error message displayed already */ 3228 3229 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 3230 if (rc != 0) { 3231 if_printf(ifp, "enable_vi failed: %d\n", rc); 3232 goto done; 3233 } 3234 3235 /* 3236 * Can't fail from this point onwards. Review cxgbe_uninit_synchronized 3237 * if this changes. 3238 */ 3239 3240 for_each_txq(pi, i, txq) { 3241 TXQ_LOCK(txq); 3242 txq->eq.flags |= EQ_ENABLED; 3243 TXQ_UNLOCK(txq); 3244 } 3245 3246 /* 3247 * The first iq of the first port to come up is used for tracing. 3248 */ 3249 if (sc->traceq < 0) { 3250 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id; 3251 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : 3252 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | 3253 V_QUEUENUMBER(sc->traceq)); 3254 pi->flags |= HAS_TRACEQ; 3255 } 3256 3257 /* all ok */ 3258 setbit(&sc->open_device_map, pi->port_id); 3259 PORT_LOCK(pi); 3260 ifp->if_drv_flags |= IFF_DRV_RUNNING; 3261 PORT_UNLOCK(pi); 3262 3263 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 3264 done: 3265 if (rc != 0) 3266 cxgbe_uninit_synchronized(pi); 3267 3268 return (rc); 3269 } 3270 3271 /* 3272 * Idempotent. 3273 */ 3274 static int 3275 cxgbe_uninit_synchronized(struct port_info *pi) 3276 { 3277 struct adapter *sc = pi->adapter; 3278 struct ifnet *ifp = pi->ifp; 3279 int rc, i; 3280 struct sge_txq *txq; 3281 3282 ASSERT_SYNCHRONIZED_OP(sc); 3283 3284 if (!(pi->flags & PORT_INIT_DONE)) { 3285 KASSERT(!(ifp->if_drv_flags & IFF_DRV_RUNNING), 3286 ("uninited port is running")); 3287 return (0); 3288 } 3289 3290 /* 3291 * Disable the VI so that all its data in either direction is discarded 3292 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 3293 * tick) intact as the TP can deliver negative advice or data that it's 3294 * holding in its RAM (for an offloaded connection) even after the VI is 3295 * disabled. 3296 */ 3297 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 3298 if (rc) { 3299 if_printf(ifp, "disable_vi failed: %d\n", rc); 3300 return (rc); 3301 } 3302 3303 for_each_txq(pi, i, txq) { 3304 TXQ_LOCK(txq); 3305 txq->eq.flags &= ~EQ_ENABLED; 3306 TXQ_UNLOCK(txq); 3307 } 3308 3309 clrbit(&sc->open_device_map, pi->port_id); 3310 PORT_LOCK(pi); 3311 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3312 PORT_UNLOCK(pi); 3313 3314 pi->link_cfg.link_ok = 0; 3315 pi->link_cfg.speed = 0; 3316 pi->linkdnrc = -1; 3317 t4_os_link_changed(sc, pi->port_id, 0, -1); 3318 3319 return (0); 3320 } 3321 3322 /* 3323 * It is ok for this function to fail midway and return right away. t4_detach 3324 * will walk the entire sc->irq list and clean up whatever is valid. 3325 */ 3326 static int 3327 setup_intr_handlers(struct adapter *sc) 3328 { 3329 int rc, rid, p, q; 3330 char s[8]; 3331 struct irq *irq; 3332 struct port_info *pi; 3333 struct sge_rxq *rxq; 3334 #ifdef TCP_OFFLOAD 3335 struct sge_ofld_rxq *ofld_rxq; 3336 #endif 3337 #ifdef DEV_NETMAP 3338 struct sge_nm_rxq *nm_rxq; 3339 #endif 3340 3341 /* 3342 * Setup interrupts. 3343 */ 3344 irq = &sc->irq[0]; 3345 rid = sc->intr_type == INTR_INTX ? 0 : 1; 3346 if (sc->intr_count == 1) 3347 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all")); 3348 3349 /* Multiple interrupts. */ 3350 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 3351 ("%s: too few intr.", __func__)); 3352 3353 /* The first one is always error intr */ 3354 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 3355 if (rc != 0) 3356 return (rc); 3357 irq++; 3358 rid++; 3359 3360 /* The second one is always the firmware event queue */ 3361 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt"); 3362 if (rc != 0) 3363 return (rc); 3364 irq++; 3365 rid++; 3366 3367 for_each_port(sc, p) { 3368 pi = sc->port[p]; 3369 3370 if (pi->flags & INTR_RXQ) { 3371 for_each_rxq(pi, q, rxq) { 3372 snprintf(s, sizeof(s), "%d.%d", p, q); 3373 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq, 3374 s); 3375 if (rc != 0) 3376 return (rc); 3377 irq++; 3378 rid++; 3379 } 3380 } 3381 #ifdef TCP_OFFLOAD 3382 if (pi->flags & INTR_OFLD_RXQ) { 3383 for_each_ofld_rxq(pi, q, ofld_rxq) { 3384 snprintf(s, sizeof(s), "%d,%d", p, q); 3385 rc = t4_alloc_irq(sc, irq, rid, t4_intr, 3386 ofld_rxq, s); 3387 if (rc != 0) 3388 return (rc); 3389 irq++; 3390 rid++; 3391 } 3392 } 3393 #endif 3394 #ifdef DEV_NETMAP 3395 if (pi->flags & INTR_NM_RXQ) { 3396 for_each_nm_rxq(pi, q, nm_rxq) { 3397 snprintf(s, sizeof(s), "%d-%d", p, q); 3398 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr, 3399 nm_rxq, s); 3400 if (rc != 0) 3401 return (rc); 3402 irq++; 3403 rid++; 3404 } 3405 } 3406 #endif 3407 } 3408 MPASS(irq == &sc->irq[sc->intr_count]); 3409 3410 return (0); 3411 } 3412 3413 int 3414 adapter_full_init(struct adapter *sc) 3415 { 3416 int rc, i; 3417 3418 ASSERT_SYNCHRONIZED_OP(sc); 3419 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3420 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 3421 ("%s: FULL_INIT_DONE already", __func__)); 3422 3423 /* 3424 * queues that belong to the adapter (not any particular port). 3425 */ 3426 rc = t4_setup_adapter_queues(sc); 3427 if (rc != 0) 3428 goto done; 3429 3430 for (i = 0; i < nitems(sc->tq); i++) { 3431 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 3432 taskqueue_thread_enqueue, &sc->tq[i]); 3433 if (sc->tq[i] == NULL) { 3434 device_printf(sc->dev, 3435 "failed to allocate task queue %d\n", i); 3436 rc = ENOMEM; 3437 goto done; 3438 } 3439 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3440 device_get_nameunit(sc->dev), i); 3441 } 3442 3443 t4_intr_enable(sc); 3444 sc->flags |= FULL_INIT_DONE; 3445 done: 3446 if (rc != 0) 3447 adapter_full_uninit(sc); 3448 3449 return (rc); 3450 } 3451 3452 int 3453 adapter_full_uninit(struct adapter *sc) 3454 { 3455 int i; 3456 3457 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3458 3459 t4_teardown_adapter_queues(sc); 3460 3461 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3462 taskqueue_free(sc->tq[i]); 3463 sc->tq[i] = NULL; 3464 } 3465 3466 sc->flags &= ~FULL_INIT_DONE; 3467 3468 return (0); 3469 } 3470 3471 #ifdef RSS 3472 #define SUPPORTED_RSS_HASHTYPES (RSS_HASHTYPE_RSS_IPV4 | \ 3473 RSS_HASHTYPE_RSS_TCP_IPV4 | RSS_HASHTYPE_RSS_IPV6 | \ 3474 RSS_HASHTYPE_RSS_TCP_IPV6 | RSS_HASHTYPE_RSS_UDP_IPV4 | \ 3475 RSS_HASHTYPE_RSS_UDP_IPV6) 3476 3477 /* Translates kernel hash types to hardware. */ 3478 static int 3479 hashconfig_to_hashen(int hashconfig) 3480 { 3481 int hashen = 0; 3482 3483 if (hashconfig & RSS_HASHTYPE_RSS_IPV4) 3484 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN; 3485 if (hashconfig & RSS_HASHTYPE_RSS_IPV6) 3486 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN; 3487 if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV4) { 3488 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN | 3489 F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN; 3490 } 3491 if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV6) { 3492 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN | 3493 F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN; 3494 } 3495 if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV4) 3496 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN; 3497 if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV6) 3498 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN; 3499 3500 return (hashen); 3501 } 3502 3503 /* Translates hardware hash types to kernel. */ 3504 static int 3505 hashen_to_hashconfig(int hashen) 3506 { 3507 int hashconfig = 0; 3508 3509 if (hashen & F_FW_RSS_VI_CONFIG_CMD_UDPEN) { 3510 /* 3511 * If UDP hashing was enabled it must have been enabled for 3512 * either IPv4 or IPv6 (inclusive or). Enabling UDP without 3513 * enabling any 4-tuple hash is nonsense configuration. 3514 */ 3515 MPASS(hashen & (F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN | 3516 F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)); 3517 3518 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) 3519 hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV4; 3520 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) 3521 hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV6; 3522 } 3523 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) 3524 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV4; 3525 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) 3526 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV6; 3527 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN) 3528 hashconfig |= RSS_HASHTYPE_RSS_IPV4; 3529 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN) 3530 hashconfig |= RSS_HASHTYPE_RSS_IPV6; 3531 3532 return (hashconfig); 3533 } 3534 #endif 3535 3536 int 3537 port_full_init(struct port_info *pi) 3538 { 3539 struct adapter *sc = pi->adapter; 3540 struct ifnet *ifp = pi->ifp; 3541 uint16_t *rss; 3542 struct sge_rxq *rxq; 3543 int rc, i, j, hashen; 3544 #ifdef RSS 3545 int nbuckets = rss_getnumbuckets(); 3546 int hashconfig = rss_gethashconfig(); 3547 int extra; 3548 uint32_t raw_rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; 3549 uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; 3550 #endif 3551 3552 ASSERT_SYNCHRONIZED_OP(sc); 3553 KASSERT((pi->flags & PORT_INIT_DONE) == 0, 3554 ("%s: PORT_INIT_DONE already", __func__)); 3555 3556 sysctl_ctx_init(&pi->ctx); 3557 pi->flags |= PORT_SYSCTL_CTX; 3558 3559 /* 3560 * Allocate tx/rx/fl queues for this port. 3561 */ 3562 rc = t4_setup_port_queues(pi); 3563 if (rc != 0) 3564 goto done; /* error message displayed already */ 3565 3566 /* 3567 * Setup RSS for this port. Save a copy of the RSS table for later use. 3568 */ 3569 if (pi->nrxq > pi->rss_size) { 3570 if_printf(ifp, "nrxq (%d) > hw RSS table size (%d); " 3571 "some queues will never receive traffic.\n", pi->nrxq, 3572 pi->rss_size); 3573 } else if (pi->rss_size % pi->nrxq) { 3574 if_printf(ifp, "nrxq (%d), hw RSS table size (%d); " 3575 "expect uneven traffic distribution.\n", pi->nrxq, 3576 pi->rss_size); 3577 } 3578 #ifdef RSS 3579 MPASS(RSS_KEYSIZE == 40); 3580 if (pi->nrxq != nbuckets) { 3581 if_printf(ifp, "nrxq (%d) != kernel RSS buckets (%d);" 3582 "performance will be impacted.\n", pi->nrxq, nbuckets); 3583 } 3584 3585 rss_getkey((void *)&raw_rss_key[0]); 3586 for (i = 0; i < nitems(rss_key); i++) { 3587 rss_key[i] = htobe32(raw_rss_key[nitems(rss_key) - 1 - i]); 3588 } 3589 t4_write_rss_key(sc, (void *)&rss_key[0], -1); 3590 #endif 3591 rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); 3592 for (i = 0; i < pi->rss_size;) { 3593 #ifdef RSS 3594 j = rss_get_indirection_to_bucket(i); 3595 j %= pi->nrxq; 3596 rxq = &sc->sge.rxq[pi->first_rxq + j]; 3597 rss[i++] = rxq->iq.abs_id; 3598 #else 3599 for_each_rxq(pi, j, rxq) { 3600 rss[i++] = rxq->iq.abs_id; 3601 if (i == pi->rss_size) 3602 break; 3603 } 3604 #endif 3605 } 3606 3607 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss, 3608 pi->rss_size); 3609 if (rc != 0) { 3610 if_printf(ifp, "rss_config failed: %d\n", rc); 3611 goto done; 3612 } 3613 3614 #ifdef RSS 3615 hashen = hashconfig_to_hashen(hashconfig); 3616 3617 /* 3618 * We may have had to enable some hashes even though the global config 3619 * wants them disabled. This is a potential problem that must be 3620 * reported to the user. 3621 */ 3622 extra = hashen_to_hashconfig(hashen) ^ hashconfig; 3623 3624 /* 3625 * If we consider only the supported hash types, then the enabled hashes 3626 * are a superset of the requested hashes. In other words, there cannot 3627 * be any supported hash that was requested but not enabled, but there 3628 * can be hashes that were not requested but had to be enabled. 3629 */ 3630 extra &= SUPPORTED_RSS_HASHTYPES; 3631 MPASS((extra & hashconfig) == 0); 3632 3633 if (extra) { 3634 if_printf(ifp, 3635 "global RSS config (0x%x) cannot be accomodated.\n", 3636 hashconfig); 3637 } 3638 if (extra & RSS_HASHTYPE_RSS_IPV4) 3639 if_printf(ifp, "IPv4 2-tuple hashing forced on.\n"); 3640 if (extra & RSS_HASHTYPE_RSS_TCP_IPV4) 3641 if_printf(ifp, "TCP/IPv4 4-tuple hashing forced on.\n"); 3642 if (extra & RSS_HASHTYPE_RSS_IPV6) 3643 if_printf(ifp, "IPv6 2-tuple hashing forced on.\n"); 3644 if (extra & RSS_HASHTYPE_RSS_TCP_IPV6) 3645 if_printf(ifp, "TCP/IPv6 4-tuple hashing forced on.\n"); 3646 if (extra & RSS_HASHTYPE_RSS_UDP_IPV4) 3647 if_printf(ifp, "UDP/IPv4 4-tuple hashing forced on.\n"); 3648 if (extra & RSS_HASHTYPE_RSS_UDP_IPV6) 3649 if_printf(ifp, "UDP/IPv6 4-tuple hashing forced on.\n"); 3650 #else 3651 hashen = F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN | 3652 F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN | 3653 F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN | 3654 F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN | F_FW_RSS_VI_CONFIG_CMD_UDPEN; 3655 #endif 3656 rc = -t4_config_vi_rss(sc, sc->mbox, pi->viid, hashen, rss[0]); 3657 if (rc != 0) { 3658 if_printf(ifp, "rss hash/defaultq config failed: %d\n", rc); 3659 goto done; 3660 } 3661 3662 pi->rss = rss; 3663 pi->flags |= PORT_INIT_DONE; 3664 done: 3665 if (rc != 0) 3666 port_full_uninit(pi); 3667 3668 return (rc); 3669 } 3670 3671 /* 3672 * Idempotent. 3673 */ 3674 int 3675 port_full_uninit(struct port_info *pi) 3676 { 3677 struct adapter *sc = pi->adapter; 3678 int i; 3679 struct sge_rxq *rxq; 3680 struct sge_txq *txq; 3681 #ifdef TCP_OFFLOAD 3682 struct sge_ofld_rxq *ofld_rxq; 3683 struct sge_wrq *ofld_txq; 3684 #endif 3685 3686 if (pi->flags & PORT_INIT_DONE) { 3687 3688 /* Need to quiesce queues. */ 3689 3690 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]); 3691 3692 for_each_txq(pi, i, txq) { 3693 quiesce_txq(sc, txq); 3694 } 3695 3696 #ifdef TCP_OFFLOAD 3697 for_each_ofld_txq(pi, i, ofld_txq) { 3698 quiesce_wrq(sc, ofld_txq); 3699 } 3700 #endif 3701 3702 for_each_rxq(pi, i, rxq) { 3703 quiesce_iq(sc, &rxq->iq); 3704 quiesce_fl(sc, &rxq->fl); 3705 } 3706 3707 #ifdef TCP_OFFLOAD 3708 for_each_ofld_rxq(pi, i, ofld_rxq) { 3709 quiesce_iq(sc, &ofld_rxq->iq); 3710 quiesce_fl(sc, &ofld_rxq->fl); 3711 } 3712 #endif 3713 free(pi->rss, M_CXGBE); 3714 } 3715 3716 t4_teardown_port_queues(pi); 3717 pi->flags &= ~PORT_INIT_DONE; 3718 3719 return (0); 3720 } 3721 3722 static void 3723 quiesce_txq(struct adapter *sc, struct sge_txq *txq) 3724 { 3725 struct sge_eq *eq = &txq->eq; 3726 struct sge_qstat *spg = (void *)&eq->desc[eq->sidx]; 3727 3728 (void) sc; /* unused */ 3729 3730 #ifdef INVARIANTS 3731 TXQ_LOCK(txq); 3732 MPASS((eq->flags & EQ_ENABLED) == 0); 3733 TXQ_UNLOCK(txq); 3734 #endif 3735 3736 /* Wait for the mp_ring to empty. */ 3737 while (!mp_ring_is_idle(txq->r)) { 3738 mp_ring_check_drainage(txq->r, 0); 3739 pause("rquiesce", 1); 3740 } 3741 3742 /* Then wait for the hardware to finish. */ 3743 while (spg->cidx != htobe16(eq->pidx)) 3744 pause("equiesce", 1); 3745 3746 /* Finally, wait for the driver to reclaim all descriptors. */ 3747 while (eq->cidx != eq->pidx) 3748 pause("dquiesce", 1); 3749 } 3750 3751 static void 3752 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq) 3753 { 3754 3755 /* XXXTX */ 3756 } 3757 3758 static void 3759 quiesce_iq(struct adapter *sc, struct sge_iq *iq) 3760 { 3761 (void) sc; /* unused */ 3762 3763 /* Synchronize with the interrupt handler */ 3764 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 3765 pause("iqfree", 1); 3766 } 3767 3768 static void 3769 quiesce_fl(struct adapter *sc, struct sge_fl *fl) 3770 { 3771 mtx_lock(&sc->sfl_lock); 3772 FL_LOCK(fl); 3773 fl->flags |= FL_DOOMED; 3774 FL_UNLOCK(fl); 3775 mtx_unlock(&sc->sfl_lock); 3776 3777 callout_drain(&sc->sfl_callout); 3778 KASSERT((fl->flags & FL_STARVING) == 0, 3779 ("%s: still starving", __func__)); 3780 } 3781 3782 static int 3783 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 3784 driver_intr_t *handler, void *arg, char *name) 3785 { 3786 int rc; 3787 3788 irq->rid = rid; 3789 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 3790 RF_SHAREABLE | RF_ACTIVE); 3791 if (irq->res == NULL) { 3792 device_printf(sc->dev, 3793 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 3794 return (ENOMEM); 3795 } 3796 3797 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 3798 NULL, handler, arg, &irq->tag); 3799 if (rc != 0) { 3800 device_printf(sc->dev, 3801 "failed to setup interrupt for rid %d, name %s: %d\n", 3802 rid, name, rc); 3803 } else if (name) 3804 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 3805 3806 return (rc); 3807 } 3808 3809 static int 3810 t4_free_irq(struct adapter *sc, struct irq *irq) 3811 { 3812 if (irq->tag) 3813 bus_teardown_intr(sc->dev, irq->res, irq->tag); 3814 if (irq->res) 3815 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 3816 3817 bzero(irq, sizeof(*irq)); 3818 3819 return (0); 3820 } 3821 3822 static void 3823 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 3824 unsigned int end) 3825 { 3826 uint32_t *p = (uint32_t *)(buf + start); 3827 3828 for ( ; start <= end; start += sizeof(uint32_t)) 3829 *p++ = t4_read_reg(sc, start); 3830 } 3831 3832 static void 3833 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 3834 { 3835 int i, n; 3836 const unsigned int *reg_ranges; 3837 static const unsigned int t4_reg_ranges[] = { 3838 0x1008, 0x1108, 3839 0x1180, 0x11b4, 3840 0x11fc, 0x123c, 3841 0x1300, 0x173c, 3842 0x1800, 0x18fc, 3843 0x3000, 0x30d8, 3844 0x30e0, 0x5924, 3845 0x5960, 0x59d4, 3846 0x5a00, 0x5af8, 3847 0x6000, 0x6098, 3848 0x6100, 0x6150, 3849 0x6200, 0x6208, 3850 0x6240, 0x6248, 3851 0x6280, 0x6338, 3852 0x6370, 0x638c, 3853 0x6400, 0x643c, 3854 0x6500, 0x6524, 3855 0x6a00, 0x6a38, 3856 0x6a60, 0x6a78, 3857 0x6b00, 0x6b84, 3858 0x6bf0, 0x6c84, 3859 0x6cf0, 0x6d84, 3860 0x6df0, 0x6e84, 3861 0x6ef0, 0x6f84, 3862 0x6ff0, 0x7084, 3863 0x70f0, 0x7184, 3864 0x71f0, 0x7284, 3865 0x72f0, 0x7384, 3866 0x73f0, 0x7450, 3867 0x7500, 0x7530, 3868 0x7600, 0x761c, 3869 0x7680, 0x76cc, 3870 0x7700, 0x7798, 3871 0x77c0, 0x77fc, 3872 0x7900, 0x79fc, 3873 0x7b00, 0x7c38, 3874 0x7d00, 0x7efc, 3875 0x8dc0, 0x8e1c, 3876 0x8e30, 0x8e78, 3877 0x8ea0, 0x8f6c, 3878 0x8fc0, 0x9074, 3879 0x90fc, 0x90fc, 3880 0x9400, 0x9458, 3881 0x9600, 0x96bc, 3882 0x9800, 0x9808, 3883 0x9820, 0x983c, 3884 0x9850, 0x9864, 3885 0x9c00, 0x9c6c, 3886 0x9c80, 0x9cec, 3887 0x9d00, 0x9d6c, 3888 0x9d80, 0x9dec, 3889 0x9e00, 0x9e6c, 3890 0x9e80, 0x9eec, 3891 0x9f00, 0x9f6c, 3892 0x9f80, 0x9fec, 3893 0xd004, 0xd03c, 3894 0xdfc0, 0xdfe0, 3895 0xe000, 0xea7c, 3896 0xf000, 0x11110, 3897 0x11118, 0x11190, 3898 0x19040, 0x1906c, 3899 0x19078, 0x19080, 3900 0x1908c, 0x19124, 3901 0x19150, 0x191b0, 3902 0x191d0, 0x191e8, 3903 0x19238, 0x1924c, 3904 0x193f8, 0x19474, 3905 0x19490, 0x194f8, 3906 0x19800, 0x19f30, 3907 0x1a000, 0x1a06c, 3908 0x1a0b0, 0x1a120, 3909 0x1a128, 0x1a138, 3910 0x1a190, 0x1a1c4, 3911 0x1a1fc, 0x1a1fc, 3912 0x1e040, 0x1e04c, 3913 0x1e284, 0x1e28c, 3914 0x1e2c0, 0x1e2c0, 3915 0x1e2e0, 0x1e2e0, 3916 0x1e300, 0x1e384, 3917 0x1e3c0, 0x1e3c8, 3918 0x1e440, 0x1e44c, 3919 0x1e684, 0x1e68c, 3920 0x1e6c0, 0x1e6c0, 3921 0x1e6e0, 0x1e6e0, 3922 0x1e700, 0x1e784, 3923 0x1e7c0, 0x1e7c8, 3924 0x1e840, 0x1e84c, 3925 0x1ea84, 0x1ea8c, 3926 0x1eac0, 0x1eac0, 3927 0x1eae0, 0x1eae0, 3928 0x1eb00, 0x1eb84, 3929 0x1ebc0, 0x1ebc8, 3930 0x1ec40, 0x1ec4c, 3931 0x1ee84, 0x1ee8c, 3932 0x1eec0, 0x1eec0, 3933 0x1eee0, 0x1eee0, 3934 0x1ef00, 0x1ef84, 3935 0x1efc0, 0x1efc8, 3936 0x1f040, 0x1f04c, 3937 0x1f284, 0x1f28c, 3938 0x1f2c0, 0x1f2c0, 3939 0x1f2e0, 0x1f2e0, 3940 0x1f300, 0x1f384, 3941 0x1f3c0, 0x1f3c8, 3942 0x1f440, 0x1f44c, 3943 0x1f684, 0x1f68c, 3944 0x1f6c0, 0x1f6c0, 3945 0x1f6e0, 0x1f6e0, 3946 0x1f700, 0x1f784, 3947 0x1f7c0, 0x1f7c8, 3948 0x1f840, 0x1f84c, 3949 0x1fa84, 0x1fa8c, 3950 0x1fac0, 0x1fac0, 3951 0x1fae0, 0x1fae0, 3952 0x1fb00, 0x1fb84, 3953 0x1fbc0, 0x1fbc8, 3954 0x1fc40, 0x1fc4c, 3955 0x1fe84, 0x1fe8c, 3956 0x1fec0, 0x1fec0, 3957 0x1fee0, 0x1fee0, 3958 0x1ff00, 0x1ff84, 3959 0x1ffc0, 0x1ffc8, 3960 0x20000, 0x2002c, 3961 0x20100, 0x2013c, 3962 0x20190, 0x201c8, 3963 0x20200, 0x20318, 3964 0x20400, 0x20528, 3965 0x20540, 0x20614, 3966 0x21000, 0x21040, 3967 0x2104c, 0x21060, 3968 0x210c0, 0x210ec, 3969 0x21200, 0x21268, 3970 0x21270, 0x21284, 3971 0x212fc, 0x21388, 3972 0x21400, 0x21404, 3973 0x21500, 0x21518, 3974 0x2152c, 0x2153c, 3975 0x21550, 0x21554, 3976 0x21600, 0x21600, 3977 0x21608, 0x21628, 3978 0x21630, 0x2163c, 3979 0x21700, 0x2171c, 3980 0x21780, 0x2178c, 3981 0x21800, 0x21c38, 3982 0x21c80, 0x21d7c, 3983 0x21e00, 0x21e04, 3984 0x22000, 0x2202c, 3985 0x22100, 0x2213c, 3986 0x22190, 0x221c8, 3987 0x22200, 0x22318, 3988 0x22400, 0x22528, 3989 0x22540, 0x22614, 3990 0x23000, 0x23040, 3991 0x2304c, 0x23060, 3992 0x230c0, 0x230ec, 3993 0x23200, 0x23268, 3994 0x23270, 0x23284, 3995 0x232fc, 0x23388, 3996 0x23400, 0x23404, 3997 0x23500, 0x23518, 3998 0x2352c, 0x2353c, 3999 0x23550, 0x23554, 4000 0x23600, 0x23600, 4001 0x23608, 0x23628, 4002 0x23630, 0x2363c, 4003 0x23700, 0x2371c, 4004 0x23780, 0x2378c, 4005 0x23800, 0x23c38, 4006 0x23c80, 0x23d7c, 4007 0x23e00, 0x23e04, 4008 0x24000, 0x2402c, 4009 0x24100, 0x2413c, 4010 0x24190, 0x241c8, 4011 0x24200, 0x24318, 4012 0x24400, 0x24528, 4013 0x24540, 0x24614, 4014 0x25000, 0x25040, 4015 0x2504c, 0x25060, 4016 0x250c0, 0x250ec, 4017 0x25200, 0x25268, 4018 0x25270, 0x25284, 4019 0x252fc, 0x25388, 4020 0x25400, 0x25404, 4021 0x25500, 0x25518, 4022 0x2552c, 0x2553c, 4023 0x25550, 0x25554, 4024 0x25600, 0x25600, 4025 0x25608, 0x25628, 4026 0x25630, 0x2563c, 4027 0x25700, 0x2571c, 4028 0x25780, 0x2578c, 4029 0x25800, 0x25c38, 4030 0x25c80, 0x25d7c, 4031 0x25e00, 0x25e04, 4032 0x26000, 0x2602c, 4033 0x26100, 0x2613c, 4034 0x26190, 0x261c8, 4035 0x26200, 0x26318, 4036 0x26400, 0x26528, 4037 0x26540, 0x26614, 4038 0x27000, 0x27040, 4039 0x2704c, 0x27060, 4040 0x270c0, 0x270ec, 4041 0x27200, 0x27268, 4042 0x27270, 0x27284, 4043 0x272fc, 0x27388, 4044 0x27400, 0x27404, 4045 0x27500, 0x27518, 4046 0x2752c, 0x2753c, 4047 0x27550, 0x27554, 4048 0x27600, 0x27600, 4049 0x27608, 0x27628, 4050 0x27630, 0x2763c, 4051 0x27700, 0x2771c, 4052 0x27780, 0x2778c, 4053 0x27800, 0x27c38, 4054 0x27c80, 0x27d7c, 4055 0x27e00, 0x27e04 4056 }; 4057 static const unsigned int t5_reg_ranges[] = { 4058 0x1008, 0x1148, 4059 0x1180, 0x11b4, 4060 0x11fc, 0x123c, 4061 0x1280, 0x173c, 4062 0x1800, 0x18fc, 4063 0x3000, 0x3028, 4064 0x3060, 0x30d8, 4065 0x30e0, 0x30fc, 4066 0x3140, 0x357c, 4067 0x35a8, 0x35cc, 4068 0x35ec, 0x35ec, 4069 0x3600, 0x5624, 4070 0x56cc, 0x575c, 4071 0x580c, 0x5814, 4072 0x5890, 0x58bc, 4073 0x5940, 0x59dc, 4074 0x59fc, 0x5a18, 4075 0x5a60, 0x5a9c, 4076 0x5b94, 0x5bfc, 4077 0x6000, 0x6040, 4078 0x6058, 0x614c, 4079 0x7700, 0x7798, 4080 0x77c0, 0x78fc, 4081 0x7b00, 0x7c54, 4082 0x7d00, 0x7efc, 4083 0x8dc0, 0x8de0, 4084 0x8df8, 0x8e84, 4085 0x8ea0, 0x8f84, 4086 0x8fc0, 0x90f8, 4087 0x9400, 0x9470, 4088 0x9600, 0x96f4, 4089 0x9800, 0x9808, 4090 0x9820, 0x983c, 4091 0x9850, 0x9864, 4092 0x9c00, 0x9c6c, 4093 0x9c80, 0x9cec, 4094 0x9d00, 0x9d6c, 4095 0x9d80, 0x9dec, 4096 0x9e00, 0x9e6c, 4097 0x9e80, 0x9eec, 4098 0x9f00, 0x9f6c, 4099 0x9f80, 0xa020, 4100 0xd004, 0xd03c, 4101 0xdfc0, 0xdfe0, 4102 0xe000, 0x11088, 4103 0x1109c, 0x11110, 4104 0x11118, 0x1117c, 4105 0x11190, 0x11204, 4106 0x19040, 0x1906c, 4107 0x19078, 0x19080, 4108 0x1908c, 0x19124, 4109 0x19150, 0x191b0, 4110 0x191d0, 0x191e8, 4111 0x19238, 0x19290, 4112 0x193f8, 0x19474, 4113 0x19490, 0x194cc, 4114 0x194f0, 0x194f8, 4115 0x19c00, 0x19c60, 4116 0x19c94, 0x19e10, 4117 0x19e50, 0x19f34, 4118 0x19f40, 0x19f50, 4119 0x19f90, 0x19fe4, 4120 0x1a000, 0x1a06c, 4121 0x1a0b0, 0x1a120, 4122 0x1a128, 0x1a138, 4123 0x1a190, 0x1a1c4, 4124 0x1a1fc, 0x1a1fc, 4125 0x1e008, 0x1e00c, 4126 0x1e040, 0x1e04c, 4127 0x1e284, 0x1e290, 4128 0x1e2c0, 0x1e2c0, 4129 0x1e2e0, 0x1e2e0, 4130 0x1e300, 0x1e384, 4131 0x1e3c0, 0x1e3c8, 4132 0x1e408, 0x1e40c, 4133 0x1e440, 0x1e44c, 4134 0x1e684, 0x1e690, 4135 0x1e6c0, 0x1e6c0, 4136 0x1e6e0, 0x1e6e0, 4137 0x1e700, 0x1e784, 4138 0x1e7c0, 0x1e7c8, 4139 0x1e808, 0x1e80c, 4140 0x1e840, 0x1e84c, 4141 0x1ea84, 0x1ea90, 4142 0x1eac0, 0x1eac0, 4143 0x1eae0, 0x1eae0, 4144 0x1eb00, 0x1eb84, 4145 0x1ebc0, 0x1ebc8, 4146 0x1ec08, 0x1ec0c, 4147 0x1ec40, 0x1ec4c, 4148 0x1ee84, 0x1ee90, 4149 0x1eec0, 0x1eec0, 4150 0x1eee0, 0x1eee0, 4151 0x1ef00, 0x1ef84, 4152 0x1efc0, 0x1efc8, 4153 0x1f008, 0x1f00c, 4154 0x1f040, 0x1f04c, 4155 0x1f284, 0x1f290, 4156 0x1f2c0, 0x1f2c0, 4157 0x1f2e0, 0x1f2e0, 4158 0x1f300, 0x1f384, 4159 0x1f3c0, 0x1f3c8, 4160 0x1f408, 0x1f40c, 4161 0x1f440, 0x1f44c, 4162 0x1f684, 0x1f690, 4163 0x1f6c0, 0x1f6c0, 4164 0x1f6e0, 0x1f6e0, 4165 0x1f700, 0x1f784, 4166 0x1f7c0, 0x1f7c8, 4167 0x1f808, 0x1f80c, 4168 0x1f840, 0x1f84c, 4169 0x1fa84, 0x1fa90, 4170 0x1fac0, 0x1fac0, 4171 0x1fae0, 0x1fae0, 4172 0x1fb00, 0x1fb84, 4173 0x1fbc0, 0x1fbc8, 4174 0x1fc08, 0x1fc0c, 4175 0x1fc40, 0x1fc4c, 4176 0x1fe84, 0x1fe90, 4177 0x1fec0, 0x1fec0, 4178 0x1fee0, 0x1fee0, 4179 0x1ff00, 0x1ff84, 4180 0x1ffc0, 0x1ffc8, 4181 0x30000, 0x30030, 4182 0x30100, 0x30144, 4183 0x30190, 0x301d0, 4184 0x30200, 0x30318, 4185 0x30400, 0x3052c, 4186 0x30540, 0x3061c, 4187 0x30800, 0x30834, 4188 0x308c0, 0x30908, 4189 0x30910, 0x309ac, 4190 0x30a00, 0x30a2c, 4191 0x30a44, 0x30a50, 4192 0x30a74, 0x30c24, 4193 0x30d00, 0x30d00, 4194 0x30d08, 0x30d14, 4195 0x30d1c, 0x30d20, 4196 0x30d3c, 0x30d50, 4197 0x31200, 0x3120c, 4198 0x31220, 0x31220, 4199 0x31240, 0x31240, 4200 0x31600, 0x3160c, 4201 0x31a00, 0x31a1c, 4202 0x31e00, 0x31e20, 4203 0x31e38, 0x31e3c, 4204 0x31e80, 0x31e80, 4205 0x31e88, 0x31ea8, 4206 0x31eb0, 0x31eb4, 4207 0x31ec8, 0x31ed4, 4208 0x31fb8, 0x32004, 4209 0x32200, 0x32200, 4210 0x32208, 0x32240, 4211 0x32248, 0x32280, 4212 0x32288, 0x322c0, 4213 0x322c8, 0x322fc, 4214 0x32600, 0x32630, 4215 0x32a00, 0x32abc, 4216 0x32b00, 0x32b70, 4217 0x33000, 0x33048, 4218 0x33060, 0x3309c, 4219 0x330f0, 0x33148, 4220 0x33160, 0x3319c, 4221 0x331f0, 0x332e4, 4222 0x332f8, 0x333e4, 4223 0x333f8, 0x33448, 4224 0x33460, 0x3349c, 4225 0x334f0, 0x33548, 4226 0x33560, 0x3359c, 4227 0x335f0, 0x336e4, 4228 0x336f8, 0x337e4, 4229 0x337f8, 0x337fc, 4230 0x33814, 0x33814, 4231 0x3382c, 0x3382c, 4232 0x33880, 0x3388c, 4233 0x338e8, 0x338ec, 4234 0x33900, 0x33948, 4235 0x33960, 0x3399c, 4236 0x339f0, 0x33ae4, 4237 0x33af8, 0x33b10, 4238 0x33b28, 0x33b28, 4239 0x33b3c, 0x33b50, 4240 0x33bf0, 0x33c10, 4241 0x33c28, 0x33c28, 4242 0x33c3c, 0x33c50, 4243 0x33cf0, 0x33cfc, 4244 0x34000, 0x34030, 4245 0x34100, 0x34144, 4246 0x34190, 0x341d0, 4247 0x34200, 0x34318, 4248 0x34400, 0x3452c, 4249 0x34540, 0x3461c, 4250 0x34800, 0x34834, 4251 0x348c0, 0x34908, 4252 0x34910, 0x349ac, 4253 0x34a00, 0x34a2c, 4254 0x34a44, 0x34a50, 4255 0x34a74, 0x34c24, 4256 0x34d00, 0x34d00, 4257 0x34d08, 0x34d14, 4258 0x34d1c, 0x34d20, 4259 0x34d3c, 0x34d50, 4260 0x35200, 0x3520c, 4261 0x35220, 0x35220, 4262 0x35240, 0x35240, 4263 0x35600, 0x3560c, 4264 0x35a00, 0x35a1c, 4265 0x35e00, 0x35e20, 4266 0x35e38, 0x35e3c, 4267 0x35e80, 0x35e80, 4268 0x35e88, 0x35ea8, 4269 0x35eb0, 0x35eb4, 4270 0x35ec8, 0x35ed4, 4271 0x35fb8, 0x36004, 4272 0x36200, 0x36200, 4273 0x36208, 0x36240, 4274 0x36248, 0x36280, 4275 0x36288, 0x362c0, 4276 0x362c8, 0x362fc, 4277 0x36600, 0x36630, 4278 0x36a00, 0x36abc, 4279 0x36b00, 0x36b70, 4280 0x37000, 0x37048, 4281 0x37060, 0x3709c, 4282 0x370f0, 0x37148, 4283 0x37160, 0x3719c, 4284 0x371f0, 0x372e4, 4285 0x372f8, 0x373e4, 4286 0x373f8, 0x37448, 4287 0x37460, 0x3749c, 4288 0x374f0, 0x37548, 4289 0x37560, 0x3759c, 4290 0x375f0, 0x376e4, 4291 0x376f8, 0x377e4, 4292 0x377f8, 0x377fc, 4293 0x37814, 0x37814, 4294 0x3782c, 0x3782c, 4295 0x37880, 0x3788c, 4296 0x378e8, 0x378ec, 4297 0x37900, 0x37948, 4298 0x37960, 0x3799c, 4299 0x379f0, 0x37ae4, 4300 0x37af8, 0x37b10, 4301 0x37b28, 0x37b28, 4302 0x37b3c, 0x37b50, 4303 0x37bf0, 0x37c10, 4304 0x37c28, 0x37c28, 4305 0x37c3c, 0x37c50, 4306 0x37cf0, 0x37cfc, 4307 0x38000, 0x38030, 4308 0x38100, 0x38144, 4309 0x38190, 0x381d0, 4310 0x38200, 0x38318, 4311 0x38400, 0x3852c, 4312 0x38540, 0x3861c, 4313 0x38800, 0x38834, 4314 0x388c0, 0x38908, 4315 0x38910, 0x389ac, 4316 0x38a00, 0x38a2c, 4317 0x38a44, 0x38a50, 4318 0x38a74, 0x38c24, 4319 0x38d00, 0x38d00, 4320 0x38d08, 0x38d14, 4321 0x38d1c, 0x38d20, 4322 0x38d3c, 0x38d50, 4323 0x39200, 0x3920c, 4324 0x39220, 0x39220, 4325 0x39240, 0x39240, 4326 0x39600, 0x3960c, 4327 0x39a00, 0x39a1c, 4328 0x39e00, 0x39e20, 4329 0x39e38, 0x39e3c, 4330 0x39e80, 0x39e80, 4331 0x39e88, 0x39ea8, 4332 0x39eb0, 0x39eb4, 4333 0x39ec8, 0x39ed4, 4334 0x39fb8, 0x3a004, 4335 0x3a200, 0x3a200, 4336 0x3a208, 0x3a240, 4337 0x3a248, 0x3a280, 4338 0x3a288, 0x3a2c0, 4339 0x3a2c8, 0x3a2fc, 4340 0x3a600, 0x3a630, 4341 0x3aa00, 0x3aabc, 4342 0x3ab00, 0x3ab70, 4343 0x3b000, 0x3b048, 4344 0x3b060, 0x3b09c, 4345 0x3b0f0, 0x3b148, 4346 0x3b160, 0x3b19c, 4347 0x3b1f0, 0x3b2e4, 4348 0x3b2f8, 0x3b3e4, 4349 0x3b3f8, 0x3b448, 4350 0x3b460, 0x3b49c, 4351 0x3b4f0, 0x3b548, 4352 0x3b560, 0x3b59c, 4353 0x3b5f0, 0x3b6e4, 4354 0x3b6f8, 0x3b7e4, 4355 0x3b7f8, 0x3b7fc, 4356 0x3b814, 0x3b814, 4357 0x3b82c, 0x3b82c, 4358 0x3b880, 0x3b88c, 4359 0x3b8e8, 0x3b8ec, 4360 0x3b900, 0x3b948, 4361 0x3b960, 0x3b99c, 4362 0x3b9f0, 0x3bae4, 4363 0x3baf8, 0x3bb10, 4364 0x3bb28, 0x3bb28, 4365 0x3bb3c, 0x3bb50, 4366 0x3bbf0, 0x3bc10, 4367 0x3bc28, 0x3bc28, 4368 0x3bc3c, 0x3bc50, 4369 0x3bcf0, 0x3bcfc, 4370 0x3c000, 0x3c030, 4371 0x3c100, 0x3c144, 4372 0x3c190, 0x3c1d0, 4373 0x3c200, 0x3c318, 4374 0x3c400, 0x3c52c, 4375 0x3c540, 0x3c61c, 4376 0x3c800, 0x3c834, 4377 0x3c8c0, 0x3c908, 4378 0x3c910, 0x3c9ac, 4379 0x3ca00, 0x3ca2c, 4380 0x3ca44, 0x3ca50, 4381 0x3ca74, 0x3cc24, 4382 0x3cd00, 0x3cd00, 4383 0x3cd08, 0x3cd14, 4384 0x3cd1c, 0x3cd20, 4385 0x3cd3c, 0x3cd50, 4386 0x3d200, 0x3d20c, 4387 0x3d220, 0x3d220, 4388 0x3d240, 0x3d240, 4389 0x3d600, 0x3d60c, 4390 0x3da00, 0x3da1c, 4391 0x3de00, 0x3de20, 4392 0x3de38, 0x3de3c, 4393 0x3de80, 0x3de80, 4394 0x3de88, 0x3dea8, 4395 0x3deb0, 0x3deb4, 4396 0x3dec8, 0x3ded4, 4397 0x3dfb8, 0x3e004, 4398 0x3e200, 0x3e200, 4399 0x3e208, 0x3e240, 4400 0x3e248, 0x3e280, 4401 0x3e288, 0x3e2c0, 4402 0x3e2c8, 0x3e2fc, 4403 0x3e600, 0x3e630, 4404 0x3ea00, 0x3eabc, 4405 0x3eb00, 0x3eb70, 4406 0x3f000, 0x3f048, 4407 0x3f060, 0x3f09c, 4408 0x3f0f0, 0x3f148, 4409 0x3f160, 0x3f19c, 4410 0x3f1f0, 0x3f2e4, 4411 0x3f2f8, 0x3f3e4, 4412 0x3f3f8, 0x3f448, 4413 0x3f460, 0x3f49c, 4414 0x3f4f0, 0x3f548, 4415 0x3f560, 0x3f59c, 4416 0x3f5f0, 0x3f6e4, 4417 0x3f6f8, 0x3f7e4, 4418 0x3f7f8, 0x3f7fc, 4419 0x3f814, 0x3f814, 4420 0x3f82c, 0x3f82c, 4421 0x3f880, 0x3f88c, 4422 0x3f8e8, 0x3f8ec, 4423 0x3f900, 0x3f948, 4424 0x3f960, 0x3f99c, 4425 0x3f9f0, 0x3fae4, 4426 0x3faf8, 0x3fb10, 4427 0x3fb28, 0x3fb28, 4428 0x3fb3c, 0x3fb50, 4429 0x3fbf0, 0x3fc10, 4430 0x3fc28, 0x3fc28, 4431 0x3fc3c, 0x3fc50, 4432 0x3fcf0, 0x3fcfc, 4433 0x40000, 0x4000c, 4434 0x40040, 0x40068, 4435 0x4007c, 0x40144, 4436 0x40180, 0x4018c, 4437 0x40200, 0x40298, 4438 0x402ac, 0x4033c, 4439 0x403f8, 0x403fc, 4440 0x41304, 0x413c4, 4441 0x41400, 0x4141c, 4442 0x41480, 0x414d0, 4443 0x44000, 0x44078, 4444 0x440c0, 0x44278, 4445 0x442c0, 0x44478, 4446 0x444c0, 0x44678, 4447 0x446c0, 0x44878, 4448 0x448c0, 0x449fc, 4449 0x45000, 0x45068, 4450 0x45080, 0x45084, 4451 0x450a0, 0x450b0, 4452 0x45200, 0x45268, 4453 0x45280, 0x45284, 4454 0x452a0, 0x452b0, 4455 0x460c0, 0x460e4, 4456 0x47000, 0x4708c, 4457 0x47200, 0x47250, 4458 0x47400, 0x47420, 4459 0x47600, 0x47618, 4460 0x47800, 0x47814, 4461 0x48000, 0x4800c, 4462 0x48040, 0x48068, 4463 0x4807c, 0x48144, 4464 0x48180, 0x4818c, 4465 0x48200, 0x48298, 4466 0x482ac, 0x4833c, 4467 0x483f8, 0x483fc, 4468 0x49304, 0x493c4, 4469 0x49400, 0x4941c, 4470 0x49480, 0x494d0, 4471 0x4c000, 0x4c078, 4472 0x4c0c0, 0x4c278, 4473 0x4c2c0, 0x4c478, 4474 0x4c4c0, 0x4c678, 4475 0x4c6c0, 0x4c878, 4476 0x4c8c0, 0x4c9fc, 4477 0x4d000, 0x4d068, 4478 0x4d080, 0x4d084, 4479 0x4d0a0, 0x4d0b0, 4480 0x4d200, 0x4d268, 4481 0x4d280, 0x4d284, 4482 0x4d2a0, 0x4d2b0, 4483 0x4e0c0, 0x4e0e4, 4484 0x4f000, 0x4f08c, 4485 0x4f200, 0x4f250, 4486 0x4f400, 0x4f420, 4487 0x4f600, 0x4f618, 4488 0x4f800, 0x4f814, 4489 0x50000, 0x500cc, 4490 0x50400, 0x50400, 4491 0x50800, 0x508cc, 4492 0x50c00, 0x50c00, 4493 0x51000, 0x5101c, 4494 0x51300, 0x51308, 4495 }; 4496 4497 if (is_t4(sc)) { 4498 reg_ranges = &t4_reg_ranges[0]; 4499 n = nitems(t4_reg_ranges); 4500 } else { 4501 reg_ranges = &t5_reg_ranges[0]; 4502 n = nitems(t5_reg_ranges); 4503 } 4504 4505 regs->version = chip_id(sc) | chip_rev(sc) << 10; 4506 for (i = 0; i < n; i += 2) 4507 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 4508 } 4509 4510 static void 4511 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi) 4512 { 4513 int i; 4514 u_int v, tnl_cong_drops; 4515 struct timeval tv; 4516 const struct timeval interval = {0, 250000}; /* 250ms */ 4517 4518 getmicrotime(&tv); 4519 timevalsub(&tv, &interval); 4520 if (timevalcmp(&tv, &pi->last_refreshed, <)) 4521 return; 4522 4523 tnl_cong_drops = 0; 4524 t4_get_port_stats(sc, pi->tx_chan, &pi->stats); 4525 for (i = 0; i < NCHAN; i++) { 4526 if (pi->rx_chan_map & (1 << i)) { 4527 mtx_lock(&sc->regwin_lock); 4528 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 4529 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 4530 mtx_unlock(&sc->regwin_lock); 4531 tnl_cong_drops += v; 4532 } 4533 } 4534 pi->tnl_cong_drops = tnl_cong_drops; 4535 getmicrotime(&pi->last_refreshed); 4536 } 4537 4538 static void 4539 cxgbe_tick(void *arg) 4540 { 4541 struct port_info *pi = arg; 4542 struct adapter *sc = pi->adapter; 4543 struct ifnet *ifp = pi->ifp; 4544 4545 PORT_LOCK(pi); 4546 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 4547 PORT_UNLOCK(pi); 4548 return; /* without scheduling another callout */ 4549 } 4550 4551 cxgbe_refresh_stats(sc, pi); 4552 4553 callout_schedule(&pi->tick, hz); 4554 PORT_UNLOCK(pi); 4555 } 4556 4557 static void 4558 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 4559 { 4560 struct ifnet *vlan; 4561 4562 if (arg != ifp || ifp->if_type != IFT_ETHER) 4563 return; 4564 4565 vlan = VLAN_DEVAT(ifp, vid); 4566 VLAN_SETCOOKIE(vlan, ifp); 4567 } 4568 4569 static int 4570 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 4571 { 4572 4573 #ifdef INVARIANTS 4574 panic("%s: opcode 0x%02x on iq %p with payload %p", 4575 __func__, rss->opcode, iq, m); 4576 #else 4577 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 4578 __func__, rss->opcode, iq, m); 4579 m_freem(m); 4580 #endif 4581 return (EDOOFUS); 4582 } 4583 4584 int 4585 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 4586 { 4587 uintptr_t *loc, new; 4588 4589 if (opcode >= nitems(sc->cpl_handler)) 4590 return (EINVAL); 4591 4592 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 4593 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 4594 atomic_store_rel_ptr(loc, new); 4595 4596 return (0); 4597 } 4598 4599 static int 4600 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 4601 { 4602 4603 #ifdef INVARIANTS 4604 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 4605 #else 4606 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 4607 __func__, iq, ctrl); 4608 #endif 4609 return (EDOOFUS); 4610 } 4611 4612 int 4613 t4_register_an_handler(struct adapter *sc, an_handler_t h) 4614 { 4615 uintptr_t *loc, new; 4616 4617 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 4618 loc = (uintptr_t *) &sc->an_handler; 4619 atomic_store_rel_ptr(loc, new); 4620 4621 return (0); 4622 } 4623 4624 static int 4625 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 4626 { 4627 const struct cpl_fw6_msg *cpl = 4628 __containerof(rpl, struct cpl_fw6_msg, data[0]); 4629 4630 #ifdef INVARIANTS 4631 panic("%s: fw_msg type %d", __func__, cpl->type); 4632 #else 4633 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 4634 #endif 4635 return (EDOOFUS); 4636 } 4637 4638 int 4639 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 4640 { 4641 uintptr_t *loc, new; 4642 4643 if (type >= nitems(sc->fw_msg_handler)) 4644 return (EINVAL); 4645 4646 /* 4647 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 4648 * handler dispatch table. Reject any attempt to install a handler for 4649 * this subtype. 4650 */ 4651 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 4652 return (EINVAL); 4653 4654 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 4655 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 4656 atomic_store_rel_ptr(loc, new); 4657 4658 return (0); 4659 } 4660 4661 static int 4662 t4_sysctls(struct adapter *sc) 4663 { 4664 struct sysctl_ctx_list *ctx; 4665 struct sysctl_oid *oid; 4666 struct sysctl_oid_list *children, *c0; 4667 static char *caps[] = { 4668 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 4669 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL" /* caps[1] niccaps */ 4670 "\6HASHFILTER\7ETHOFLD", 4671 "\20\1TOE", /* caps[2] toecaps */ 4672 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 4673 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 4674 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 4675 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 4676 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 4677 "\4PO_INITIAOR\5PO_TARGET" 4678 }; 4679 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 4680 4681 ctx = device_get_sysctl_ctx(sc->dev); 4682 4683 /* 4684 * dev.t4nex.X. 4685 */ 4686 oid = device_get_sysctl_tree(sc->dev); 4687 c0 = children = SYSCTL_CHILDREN(oid); 4688 4689 sc->sc_do_rxcopy = 1; 4690 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, 4691 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); 4692 4693 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 4694 sc->params.nports, "# of ports"); 4695 4696 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 4697 NULL, chip_rev(sc), "chip hardware revision"); 4698 4699 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 4700 CTLFLAG_RD, sc->fw_version, 0, "firmware version"); 4701 4702 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 4703 CTLFLAG_RD, sc->cfg_file, 0, "configuration file"); 4704 4705 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 4706 sc->cfcsum, "config file checksum"); 4707 4708 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 4709 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 4710 sysctl_bitfield, "A", "available doorbells"); 4711 4712 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 4713 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 4714 sysctl_bitfield, "A", "available link capabilities"); 4715 4716 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 4717 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 4718 sysctl_bitfield, "A", "available NIC capabilities"); 4719 4720 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 4721 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 4722 sysctl_bitfield, "A", "available TCP offload capabilities"); 4723 4724 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 4725 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 4726 sysctl_bitfield, "A", "available RDMA capabilities"); 4727 4728 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 4729 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 4730 sysctl_bitfield, "A", "available iSCSI capabilities"); 4731 4732 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 4733 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 4734 sysctl_bitfield, "A", "available FCoE capabilities"); 4735 4736 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 4737 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 4738 4739 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 4740 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 4741 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 4742 "interrupt holdoff timer values (us)"); 4743 4744 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 4745 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 4746 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 4747 "interrupt holdoff packet counter values"); 4748 4749 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, 4750 NULL, sc->tids.nftids, "number of filters"); 4751 4752 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | 4753 CTLFLAG_RD, sc, 0, sysctl_temperature, "I", 4754 "chip temperature (in Celsius)"); 4755 4756 t4_sge_sysctls(sc, ctx, children); 4757 4758 sc->lro_timeout = 100; 4759 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, 4760 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); 4761 4762 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "debug_flags", CTLFLAG_RW, 4763 &sc->debug_flags, 0, "flags to enable runtime debugging"); 4764 4765 #ifdef SBUF_DRAIN 4766 /* 4767 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 4768 */ 4769 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 4770 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 4771 "logs and miscellaneous information"); 4772 children = SYSCTL_CHILDREN(oid); 4773 4774 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 4775 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4776 sysctl_cctrl, "A", "congestion control"); 4777 4778 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 4779 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4780 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 4781 4782 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 4783 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 4784 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 4785 4786 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 4787 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 4788 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 4789 4790 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 4791 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 4792 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 4793 4794 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 4795 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 4796 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 4797 4798 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 4799 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 4800 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 4801 4802 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 4803 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4804 sysctl_cim_la, "A", "CIM logic analyzer"); 4805 4806 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", 4807 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4808 sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); 4809 4810 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 4811 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 4812 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 4813 4814 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 4815 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 4816 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 4817 4818 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 4819 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 4820 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 4821 4822 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 4823 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 4824 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 4825 4826 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 4827 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 4828 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 4829 4830 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 4831 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 4832 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 4833 4834 if (is_t5(sc)) { 4835 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 4836 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 4837 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 4838 4839 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 4840 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 4841 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 4842 } 4843 4844 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", 4845 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4846 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); 4847 4848 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 4849 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4850 sysctl_cim_qcfg, "A", "CIM queue configuration"); 4851 4852 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 4853 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4854 sysctl_cpl_stats, "A", "CPL statistics"); 4855 4856 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 4857 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4858 sysctl_ddp_stats, "A", "non-TCP DDP statistics"); 4859 4860 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 4861 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4862 sysctl_devlog, "A", "firmware's device log"); 4863 4864 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 4865 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4866 sysctl_fcoe_stats, "A", "FCoE statistics"); 4867 4868 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 4869 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4870 sysctl_hw_sched, "A", "hardware scheduler "); 4871 4872 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 4873 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4874 sysctl_l2t, "A", "hardware L2 table"); 4875 4876 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 4877 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4878 sysctl_lb_stats, "A", "loopback statistics"); 4879 4880 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 4881 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4882 sysctl_meminfo, "A", "memory regions"); 4883 4884 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", 4885 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4886 sysctl_mps_tcam, "A", "MPS TCAM entries"); 4887 4888 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 4889 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4890 sysctl_path_mtus, "A", "path MTUs"); 4891 4892 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 4893 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4894 sysctl_pm_stats, "A", "PM statistics"); 4895 4896 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 4897 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4898 sysctl_rdma_stats, "A", "RDMA statistics"); 4899 4900 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 4901 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4902 sysctl_tcp_stats, "A", "TCP statistics"); 4903 4904 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 4905 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4906 sysctl_tids, "A", "TID information"); 4907 4908 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 4909 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4910 sysctl_tp_err_stats, "A", "TP error statistics"); 4911 4912 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", 4913 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4914 sysctl_tp_la, "A", "TP logic analyzer"); 4915 4916 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 4917 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4918 sysctl_tx_rate, "A", "Tx rate"); 4919 4920 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", 4921 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4922 sysctl_ulprx_la, "A", "ULPRX logic analyzer"); 4923 4924 if (is_t5(sc)) { 4925 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 4926 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4927 sysctl_wcwr_stats, "A", "write combined work requests"); 4928 } 4929 #endif 4930 4931 #ifdef TCP_OFFLOAD 4932 if (is_offload(sc)) { 4933 /* 4934 * dev.t4nex.X.toe. 4935 */ 4936 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 4937 NULL, "TOE parameters"); 4938 children = SYSCTL_CHILDREN(oid); 4939 4940 sc->tt.sndbuf = 256 * 1024; 4941 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 4942 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 4943 4944 sc->tt.ddp = 0; 4945 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 4946 &sc->tt.ddp, 0, "DDP allowed"); 4947 4948 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 4949 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 4950 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 4951 4952 sc->tt.ddp_thres = 4953 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 4954 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 4955 &sc->tt.ddp_thres, 0, "DDP threshold"); 4956 4957 sc->tt.rx_coalesce = 1; 4958 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", 4959 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); 4960 4961 sc->tt.tx_align = 1; 4962 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align", 4963 CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload"); 4964 } 4965 #endif 4966 4967 4968 return (0); 4969 } 4970 4971 static int 4972 cxgbe_sysctls(struct port_info *pi) 4973 { 4974 struct sysctl_ctx_list *ctx; 4975 struct sysctl_oid *oid; 4976 struct sysctl_oid_list *children; 4977 struct adapter *sc = pi->adapter; 4978 4979 ctx = device_get_sysctl_ctx(pi->dev); 4980 4981 /* 4982 * dev.cxgbe.X. 4983 */ 4984 oid = device_get_sysctl_tree(pi->dev); 4985 children = SYSCTL_CHILDREN(oid); 4986 4987 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | 4988 CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); 4989 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { 4990 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 4991 CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I", 4992 "PHY temperature (in Celsius)"); 4993 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", 4994 CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I", 4995 "PHY firmware version"); 4996 } 4997 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 4998 &pi->nrxq, 0, "# of rx queues"); 4999 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 5000 &pi->ntxq, 0, "# of tx queues"); 5001 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 5002 &pi->first_rxq, 0, "index of first rx queue"); 5003 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 5004 &pi->first_txq, 0, "index of first tx queue"); 5005 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT | 5006 CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU", 5007 "Reserve queue 0 for non-flowid packets"); 5008 5009 #ifdef TCP_OFFLOAD 5010 if (is_offload(sc)) { 5011 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 5012 &pi->nofldrxq, 0, 5013 "# of rx queues for offloaded TCP connections"); 5014 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 5015 &pi->nofldtxq, 0, 5016 "# of tx queues for offloaded TCP connections"); 5017 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 5018 CTLFLAG_RD, &pi->first_ofld_rxq, 0, 5019 "index of first TOE rx queue"); 5020 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 5021 CTLFLAG_RD, &pi->first_ofld_txq, 0, 5022 "index of first TOE tx queue"); 5023 } 5024 #endif 5025 #ifdef DEV_NETMAP 5026 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD, 5027 &pi->nnmrxq, 0, "# of rx queues for netmap"); 5028 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD, 5029 &pi->nnmtxq, 0, "# of tx queues for netmap"); 5030 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq", 5031 CTLFLAG_RD, &pi->first_nm_rxq, 0, 5032 "index of first netmap rx queue"); 5033 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq", 5034 CTLFLAG_RD, &pi->first_nm_txq, 0, 5035 "index of first netmap tx queue"); 5036 #endif 5037 5038 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 5039 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 5040 "holdoff timer index"); 5041 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 5042 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 5043 "holdoff packet counter index"); 5044 5045 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 5046 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 5047 "rx queue size"); 5048 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 5049 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 5050 "tx queue size"); 5051 5052 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings", 5053 CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings, 5054 "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)"); 5055 5056 /* 5057 * dev.cxgbe.X.stats. 5058 */ 5059 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 5060 NULL, "port statistics"); 5061 children = SYSCTL_CHILDREN(oid); 5062 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD, 5063 &pi->tx_parse_error, 0, 5064 "# of tx packets with invalid length or # of segments"); 5065 5066 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 5067 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 5068 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ 5069 sysctl_handle_t4_reg64, "QU", desc) 5070 5071 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 5072 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 5073 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 5074 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 5075 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 5076 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 5077 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 5078 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 5079 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 5080 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 5081 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 5082 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 5083 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 5084 "# of tx frames in this range", 5085 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 5086 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 5087 "# of tx frames in this range", 5088 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 5089 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 5090 "# of tx frames in this range", 5091 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 5092 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 5093 "# of tx frames in this range", 5094 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 5095 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 5096 "# of tx frames in this range", 5097 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 5098 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 5099 "# of tx frames in this range", 5100 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 5101 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 5102 "# of tx frames in this range", 5103 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 5104 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 5105 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 5106 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 5107 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 5108 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 5109 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 5110 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 5111 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 5112 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 5113 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 5114 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 5115 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 5116 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 5117 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 5118 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 5119 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 5120 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 5121 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 5122 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 5123 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 5124 5125 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 5126 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 5127 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 5128 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 5129 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 5130 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 5131 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 5132 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 5133 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 5134 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 5135 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 5136 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 5137 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 5138 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 5139 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 5140 "# of frames received with bad FCS", 5141 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 5142 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 5143 "# of frames received with length error", 5144 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 5145 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 5146 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 5147 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 5148 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 5149 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 5150 "# of rx frames in this range", 5151 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 5152 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 5153 "# of rx frames in this range", 5154 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 5155 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 5156 "# of rx frames in this range", 5157 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 5158 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 5159 "# of rx frames in this range", 5160 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 5161 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 5162 "# of rx frames in this range", 5163 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 5164 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 5165 "# of rx frames in this range", 5166 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 5167 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 5168 "# of rx frames in this range", 5169 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 5170 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 5171 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 5172 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 5173 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 5174 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 5175 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 5176 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 5177 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 5178 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 5179 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 5180 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 5181 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 5182 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 5183 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 5184 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 5185 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 5186 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 5187 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 5188 5189 #undef SYSCTL_ADD_T4_REG64 5190 5191 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 5192 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 5193 &pi->stats.name, desc) 5194 5195 /* We get these from port_stats and they may be stale by upto 1s */ 5196 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 5197 "# drops due to buffer-group 0 overflows"); 5198 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 5199 "# drops due to buffer-group 1 overflows"); 5200 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 5201 "# drops due to buffer-group 2 overflows"); 5202 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 5203 "# drops due to buffer-group 3 overflows"); 5204 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 5205 "# of buffer-group 0 truncated packets"); 5206 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 5207 "# of buffer-group 1 truncated packets"); 5208 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 5209 "# of buffer-group 2 truncated packets"); 5210 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 5211 "# of buffer-group 3 truncated packets"); 5212 5213 #undef SYSCTL_ADD_T4_PORTSTAT 5214 5215 return (0); 5216 } 5217 5218 static int 5219 sysctl_int_array(SYSCTL_HANDLER_ARGS) 5220 { 5221 int rc, *i, space = 0; 5222 struct sbuf sb; 5223 5224 sbuf_new_for_sysctl(&sb, NULL, 64, req); 5225 for (i = arg1; arg2; arg2 -= sizeof(int), i++) { 5226 if (space) 5227 sbuf_printf(&sb, " "); 5228 sbuf_printf(&sb, "%d", *i); 5229 space = 1; 5230 } 5231 rc = sbuf_finish(&sb); 5232 sbuf_delete(&sb); 5233 return (rc); 5234 } 5235 5236 static int 5237 sysctl_bitfield(SYSCTL_HANDLER_ARGS) 5238 { 5239 int rc; 5240 struct sbuf *sb; 5241 5242 rc = sysctl_wire_old_buffer(req, 0); 5243 if (rc != 0) 5244 return(rc); 5245 5246 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5247 if (sb == NULL) 5248 return (ENOMEM); 5249 5250 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 5251 rc = sbuf_finish(sb); 5252 sbuf_delete(sb); 5253 5254 return (rc); 5255 } 5256 5257 static int 5258 sysctl_btphy(SYSCTL_HANDLER_ARGS) 5259 { 5260 struct port_info *pi = arg1; 5261 int op = arg2; 5262 struct adapter *sc = pi->adapter; 5263 u_int v; 5264 int rc; 5265 5266 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt"); 5267 if (rc) 5268 return (rc); 5269 /* XXX: magic numbers */ 5270 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, 5271 &v); 5272 end_synchronized_op(sc, 0); 5273 if (rc) 5274 return (rc); 5275 if (op == 0) 5276 v /= 256; 5277 5278 rc = sysctl_handle_int(oidp, &v, 0, req); 5279 return (rc); 5280 } 5281 5282 static int 5283 sysctl_noflowq(SYSCTL_HANDLER_ARGS) 5284 { 5285 struct port_info *pi = arg1; 5286 int rc, val; 5287 5288 val = pi->rsrv_noflowq; 5289 rc = sysctl_handle_int(oidp, &val, 0, req); 5290 if (rc != 0 || req->newptr == NULL) 5291 return (rc); 5292 5293 if ((val >= 1) && (pi->ntxq > 1)) 5294 pi->rsrv_noflowq = 1; 5295 else 5296 pi->rsrv_noflowq = 0; 5297 5298 return (rc); 5299 } 5300 5301 static int 5302 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 5303 { 5304 struct port_info *pi = arg1; 5305 struct adapter *sc = pi->adapter; 5306 int idx, rc, i; 5307 struct sge_rxq *rxq; 5308 #ifdef TCP_OFFLOAD 5309 struct sge_ofld_rxq *ofld_rxq; 5310 #endif 5311 uint8_t v; 5312 5313 idx = pi->tmr_idx; 5314 5315 rc = sysctl_handle_int(oidp, &idx, 0, req); 5316 if (rc != 0 || req->newptr == NULL) 5317 return (rc); 5318 5319 if (idx < 0 || idx >= SGE_NTIMERS) 5320 return (EINVAL); 5321 5322 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5323 "t4tmr"); 5324 if (rc) 5325 return (rc); 5326 5327 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 5328 for_each_rxq(pi, i, rxq) { 5329 #ifdef atomic_store_rel_8 5330 atomic_store_rel_8(&rxq->iq.intr_params, v); 5331 #else 5332 rxq->iq.intr_params = v; 5333 #endif 5334 } 5335 #ifdef TCP_OFFLOAD 5336 for_each_ofld_rxq(pi, i, ofld_rxq) { 5337 #ifdef atomic_store_rel_8 5338 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); 5339 #else 5340 ofld_rxq->iq.intr_params = v; 5341 #endif 5342 } 5343 #endif 5344 pi->tmr_idx = idx; 5345 5346 end_synchronized_op(sc, LOCK_HELD); 5347 return (0); 5348 } 5349 5350 static int 5351 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 5352 { 5353 struct port_info *pi = arg1; 5354 struct adapter *sc = pi->adapter; 5355 int idx, rc; 5356 5357 idx = pi->pktc_idx; 5358 5359 rc = sysctl_handle_int(oidp, &idx, 0, req); 5360 if (rc != 0 || req->newptr == NULL) 5361 return (rc); 5362 5363 if (idx < -1 || idx >= SGE_NCOUNTERS) 5364 return (EINVAL); 5365 5366 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5367 "t4pktc"); 5368 if (rc) 5369 return (rc); 5370 5371 if (pi->flags & PORT_INIT_DONE) 5372 rc = EBUSY; /* cannot be changed once the queues are created */ 5373 else 5374 pi->pktc_idx = idx; 5375 5376 end_synchronized_op(sc, LOCK_HELD); 5377 return (rc); 5378 } 5379 5380 static int 5381 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 5382 { 5383 struct port_info *pi = arg1; 5384 struct adapter *sc = pi->adapter; 5385 int qsize, rc; 5386 5387 qsize = pi->qsize_rxq; 5388 5389 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5390 if (rc != 0 || req->newptr == NULL) 5391 return (rc); 5392 5393 if (qsize < 128 || (qsize & 7)) 5394 return (EINVAL); 5395 5396 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5397 "t4rxqs"); 5398 if (rc) 5399 return (rc); 5400 5401 if (pi->flags & PORT_INIT_DONE) 5402 rc = EBUSY; /* cannot be changed once the queues are created */ 5403 else 5404 pi->qsize_rxq = qsize; 5405 5406 end_synchronized_op(sc, LOCK_HELD); 5407 return (rc); 5408 } 5409 5410 static int 5411 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 5412 { 5413 struct port_info *pi = arg1; 5414 struct adapter *sc = pi->adapter; 5415 int qsize, rc; 5416 5417 qsize = pi->qsize_txq; 5418 5419 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5420 if (rc != 0 || req->newptr == NULL) 5421 return (rc); 5422 5423 if (qsize < 128 || qsize > 65536) 5424 return (EINVAL); 5425 5426 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5427 "t4txqs"); 5428 if (rc) 5429 return (rc); 5430 5431 if (pi->flags & PORT_INIT_DONE) 5432 rc = EBUSY; /* cannot be changed once the queues are created */ 5433 else 5434 pi->qsize_txq = qsize; 5435 5436 end_synchronized_op(sc, LOCK_HELD); 5437 return (rc); 5438 } 5439 5440 static int 5441 sysctl_pause_settings(SYSCTL_HANDLER_ARGS) 5442 { 5443 struct port_info *pi = arg1; 5444 struct adapter *sc = pi->adapter; 5445 struct link_config *lc = &pi->link_cfg; 5446 int rc; 5447 5448 if (req->newptr == NULL) { 5449 struct sbuf *sb; 5450 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX"; 5451 5452 rc = sysctl_wire_old_buffer(req, 0); 5453 if (rc != 0) 5454 return(rc); 5455 5456 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5457 if (sb == NULL) 5458 return (ENOMEM); 5459 5460 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits); 5461 rc = sbuf_finish(sb); 5462 sbuf_delete(sb); 5463 } else { 5464 char s[2]; 5465 int n; 5466 5467 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX)); 5468 s[1] = 0; 5469 5470 rc = sysctl_handle_string(oidp, s, sizeof(s), req); 5471 if (rc != 0) 5472 return(rc); 5473 5474 if (s[1] != 0) 5475 return (EINVAL); 5476 if (s[0] < '0' || s[0] > '9') 5477 return (EINVAL); /* not a number */ 5478 n = s[0] - '0'; 5479 if (n & ~(PAUSE_TX | PAUSE_RX)) 5480 return (EINVAL); /* some other bit is set too */ 5481 5482 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE"); 5483 if (rc) 5484 return (rc); 5485 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) { 5486 int link_ok = lc->link_ok; 5487 5488 lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX); 5489 lc->requested_fc |= n; 5490 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc); 5491 lc->link_ok = link_ok; /* restore */ 5492 } 5493 end_synchronized_op(sc, 0); 5494 } 5495 5496 return (rc); 5497 } 5498 5499 static int 5500 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 5501 { 5502 struct adapter *sc = arg1; 5503 int reg = arg2; 5504 uint64_t val; 5505 5506 val = t4_read_reg64(sc, reg); 5507 5508 return (sysctl_handle_64(oidp, &val, 0, req)); 5509 } 5510 5511 static int 5512 sysctl_temperature(SYSCTL_HANDLER_ARGS) 5513 { 5514 struct adapter *sc = arg1; 5515 int rc, t; 5516 uint32_t param, val; 5517 5518 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); 5519 if (rc) 5520 return (rc); 5521 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 5522 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 5523 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); 5524 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 5525 end_synchronized_op(sc, 0); 5526 if (rc) 5527 return (rc); 5528 5529 /* unknown is returned as 0 but we display -1 in that case */ 5530 t = val == 0 ? -1 : val; 5531 5532 rc = sysctl_handle_int(oidp, &t, 0, req); 5533 return (rc); 5534 } 5535 5536 #ifdef SBUF_DRAIN 5537 static int 5538 sysctl_cctrl(SYSCTL_HANDLER_ARGS) 5539 { 5540 struct adapter *sc = arg1; 5541 struct sbuf *sb; 5542 int rc, i; 5543 uint16_t incr[NMTUS][NCCTRL_WIN]; 5544 static const char *dec_fac[] = { 5545 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 5546 "0.9375" 5547 }; 5548 5549 rc = sysctl_wire_old_buffer(req, 0); 5550 if (rc != 0) 5551 return (rc); 5552 5553 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5554 if (sb == NULL) 5555 return (ENOMEM); 5556 5557 t4_read_cong_tbl(sc, incr); 5558 5559 for (i = 0; i < NCCTRL_WIN; ++i) { 5560 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 5561 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 5562 incr[5][i], incr[6][i], incr[7][i]); 5563 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 5564 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 5565 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 5566 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 5567 } 5568 5569 rc = sbuf_finish(sb); 5570 sbuf_delete(sb); 5571 5572 return (rc); 5573 } 5574 5575 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 5576 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 5577 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 5578 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 5579 }; 5580 5581 static int 5582 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 5583 { 5584 struct adapter *sc = arg1; 5585 struct sbuf *sb; 5586 int rc, i, n, qid = arg2; 5587 uint32_t *buf, *p; 5588 char *qtype; 5589 u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5; 5590 5591 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 5592 ("%s: bad qid %d\n", __func__, qid)); 5593 5594 if (qid < CIM_NUM_IBQ) { 5595 /* inbound queue */ 5596 qtype = "IBQ"; 5597 n = 4 * CIM_IBQ_SIZE; 5598 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5599 rc = t4_read_cim_ibq(sc, qid, buf, n); 5600 } else { 5601 /* outbound queue */ 5602 qtype = "OBQ"; 5603 qid -= CIM_NUM_IBQ; 5604 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 5605 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5606 rc = t4_read_cim_obq(sc, qid, buf, n); 5607 } 5608 5609 if (rc < 0) { 5610 rc = -rc; 5611 goto done; 5612 } 5613 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 5614 5615 rc = sysctl_wire_old_buffer(req, 0); 5616 if (rc != 0) 5617 goto done; 5618 5619 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5620 if (sb == NULL) { 5621 rc = ENOMEM; 5622 goto done; 5623 } 5624 5625 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 5626 for (i = 0, p = buf; i < n; i += 16, p += 4) 5627 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 5628 p[2], p[3]); 5629 5630 rc = sbuf_finish(sb); 5631 sbuf_delete(sb); 5632 done: 5633 free(buf, M_CXGBE); 5634 return (rc); 5635 } 5636 5637 static int 5638 sysctl_cim_la(SYSCTL_HANDLER_ARGS) 5639 { 5640 struct adapter *sc = arg1; 5641 u_int cfg; 5642 struct sbuf *sb; 5643 uint32_t *buf, *p; 5644 int rc; 5645 5646 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 5647 if (rc != 0) 5648 return (rc); 5649 5650 rc = sysctl_wire_old_buffer(req, 0); 5651 if (rc != 0) 5652 return (rc); 5653 5654 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5655 if (sb == NULL) 5656 return (ENOMEM); 5657 5658 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 5659 M_ZERO | M_WAITOK); 5660 5661 rc = -t4_cim_read_la(sc, buf, NULL); 5662 if (rc != 0) 5663 goto done; 5664 5665 sbuf_printf(sb, "Status Data PC%s", 5666 cfg & F_UPDBGLACAPTPCONLY ? "" : 5667 " LS0Stat LS0Addr LS0Data"); 5668 5669 KASSERT((sc->params.cim_la_size & 7) == 0, 5670 ("%s: p will walk off the end of buf", __func__)); 5671 5672 for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) { 5673 if (cfg & F_UPDBGLACAPTPCONLY) { 5674 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 5675 p[6], p[7]); 5676 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 5677 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 5678 p[4] & 0xff, p[5] >> 8); 5679 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 5680 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5681 p[1] & 0xf, p[2] >> 4); 5682 } else { 5683 sbuf_printf(sb, 5684 "\n %02x %x%07x %x%07x %08x %08x " 5685 "%08x%08x%08x%08x", 5686 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5687 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 5688 p[6], p[7]); 5689 } 5690 } 5691 5692 rc = sbuf_finish(sb); 5693 sbuf_delete(sb); 5694 done: 5695 free(buf, M_CXGBE); 5696 return (rc); 5697 } 5698 5699 static int 5700 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) 5701 { 5702 struct adapter *sc = arg1; 5703 u_int i; 5704 struct sbuf *sb; 5705 uint32_t *buf, *p; 5706 int rc; 5707 5708 rc = sysctl_wire_old_buffer(req, 0); 5709 if (rc != 0) 5710 return (rc); 5711 5712 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5713 if (sb == NULL) 5714 return (ENOMEM); 5715 5716 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, 5717 M_ZERO | M_WAITOK); 5718 5719 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); 5720 p = buf; 5721 5722 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5723 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], 5724 p[1], p[0]); 5725 } 5726 5727 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); 5728 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5729 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", 5730 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, 5731 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, 5732 (p[1] >> 2) | ((p[2] & 3) << 30), 5733 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, 5734 p[0] & 1); 5735 } 5736 5737 rc = sbuf_finish(sb); 5738 sbuf_delete(sb); 5739 free(buf, M_CXGBE); 5740 return (rc); 5741 } 5742 5743 static int 5744 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) 5745 { 5746 struct adapter *sc = arg1; 5747 u_int i; 5748 struct sbuf *sb; 5749 uint32_t *buf, *p; 5750 int rc; 5751 5752 rc = sysctl_wire_old_buffer(req, 0); 5753 if (rc != 0) 5754 return (rc); 5755 5756 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5757 if (sb == NULL) 5758 return (ENOMEM); 5759 5760 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, 5761 M_ZERO | M_WAITOK); 5762 5763 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); 5764 p = buf; 5765 5766 sbuf_printf(sb, "Cntl ID DataBE Addr Data"); 5767 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5768 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", 5769 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, 5770 p[4], p[3], p[2], p[1], p[0]); 5771 } 5772 5773 sbuf_printf(sb, "\n\nCntl ID Data"); 5774 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5775 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", 5776 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); 5777 } 5778 5779 rc = sbuf_finish(sb); 5780 sbuf_delete(sb); 5781 free(buf, M_CXGBE); 5782 return (rc); 5783 } 5784 5785 static int 5786 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 5787 { 5788 struct adapter *sc = arg1; 5789 struct sbuf *sb; 5790 int rc, i; 5791 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5792 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5793 uint16_t thres[CIM_NUM_IBQ]; 5794 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 5795 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 5796 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 5797 5798 if (is_t4(sc)) { 5799 cim_num_obq = CIM_NUM_OBQ; 5800 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 5801 obq_rdaddr = A_UP_OBQ_0_REALADDR; 5802 } else { 5803 cim_num_obq = CIM_NUM_OBQ_T5; 5804 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 5805 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 5806 } 5807 nq = CIM_NUM_IBQ + cim_num_obq; 5808 5809 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 5810 if (rc == 0) 5811 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 5812 if (rc != 0) 5813 return (rc); 5814 5815 t4_read_cimq_cfg(sc, base, size, thres); 5816 5817 rc = sysctl_wire_old_buffer(req, 0); 5818 if (rc != 0) 5819 return (rc); 5820 5821 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5822 if (sb == NULL) 5823 return (ENOMEM); 5824 5825 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 5826 5827 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 5828 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 5829 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 5830 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5831 G_QUEREMFLITS(p[2]) * 16); 5832 for ( ; i < nq; i++, p += 4, wr += 2) 5833 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 5834 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 5835 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5836 G_QUEREMFLITS(p[2]) * 16); 5837 5838 rc = sbuf_finish(sb); 5839 sbuf_delete(sb); 5840 5841 return (rc); 5842 } 5843 5844 static int 5845 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 5846 { 5847 struct adapter *sc = arg1; 5848 struct sbuf *sb; 5849 int rc; 5850 struct tp_cpl_stats stats; 5851 5852 rc = sysctl_wire_old_buffer(req, 0); 5853 if (rc != 0) 5854 return (rc); 5855 5856 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5857 if (sb == NULL) 5858 return (ENOMEM); 5859 5860 t4_tp_get_cpl_stats(sc, &stats); 5861 5862 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 5863 "channel 3\n"); 5864 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 5865 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 5866 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 5867 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 5868 5869 rc = sbuf_finish(sb); 5870 sbuf_delete(sb); 5871 5872 return (rc); 5873 } 5874 5875 static int 5876 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 5877 { 5878 struct adapter *sc = arg1; 5879 struct sbuf *sb; 5880 int rc; 5881 struct tp_usm_stats stats; 5882 5883 rc = sysctl_wire_old_buffer(req, 0); 5884 if (rc != 0) 5885 return(rc); 5886 5887 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5888 if (sb == NULL) 5889 return (ENOMEM); 5890 5891 t4_get_usm_stats(sc, &stats); 5892 5893 sbuf_printf(sb, "Frames: %u\n", stats.frames); 5894 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 5895 sbuf_printf(sb, "Drops: %u", stats.drops); 5896 5897 rc = sbuf_finish(sb); 5898 sbuf_delete(sb); 5899 5900 return (rc); 5901 } 5902 5903 const char *devlog_level_strings[] = { 5904 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 5905 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 5906 [FW_DEVLOG_LEVEL_ERR] = "ERR", 5907 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 5908 [FW_DEVLOG_LEVEL_INFO] = "INFO", 5909 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 5910 }; 5911 5912 const char *devlog_facility_strings[] = { 5913 [FW_DEVLOG_FACILITY_CORE] = "CORE", 5914 [FW_DEVLOG_FACILITY_CF] = "CF", 5915 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 5916 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 5917 [FW_DEVLOG_FACILITY_RES] = "RES", 5918 [FW_DEVLOG_FACILITY_HW] = "HW", 5919 [FW_DEVLOG_FACILITY_FLR] = "FLR", 5920 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 5921 [FW_DEVLOG_FACILITY_PHY] = "PHY", 5922 [FW_DEVLOG_FACILITY_MAC] = "MAC", 5923 [FW_DEVLOG_FACILITY_PORT] = "PORT", 5924 [FW_DEVLOG_FACILITY_VI] = "VI", 5925 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 5926 [FW_DEVLOG_FACILITY_ACL] = "ACL", 5927 [FW_DEVLOG_FACILITY_TM] = "TM", 5928 [FW_DEVLOG_FACILITY_QFC] = "QFC", 5929 [FW_DEVLOG_FACILITY_DCB] = "DCB", 5930 [FW_DEVLOG_FACILITY_ETH] = "ETH", 5931 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 5932 [FW_DEVLOG_FACILITY_RI] = "RI", 5933 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 5934 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 5935 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 5936 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 5937 }; 5938 5939 static int 5940 sysctl_devlog(SYSCTL_HANDLER_ARGS) 5941 { 5942 struct adapter *sc = arg1; 5943 struct devlog_params *dparams = &sc->params.devlog; 5944 struct fw_devlog_e *buf, *e; 5945 int i, j, rc, nentries, first = 0, m; 5946 struct sbuf *sb; 5947 uint64_t ftstamp = UINT64_MAX; 5948 5949 if (dparams->start == 0) { 5950 dparams->memtype = FW_MEMTYPE_EDC0; 5951 dparams->start = 0x84000; 5952 dparams->size = 32768; 5953 } 5954 5955 nentries = dparams->size / sizeof(struct fw_devlog_e); 5956 5957 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 5958 if (buf == NULL) 5959 return (ENOMEM); 5960 5961 m = fwmtype_to_hwmtype(dparams->memtype); 5962 rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf); 5963 if (rc != 0) 5964 goto done; 5965 5966 for (i = 0; i < nentries; i++) { 5967 e = &buf[i]; 5968 5969 if (e->timestamp == 0) 5970 break; /* end */ 5971 5972 e->timestamp = be64toh(e->timestamp); 5973 e->seqno = be32toh(e->seqno); 5974 for (j = 0; j < 8; j++) 5975 e->params[j] = be32toh(e->params[j]); 5976 5977 if (e->timestamp < ftstamp) { 5978 ftstamp = e->timestamp; 5979 first = i; 5980 } 5981 } 5982 5983 if (buf[first].timestamp == 0) 5984 goto done; /* nothing in the log */ 5985 5986 rc = sysctl_wire_old_buffer(req, 0); 5987 if (rc != 0) 5988 goto done; 5989 5990 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5991 if (sb == NULL) { 5992 rc = ENOMEM; 5993 goto done; 5994 } 5995 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 5996 "Seq#", "Tstamp", "Level", "Facility", "Message"); 5997 5998 i = first; 5999 do { 6000 e = &buf[i]; 6001 if (e->timestamp == 0) 6002 break; /* end */ 6003 6004 sbuf_printf(sb, "%10d %15ju %8s %8s ", 6005 e->seqno, e->timestamp, 6006 (e->level < nitems(devlog_level_strings) ? 6007 devlog_level_strings[e->level] : "UNKNOWN"), 6008 (e->facility < nitems(devlog_facility_strings) ? 6009 devlog_facility_strings[e->facility] : "UNKNOWN")); 6010 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 6011 e->params[2], e->params[3], e->params[4], 6012 e->params[5], e->params[6], e->params[7]); 6013 6014 if (++i == nentries) 6015 i = 0; 6016 } while (i != first); 6017 6018 rc = sbuf_finish(sb); 6019 sbuf_delete(sb); 6020 done: 6021 free(buf, M_CXGBE); 6022 return (rc); 6023 } 6024 6025 static int 6026 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 6027 { 6028 struct adapter *sc = arg1; 6029 struct sbuf *sb; 6030 int rc; 6031 struct tp_fcoe_stats stats[4]; 6032 6033 rc = sysctl_wire_old_buffer(req, 0); 6034 if (rc != 0) 6035 return (rc); 6036 6037 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6038 if (sb == NULL) 6039 return (ENOMEM); 6040 6041 t4_get_fcoe_stats(sc, 0, &stats[0]); 6042 t4_get_fcoe_stats(sc, 1, &stats[1]); 6043 t4_get_fcoe_stats(sc, 2, &stats[2]); 6044 t4_get_fcoe_stats(sc, 3, &stats[3]); 6045 6046 sbuf_printf(sb, " channel 0 channel 1 " 6047 "channel 2 channel 3\n"); 6048 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 6049 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 6050 stats[3].octetsDDP); 6051 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 6052 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 6053 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 6054 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 6055 stats[3].framesDrop); 6056 6057 rc = sbuf_finish(sb); 6058 sbuf_delete(sb); 6059 6060 return (rc); 6061 } 6062 6063 static int 6064 sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 6065 { 6066 struct adapter *sc = arg1; 6067 struct sbuf *sb; 6068 int rc, i; 6069 unsigned int map, kbps, ipg, mode; 6070 unsigned int pace_tab[NTX_SCHED]; 6071 6072 rc = sysctl_wire_old_buffer(req, 0); 6073 if (rc != 0) 6074 return (rc); 6075 6076 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6077 if (sb == NULL) 6078 return (ENOMEM); 6079 6080 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 6081 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 6082 t4_read_pace_tbl(sc, pace_tab); 6083 6084 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 6085 "Class IPG (0.1 ns) Flow IPG (us)"); 6086 6087 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 6088 t4_get_tx_sched(sc, i, &kbps, &ipg); 6089 sbuf_printf(sb, "\n %u %-5s %u ", i, 6090 (mode & (1 << i)) ? "flow" : "class", map & 3); 6091 if (kbps) 6092 sbuf_printf(sb, "%9u ", kbps); 6093 else 6094 sbuf_printf(sb, " disabled "); 6095 6096 if (ipg) 6097 sbuf_printf(sb, "%13u ", ipg); 6098 else 6099 sbuf_printf(sb, " disabled "); 6100 6101 if (pace_tab[i]) 6102 sbuf_printf(sb, "%10u", pace_tab[i]); 6103 else 6104 sbuf_printf(sb, " disabled"); 6105 } 6106 6107 rc = sbuf_finish(sb); 6108 sbuf_delete(sb); 6109 6110 return (rc); 6111 } 6112 6113 static int 6114 sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 6115 { 6116 struct adapter *sc = arg1; 6117 struct sbuf *sb; 6118 int rc, i, j; 6119 uint64_t *p0, *p1; 6120 struct lb_port_stats s[2]; 6121 static const char *stat_name[] = { 6122 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 6123 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 6124 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 6125 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 6126 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 6127 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 6128 "BG2FramesTrunc:", "BG3FramesTrunc:" 6129 }; 6130 6131 rc = sysctl_wire_old_buffer(req, 0); 6132 if (rc != 0) 6133 return (rc); 6134 6135 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6136 if (sb == NULL) 6137 return (ENOMEM); 6138 6139 memset(s, 0, sizeof(s)); 6140 6141 for (i = 0; i < 4; i += 2) { 6142 t4_get_lb_stats(sc, i, &s[0]); 6143 t4_get_lb_stats(sc, i + 1, &s[1]); 6144 6145 p0 = &s[0].octets; 6146 p1 = &s[1].octets; 6147 sbuf_printf(sb, "%s Loopback %u" 6148 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 6149 6150 for (j = 0; j < nitems(stat_name); j++) 6151 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 6152 *p0++, *p1++); 6153 } 6154 6155 rc = sbuf_finish(sb); 6156 sbuf_delete(sb); 6157 6158 return (rc); 6159 } 6160 6161 static int 6162 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) 6163 { 6164 int rc = 0; 6165 struct port_info *pi = arg1; 6166 struct sbuf *sb; 6167 static const char *linkdnreasons[] = { 6168 "non-specific", "remote fault", "autoneg failed", "reserved3", 6169 "PHY overheated", "unknown", "rx los", "reserved7" 6170 }; 6171 6172 rc = sysctl_wire_old_buffer(req, 0); 6173 if (rc != 0) 6174 return(rc); 6175 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); 6176 if (sb == NULL) 6177 return (ENOMEM); 6178 6179 if (pi->linkdnrc < 0) 6180 sbuf_printf(sb, "n/a"); 6181 else if (pi->linkdnrc < nitems(linkdnreasons)) 6182 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]); 6183 else 6184 sbuf_printf(sb, "%d", pi->linkdnrc); 6185 6186 rc = sbuf_finish(sb); 6187 sbuf_delete(sb); 6188 6189 return (rc); 6190 } 6191 6192 struct mem_desc { 6193 unsigned int base; 6194 unsigned int limit; 6195 unsigned int idx; 6196 }; 6197 6198 static int 6199 mem_desc_cmp(const void *a, const void *b) 6200 { 6201 return ((const struct mem_desc *)a)->base - 6202 ((const struct mem_desc *)b)->base; 6203 } 6204 6205 static void 6206 mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 6207 unsigned int to) 6208 { 6209 unsigned int size; 6210 6211 size = to - from + 1; 6212 if (size == 0) 6213 return; 6214 6215 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 6216 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 6217 } 6218 6219 static int 6220 sysctl_meminfo(SYSCTL_HANDLER_ARGS) 6221 { 6222 struct adapter *sc = arg1; 6223 struct sbuf *sb; 6224 int rc, i, n; 6225 uint32_t lo, hi, used, alloc; 6226 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 6227 static const char *region[] = { 6228 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 6229 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 6230 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 6231 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 6232 "RQUDP region:", "PBL region:", "TXPBL region:", 6233 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 6234 "On-chip queues:" 6235 }; 6236 struct mem_desc avail[4]; 6237 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 6238 struct mem_desc *md = mem; 6239 6240 rc = sysctl_wire_old_buffer(req, 0); 6241 if (rc != 0) 6242 return (rc); 6243 6244 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6245 if (sb == NULL) 6246 return (ENOMEM); 6247 6248 for (i = 0; i < nitems(mem); i++) { 6249 mem[i].limit = 0; 6250 mem[i].idx = i; 6251 } 6252 6253 /* Find and sort the populated memory ranges */ 6254 i = 0; 6255 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 6256 if (lo & F_EDRAM0_ENABLE) { 6257 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 6258 avail[i].base = G_EDRAM0_BASE(hi) << 20; 6259 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 6260 avail[i].idx = 0; 6261 i++; 6262 } 6263 if (lo & F_EDRAM1_ENABLE) { 6264 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 6265 avail[i].base = G_EDRAM1_BASE(hi) << 20; 6266 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 6267 avail[i].idx = 1; 6268 i++; 6269 } 6270 if (lo & F_EXT_MEM_ENABLE) { 6271 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 6272 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 6273 avail[i].limit = avail[i].base + 6274 (G_EXT_MEM_SIZE(hi) << 20); 6275 avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */ 6276 i++; 6277 } 6278 if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) { 6279 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 6280 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 6281 avail[i].limit = avail[i].base + 6282 (G_EXT_MEM1_SIZE(hi) << 20); 6283 avail[i].idx = 4; 6284 i++; 6285 } 6286 if (!i) /* no memory available */ 6287 return 0; 6288 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 6289 6290 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 6291 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 6292 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 6293 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 6294 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 6295 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 6296 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 6297 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 6298 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 6299 6300 /* the next few have explicit upper bounds */ 6301 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 6302 md->limit = md->base - 1 + 6303 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 6304 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 6305 md++; 6306 6307 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 6308 md->limit = md->base - 1 + 6309 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 6310 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 6311 md++; 6312 6313 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6314 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 6315 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 6316 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 6317 } else { 6318 md->base = 0; 6319 md->idx = nitems(region); /* hide it */ 6320 } 6321 md++; 6322 6323 #define ulp_region(reg) \ 6324 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 6325 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 6326 6327 ulp_region(RX_ISCSI); 6328 ulp_region(RX_TDDP); 6329 ulp_region(TX_TPT); 6330 ulp_region(RX_STAG); 6331 ulp_region(RX_RQ); 6332 ulp_region(RX_RQUDP); 6333 ulp_region(RX_PBL); 6334 ulp_region(TX_PBL); 6335 #undef ulp_region 6336 6337 md->base = 0; 6338 md->idx = nitems(region); 6339 if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) { 6340 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); 6341 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc, 6342 A_SGE_DBVFIFO_SIZE))) << 2) - 1; 6343 } 6344 md++; 6345 6346 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 6347 md->limit = md->base + sc->tids.ntids - 1; 6348 md++; 6349 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 6350 md->limit = md->base + sc->tids.ntids - 1; 6351 md++; 6352 6353 md->base = sc->vres.ocq.start; 6354 if (sc->vres.ocq.size) 6355 md->limit = md->base + sc->vres.ocq.size - 1; 6356 else 6357 md->idx = nitems(region); /* hide it */ 6358 md++; 6359 6360 /* add any address-space holes, there can be up to 3 */ 6361 for (n = 0; n < i - 1; n++) 6362 if (avail[n].limit < avail[n + 1].base) 6363 (md++)->base = avail[n].limit; 6364 if (avail[n].limit) 6365 (md++)->base = avail[n].limit; 6366 6367 n = md - mem; 6368 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 6369 6370 for (lo = 0; lo < i; lo++) 6371 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 6372 avail[lo].limit - 1); 6373 6374 sbuf_printf(sb, "\n"); 6375 for (i = 0; i < n; i++) { 6376 if (mem[i].idx >= nitems(region)) 6377 continue; /* skip holes */ 6378 if (!mem[i].limit) 6379 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 6380 mem_region_show(sb, region[mem[i].idx], mem[i].base, 6381 mem[i].limit); 6382 } 6383 6384 sbuf_printf(sb, "\n"); 6385 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 6386 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 6387 mem_region_show(sb, "uP RAM:", lo, hi); 6388 6389 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 6390 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 6391 mem_region_show(sb, "uP Extmem2:", lo, hi); 6392 6393 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 6394 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 6395 G_PMRXMAXPAGE(lo), 6396 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 6397 (lo & F_PMRXNUMCHN) ? 2 : 1); 6398 6399 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 6400 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 6401 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 6402 G_PMTXMAXPAGE(lo), 6403 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 6404 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 6405 sbuf_printf(sb, "%u p-structs\n", 6406 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 6407 6408 for (i = 0; i < 4; i++) { 6409 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 6410 if (is_t4(sc)) { 6411 used = G_USED(lo); 6412 alloc = G_ALLOC(lo); 6413 } else { 6414 used = G_T5_USED(lo); 6415 alloc = G_T5_ALLOC(lo); 6416 } 6417 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 6418 i, used, alloc); 6419 } 6420 for (i = 0; i < 4; i++) { 6421 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 6422 if (is_t4(sc)) { 6423 used = G_USED(lo); 6424 alloc = G_ALLOC(lo); 6425 } else { 6426 used = G_T5_USED(lo); 6427 alloc = G_T5_ALLOC(lo); 6428 } 6429 sbuf_printf(sb, 6430 "\nLoopback %d using %u pages out of %u allocated", 6431 i, used, alloc); 6432 } 6433 6434 rc = sbuf_finish(sb); 6435 sbuf_delete(sb); 6436 6437 return (rc); 6438 } 6439 6440 static inline void 6441 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) 6442 { 6443 *mask = x | y; 6444 y = htobe64(y); 6445 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); 6446 } 6447 6448 static int 6449 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) 6450 { 6451 struct adapter *sc = arg1; 6452 struct sbuf *sb; 6453 int rc, i, n; 6454 6455 rc = sysctl_wire_old_buffer(req, 0); 6456 if (rc != 0) 6457 return (rc); 6458 6459 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6460 if (sb == NULL) 6461 return (ENOMEM); 6462 6463 sbuf_printf(sb, 6464 "Idx Ethernet address Mask Vld Ports PF" 6465 " VF Replication P0 P1 P2 P3 ML"); 6466 n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES : 6467 NUM_MPS_T5_CLS_SRAM_L_INSTANCES; 6468 for (i = 0; i < n; i++) { 6469 uint64_t tcamx, tcamy, mask; 6470 uint32_t cls_lo, cls_hi; 6471 uint8_t addr[ETHER_ADDR_LEN]; 6472 6473 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); 6474 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); 6475 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 6476 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 6477 6478 if (tcamx & tcamy) 6479 continue; 6480 6481 tcamxy2valmask(tcamx, tcamy, addr, &mask); 6482 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" 6483 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], 6484 addr[3], addr[4], addr[5], (uintmax_t)mask, 6485 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', 6486 G_PORTMAP(cls_hi), G_PF(cls_lo), 6487 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); 6488 6489 if (cls_lo & F_REPLICATE) { 6490 struct fw_ldst_cmd ldst_cmd; 6491 6492 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 6493 ldst_cmd.op_to_addrspace = 6494 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 6495 F_FW_CMD_REQUEST | F_FW_CMD_READ | 6496 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 6497 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 6498 ldst_cmd.u.mps.rplc.fid_idx = 6499 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 6500 V_FW_LDST_CMD_IDX(i)); 6501 6502 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 6503 "t4mps"); 6504 if (rc) 6505 break; 6506 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 6507 sizeof(ldst_cmd), &ldst_cmd); 6508 end_synchronized_op(sc, 0); 6509 6510 if (rc != 0) { 6511 sbuf_printf(sb, 6512 " ------------ error %3u ------------", rc); 6513 rc = 0; 6514 } else { 6515 sbuf_printf(sb, " %08x %08x %08x %08x", 6516 be32toh(ldst_cmd.u.mps.rplc.rplc127_96), 6517 be32toh(ldst_cmd.u.mps.rplc.rplc95_64), 6518 be32toh(ldst_cmd.u.mps.rplc.rplc63_32), 6519 be32toh(ldst_cmd.u.mps.rplc.rplc31_0)); 6520 } 6521 } else 6522 sbuf_printf(sb, "%36s", ""); 6523 6524 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), 6525 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), 6526 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); 6527 } 6528 6529 if (rc) 6530 (void) sbuf_finish(sb); 6531 else 6532 rc = sbuf_finish(sb); 6533 sbuf_delete(sb); 6534 6535 return (rc); 6536 } 6537 6538 static int 6539 sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 6540 { 6541 struct adapter *sc = arg1; 6542 struct sbuf *sb; 6543 int rc; 6544 uint16_t mtus[NMTUS]; 6545 6546 rc = sysctl_wire_old_buffer(req, 0); 6547 if (rc != 0) 6548 return (rc); 6549 6550 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6551 if (sb == NULL) 6552 return (ENOMEM); 6553 6554 t4_read_mtu_tbl(sc, mtus, NULL); 6555 6556 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 6557 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 6558 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 6559 mtus[14], mtus[15]); 6560 6561 rc = sbuf_finish(sb); 6562 sbuf_delete(sb); 6563 6564 return (rc); 6565 } 6566 6567 static int 6568 sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 6569 { 6570 struct adapter *sc = arg1; 6571 struct sbuf *sb; 6572 int rc, i; 6573 uint32_t cnt[PM_NSTATS]; 6574 uint64_t cyc[PM_NSTATS]; 6575 static const char *rx_stats[] = { 6576 "Read:", "Write bypass:", "Write mem:", "Flush:" 6577 }; 6578 static const char *tx_stats[] = { 6579 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:" 6580 }; 6581 6582 rc = sysctl_wire_old_buffer(req, 0); 6583 if (rc != 0) 6584 return (rc); 6585 6586 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6587 if (sb == NULL) 6588 return (ENOMEM); 6589 6590 t4_pmtx_get_stats(sc, cnt, cyc); 6591 sbuf_printf(sb, " Tx pcmds Tx bytes"); 6592 for (i = 0; i < ARRAY_SIZE(tx_stats); i++) 6593 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i], 6594 cyc[i]); 6595 6596 t4_pmrx_get_stats(sc, cnt, cyc); 6597 sbuf_printf(sb, "\n Rx pcmds Rx bytes"); 6598 for (i = 0; i < ARRAY_SIZE(rx_stats); i++) 6599 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i], 6600 cyc[i]); 6601 6602 rc = sbuf_finish(sb); 6603 sbuf_delete(sb); 6604 6605 return (rc); 6606 } 6607 6608 static int 6609 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 6610 { 6611 struct adapter *sc = arg1; 6612 struct sbuf *sb; 6613 int rc; 6614 struct tp_rdma_stats stats; 6615 6616 rc = sysctl_wire_old_buffer(req, 0); 6617 if (rc != 0) 6618 return (rc); 6619 6620 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6621 if (sb == NULL) 6622 return (ENOMEM); 6623 6624 t4_tp_get_rdma_stats(sc, &stats); 6625 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 6626 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 6627 6628 rc = sbuf_finish(sb); 6629 sbuf_delete(sb); 6630 6631 return (rc); 6632 } 6633 6634 static int 6635 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 6636 { 6637 struct adapter *sc = arg1; 6638 struct sbuf *sb; 6639 int rc; 6640 struct tp_tcp_stats v4, v6; 6641 6642 rc = sysctl_wire_old_buffer(req, 0); 6643 if (rc != 0) 6644 return (rc); 6645 6646 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6647 if (sb == NULL) 6648 return (ENOMEM); 6649 6650 t4_tp_get_tcp_stats(sc, &v4, &v6); 6651 sbuf_printf(sb, 6652 " IP IPv6\n"); 6653 sbuf_printf(sb, "OutRsts: %20u %20u\n", 6654 v4.tcpOutRsts, v6.tcpOutRsts); 6655 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 6656 v4.tcpInSegs, v6.tcpInSegs); 6657 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 6658 v4.tcpOutSegs, v6.tcpOutSegs); 6659 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 6660 v4.tcpRetransSegs, v6.tcpRetransSegs); 6661 6662 rc = sbuf_finish(sb); 6663 sbuf_delete(sb); 6664 6665 return (rc); 6666 } 6667 6668 static int 6669 sysctl_tids(SYSCTL_HANDLER_ARGS) 6670 { 6671 struct adapter *sc = arg1; 6672 struct sbuf *sb; 6673 int rc; 6674 struct tid_info *t = &sc->tids; 6675 6676 rc = sysctl_wire_old_buffer(req, 0); 6677 if (rc != 0) 6678 return (rc); 6679 6680 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6681 if (sb == NULL) 6682 return (ENOMEM); 6683 6684 if (t->natids) { 6685 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 6686 t->atids_in_use); 6687 } 6688 6689 if (t->ntids) { 6690 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6691 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 6692 6693 if (b) { 6694 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 6695 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6696 t->ntids - 1); 6697 } else { 6698 sbuf_printf(sb, "TID range: %u-%u", 6699 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6700 t->ntids - 1); 6701 } 6702 } else 6703 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 6704 sbuf_printf(sb, ", in use: %u\n", 6705 atomic_load_acq_int(&t->tids_in_use)); 6706 } 6707 6708 if (t->nstids) { 6709 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 6710 t->stid_base + t->nstids - 1, t->stids_in_use); 6711 } 6712 6713 if (t->nftids) { 6714 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 6715 t->ftid_base + t->nftids - 1); 6716 } 6717 6718 if (t->netids) { 6719 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base, 6720 t->etid_base + t->netids - 1); 6721 } 6722 6723 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 6724 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 6725 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 6726 6727 rc = sbuf_finish(sb); 6728 sbuf_delete(sb); 6729 6730 return (rc); 6731 } 6732 6733 static int 6734 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 6735 { 6736 struct adapter *sc = arg1; 6737 struct sbuf *sb; 6738 int rc; 6739 struct tp_err_stats stats; 6740 6741 rc = sysctl_wire_old_buffer(req, 0); 6742 if (rc != 0) 6743 return (rc); 6744 6745 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6746 if (sb == NULL) 6747 return (ENOMEM); 6748 6749 t4_tp_get_err_stats(sc, &stats); 6750 6751 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6752 "channel 3\n"); 6753 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 6754 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 6755 stats.macInErrs[3]); 6756 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 6757 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 6758 stats.hdrInErrs[3]); 6759 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 6760 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 6761 stats.tcpInErrs[3]); 6762 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 6763 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 6764 stats.tcp6InErrs[3]); 6765 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 6766 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 6767 stats.tnlCongDrops[3]); 6768 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 6769 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 6770 stats.tnlTxDrops[3]); 6771 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 6772 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 6773 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 6774 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 6775 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 6776 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 6777 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 6778 stats.ofldNoNeigh, stats.ofldCongDefer); 6779 6780 rc = sbuf_finish(sb); 6781 sbuf_delete(sb); 6782 6783 return (rc); 6784 } 6785 6786 struct field_desc { 6787 const char *name; 6788 u_int start; 6789 u_int width; 6790 }; 6791 6792 static void 6793 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) 6794 { 6795 char buf[32]; 6796 int line_size = 0; 6797 6798 while (f->name) { 6799 uint64_t mask = (1ULL << f->width) - 1; 6800 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, 6801 ((uintmax_t)v >> f->start) & mask); 6802 6803 if (line_size + len >= 79) { 6804 line_size = 8; 6805 sbuf_printf(sb, "\n "); 6806 } 6807 sbuf_printf(sb, "%s ", buf); 6808 line_size += len + 1; 6809 f++; 6810 } 6811 sbuf_printf(sb, "\n"); 6812 } 6813 6814 static struct field_desc tp_la0[] = { 6815 { "RcfOpCodeOut", 60, 4 }, 6816 { "State", 56, 4 }, 6817 { "WcfState", 52, 4 }, 6818 { "RcfOpcSrcOut", 50, 2 }, 6819 { "CRxError", 49, 1 }, 6820 { "ERxError", 48, 1 }, 6821 { "SanityFailed", 47, 1 }, 6822 { "SpuriousMsg", 46, 1 }, 6823 { "FlushInputMsg", 45, 1 }, 6824 { "FlushInputCpl", 44, 1 }, 6825 { "RssUpBit", 43, 1 }, 6826 { "RssFilterHit", 42, 1 }, 6827 { "Tid", 32, 10 }, 6828 { "InitTcb", 31, 1 }, 6829 { "LineNumber", 24, 7 }, 6830 { "Emsg", 23, 1 }, 6831 { "EdataOut", 22, 1 }, 6832 { "Cmsg", 21, 1 }, 6833 { "CdataOut", 20, 1 }, 6834 { "EreadPdu", 19, 1 }, 6835 { "CreadPdu", 18, 1 }, 6836 { "TunnelPkt", 17, 1 }, 6837 { "RcfPeerFin", 16, 1 }, 6838 { "RcfReasonOut", 12, 4 }, 6839 { "TxCchannel", 10, 2 }, 6840 { "RcfTxChannel", 8, 2 }, 6841 { "RxEchannel", 6, 2 }, 6842 { "RcfRxChannel", 5, 1 }, 6843 { "RcfDataOutSrdy", 4, 1 }, 6844 { "RxDvld", 3, 1 }, 6845 { "RxOoDvld", 2, 1 }, 6846 { "RxCongestion", 1, 1 }, 6847 { "TxCongestion", 0, 1 }, 6848 { NULL } 6849 }; 6850 6851 static struct field_desc tp_la1[] = { 6852 { "CplCmdIn", 56, 8 }, 6853 { "CplCmdOut", 48, 8 }, 6854 { "ESynOut", 47, 1 }, 6855 { "EAckOut", 46, 1 }, 6856 { "EFinOut", 45, 1 }, 6857 { "ERstOut", 44, 1 }, 6858 { "SynIn", 43, 1 }, 6859 { "AckIn", 42, 1 }, 6860 { "FinIn", 41, 1 }, 6861 { "RstIn", 40, 1 }, 6862 { "DataIn", 39, 1 }, 6863 { "DataInVld", 38, 1 }, 6864 { "PadIn", 37, 1 }, 6865 { "RxBufEmpty", 36, 1 }, 6866 { "RxDdp", 35, 1 }, 6867 { "RxFbCongestion", 34, 1 }, 6868 { "TxFbCongestion", 33, 1 }, 6869 { "TxPktSumSrdy", 32, 1 }, 6870 { "RcfUlpType", 28, 4 }, 6871 { "Eread", 27, 1 }, 6872 { "Ebypass", 26, 1 }, 6873 { "Esave", 25, 1 }, 6874 { "Static0", 24, 1 }, 6875 { "Cread", 23, 1 }, 6876 { "Cbypass", 22, 1 }, 6877 { "Csave", 21, 1 }, 6878 { "CPktOut", 20, 1 }, 6879 { "RxPagePoolFull", 18, 2 }, 6880 { "RxLpbkPkt", 17, 1 }, 6881 { "TxLpbkPkt", 16, 1 }, 6882 { "RxVfValid", 15, 1 }, 6883 { "SynLearned", 14, 1 }, 6884 { "SetDelEntry", 13, 1 }, 6885 { "SetInvEntry", 12, 1 }, 6886 { "CpcmdDvld", 11, 1 }, 6887 { "CpcmdSave", 10, 1 }, 6888 { "RxPstructsFull", 8, 2 }, 6889 { "EpcmdDvld", 7, 1 }, 6890 { "EpcmdFlush", 6, 1 }, 6891 { "EpcmdTrimPrefix", 5, 1 }, 6892 { "EpcmdTrimPostfix", 4, 1 }, 6893 { "ERssIp4Pkt", 3, 1 }, 6894 { "ERssIp6Pkt", 2, 1 }, 6895 { "ERssTcpUdpPkt", 1, 1 }, 6896 { "ERssFceFipPkt", 0, 1 }, 6897 { NULL } 6898 }; 6899 6900 static struct field_desc tp_la2[] = { 6901 { "CplCmdIn", 56, 8 }, 6902 { "MpsVfVld", 55, 1 }, 6903 { "MpsPf", 52, 3 }, 6904 { "MpsVf", 44, 8 }, 6905 { "SynIn", 43, 1 }, 6906 { "AckIn", 42, 1 }, 6907 { "FinIn", 41, 1 }, 6908 { "RstIn", 40, 1 }, 6909 { "DataIn", 39, 1 }, 6910 { "DataInVld", 38, 1 }, 6911 { "PadIn", 37, 1 }, 6912 { "RxBufEmpty", 36, 1 }, 6913 { "RxDdp", 35, 1 }, 6914 { "RxFbCongestion", 34, 1 }, 6915 { "TxFbCongestion", 33, 1 }, 6916 { "TxPktSumSrdy", 32, 1 }, 6917 { "RcfUlpType", 28, 4 }, 6918 { "Eread", 27, 1 }, 6919 { "Ebypass", 26, 1 }, 6920 { "Esave", 25, 1 }, 6921 { "Static0", 24, 1 }, 6922 { "Cread", 23, 1 }, 6923 { "Cbypass", 22, 1 }, 6924 { "Csave", 21, 1 }, 6925 { "CPktOut", 20, 1 }, 6926 { "RxPagePoolFull", 18, 2 }, 6927 { "RxLpbkPkt", 17, 1 }, 6928 { "TxLpbkPkt", 16, 1 }, 6929 { "RxVfValid", 15, 1 }, 6930 { "SynLearned", 14, 1 }, 6931 { "SetDelEntry", 13, 1 }, 6932 { "SetInvEntry", 12, 1 }, 6933 { "CpcmdDvld", 11, 1 }, 6934 { "CpcmdSave", 10, 1 }, 6935 { "RxPstructsFull", 8, 2 }, 6936 { "EpcmdDvld", 7, 1 }, 6937 { "EpcmdFlush", 6, 1 }, 6938 { "EpcmdTrimPrefix", 5, 1 }, 6939 { "EpcmdTrimPostfix", 4, 1 }, 6940 { "ERssIp4Pkt", 3, 1 }, 6941 { "ERssIp6Pkt", 2, 1 }, 6942 { "ERssTcpUdpPkt", 1, 1 }, 6943 { "ERssFceFipPkt", 0, 1 }, 6944 { NULL } 6945 }; 6946 6947 static void 6948 tp_la_show(struct sbuf *sb, uint64_t *p, int idx) 6949 { 6950 6951 field_desc_show(sb, *p, tp_la0); 6952 } 6953 6954 static void 6955 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) 6956 { 6957 6958 if (idx) 6959 sbuf_printf(sb, "\n"); 6960 field_desc_show(sb, p[0], tp_la0); 6961 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6962 field_desc_show(sb, p[1], tp_la0); 6963 } 6964 6965 static void 6966 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) 6967 { 6968 6969 if (idx) 6970 sbuf_printf(sb, "\n"); 6971 field_desc_show(sb, p[0], tp_la0); 6972 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6973 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); 6974 } 6975 6976 static int 6977 sysctl_tp_la(SYSCTL_HANDLER_ARGS) 6978 { 6979 struct adapter *sc = arg1; 6980 struct sbuf *sb; 6981 uint64_t *buf, *p; 6982 int rc; 6983 u_int i, inc; 6984 void (*show_func)(struct sbuf *, uint64_t *, int); 6985 6986 rc = sysctl_wire_old_buffer(req, 0); 6987 if (rc != 0) 6988 return (rc); 6989 6990 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6991 if (sb == NULL) 6992 return (ENOMEM); 6993 6994 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); 6995 6996 t4_tp_read_la(sc, buf, NULL); 6997 p = buf; 6998 6999 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { 7000 case 2: 7001 inc = 2; 7002 show_func = tp_la_show2; 7003 break; 7004 case 3: 7005 inc = 2; 7006 show_func = tp_la_show3; 7007 break; 7008 default: 7009 inc = 1; 7010 show_func = tp_la_show; 7011 } 7012 7013 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) 7014 (*show_func)(sb, p, i); 7015 7016 rc = sbuf_finish(sb); 7017 sbuf_delete(sb); 7018 free(buf, M_CXGBE); 7019 return (rc); 7020 } 7021 7022 static int 7023 sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 7024 { 7025 struct adapter *sc = arg1; 7026 struct sbuf *sb; 7027 int rc; 7028 u64 nrate[NCHAN], orate[NCHAN]; 7029 7030 rc = sysctl_wire_old_buffer(req, 0); 7031 if (rc != 0) 7032 return (rc); 7033 7034 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7035 if (sb == NULL) 7036 return (ENOMEM); 7037 7038 t4_get_chan_txrate(sc, nrate, orate); 7039 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 7040 "channel 3\n"); 7041 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 7042 nrate[0], nrate[1], nrate[2], nrate[3]); 7043 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 7044 orate[0], orate[1], orate[2], orate[3]); 7045 7046 rc = sbuf_finish(sb); 7047 sbuf_delete(sb); 7048 7049 return (rc); 7050 } 7051 7052 static int 7053 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) 7054 { 7055 struct adapter *sc = arg1; 7056 struct sbuf *sb; 7057 uint32_t *buf, *p; 7058 int rc, i; 7059 7060 rc = sysctl_wire_old_buffer(req, 0); 7061 if (rc != 0) 7062 return (rc); 7063 7064 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7065 if (sb == NULL) 7066 return (ENOMEM); 7067 7068 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, 7069 M_ZERO | M_WAITOK); 7070 7071 t4_ulprx_read_la(sc, buf); 7072 p = buf; 7073 7074 sbuf_printf(sb, " Pcmd Type Message" 7075 " Data"); 7076 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { 7077 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", 7078 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); 7079 } 7080 7081 rc = sbuf_finish(sb); 7082 sbuf_delete(sb); 7083 free(buf, M_CXGBE); 7084 return (rc); 7085 } 7086 7087 static int 7088 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 7089 { 7090 struct adapter *sc = arg1; 7091 struct sbuf *sb; 7092 int rc, v; 7093 7094 rc = sysctl_wire_old_buffer(req, 0); 7095 if (rc != 0) 7096 return (rc); 7097 7098 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7099 if (sb == NULL) 7100 return (ENOMEM); 7101 7102 v = t4_read_reg(sc, A_SGE_STAT_CFG); 7103 if (G_STATSOURCE_T5(v) == 7) { 7104 if (G_STATMODE(v) == 0) { 7105 sbuf_printf(sb, "total %d, incomplete %d", 7106 t4_read_reg(sc, A_SGE_STAT_TOTAL), 7107 t4_read_reg(sc, A_SGE_STAT_MATCH)); 7108 } else if (G_STATMODE(v) == 1) { 7109 sbuf_printf(sb, "total %d, data overflow %d", 7110 t4_read_reg(sc, A_SGE_STAT_TOTAL), 7111 t4_read_reg(sc, A_SGE_STAT_MATCH)); 7112 } 7113 } 7114 rc = sbuf_finish(sb); 7115 sbuf_delete(sb); 7116 7117 return (rc); 7118 } 7119 #endif 7120 7121 static uint32_t 7122 fconf_to_mode(uint32_t fconf) 7123 { 7124 uint32_t mode; 7125 7126 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 7127 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 7128 7129 if (fconf & F_FRAGMENTATION) 7130 mode |= T4_FILTER_IP_FRAGMENT; 7131 7132 if (fconf & F_MPSHITTYPE) 7133 mode |= T4_FILTER_MPS_HIT_TYPE; 7134 7135 if (fconf & F_MACMATCH) 7136 mode |= T4_FILTER_MAC_IDX; 7137 7138 if (fconf & F_ETHERTYPE) 7139 mode |= T4_FILTER_ETH_TYPE; 7140 7141 if (fconf & F_PROTOCOL) 7142 mode |= T4_FILTER_IP_PROTO; 7143 7144 if (fconf & F_TOS) 7145 mode |= T4_FILTER_IP_TOS; 7146 7147 if (fconf & F_VLAN) 7148 mode |= T4_FILTER_VLAN; 7149 7150 if (fconf & F_VNIC_ID) 7151 mode |= T4_FILTER_VNIC; 7152 7153 if (fconf & F_PORT) 7154 mode |= T4_FILTER_PORT; 7155 7156 if (fconf & F_FCOE) 7157 mode |= T4_FILTER_FCoE; 7158 7159 return (mode); 7160 } 7161 7162 static uint32_t 7163 mode_to_fconf(uint32_t mode) 7164 { 7165 uint32_t fconf = 0; 7166 7167 if (mode & T4_FILTER_IP_FRAGMENT) 7168 fconf |= F_FRAGMENTATION; 7169 7170 if (mode & T4_FILTER_MPS_HIT_TYPE) 7171 fconf |= F_MPSHITTYPE; 7172 7173 if (mode & T4_FILTER_MAC_IDX) 7174 fconf |= F_MACMATCH; 7175 7176 if (mode & T4_FILTER_ETH_TYPE) 7177 fconf |= F_ETHERTYPE; 7178 7179 if (mode & T4_FILTER_IP_PROTO) 7180 fconf |= F_PROTOCOL; 7181 7182 if (mode & T4_FILTER_IP_TOS) 7183 fconf |= F_TOS; 7184 7185 if (mode & T4_FILTER_VLAN) 7186 fconf |= F_VLAN; 7187 7188 if (mode & T4_FILTER_VNIC) 7189 fconf |= F_VNIC_ID; 7190 7191 if (mode & T4_FILTER_PORT) 7192 fconf |= F_PORT; 7193 7194 if (mode & T4_FILTER_FCoE) 7195 fconf |= F_FCOE; 7196 7197 return (fconf); 7198 } 7199 7200 static uint32_t 7201 fspec_to_fconf(struct t4_filter_specification *fs) 7202 { 7203 uint32_t fconf = 0; 7204 7205 if (fs->val.frag || fs->mask.frag) 7206 fconf |= F_FRAGMENTATION; 7207 7208 if (fs->val.matchtype || fs->mask.matchtype) 7209 fconf |= F_MPSHITTYPE; 7210 7211 if (fs->val.macidx || fs->mask.macidx) 7212 fconf |= F_MACMATCH; 7213 7214 if (fs->val.ethtype || fs->mask.ethtype) 7215 fconf |= F_ETHERTYPE; 7216 7217 if (fs->val.proto || fs->mask.proto) 7218 fconf |= F_PROTOCOL; 7219 7220 if (fs->val.tos || fs->mask.tos) 7221 fconf |= F_TOS; 7222 7223 if (fs->val.vlan_vld || fs->mask.vlan_vld) 7224 fconf |= F_VLAN; 7225 7226 if (fs->val.vnic_vld || fs->mask.vnic_vld) 7227 fconf |= F_VNIC_ID; 7228 7229 if (fs->val.iport || fs->mask.iport) 7230 fconf |= F_PORT; 7231 7232 if (fs->val.fcoe || fs->mask.fcoe) 7233 fconf |= F_FCOE; 7234 7235 return (fconf); 7236 } 7237 7238 static int 7239 get_filter_mode(struct adapter *sc, uint32_t *mode) 7240 { 7241 int rc; 7242 uint32_t fconf; 7243 7244 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7245 "t4getfm"); 7246 if (rc) 7247 return (rc); 7248 7249 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 7250 A_TP_VLAN_PRI_MAP); 7251 7252 if (sc->params.tp.vlan_pri_map != fconf) { 7253 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 7254 device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map, 7255 fconf); 7256 } 7257 7258 *mode = fconf_to_mode(fconf); 7259 7260 end_synchronized_op(sc, LOCK_HELD); 7261 return (0); 7262 } 7263 7264 static int 7265 set_filter_mode(struct adapter *sc, uint32_t mode) 7266 { 7267 uint32_t fconf; 7268 int rc; 7269 7270 fconf = mode_to_fconf(mode); 7271 7272 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7273 "t4setfm"); 7274 if (rc) 7275 return (rc); 7276 7277 if (sc->tids.ftids_in_use > 0) { 7278 rc = EBUSY; 7279 goto done; 7280 } 7281 7282 #ifdef TCP_OFFLOAD 7283 if (uld_active(sc, ULD_TOM)) { 7284 rc = EBUSY; 7285 goto done; 7286 } 7287 #endif 7288 7289 rc = -t4_set_filter_mode(sc, fconf); 7290 done: 7291 end_synchronized_op(sc, LOCK_HELD); 7292 return (rc); 7293 } 7294 7295 static inline uint64_t 7296 get_filter_hits(struct adapter *sc, uint32_t fid) 7297 { 7298 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 7299 uint64_t hits; 7300 7301 memwin_info(sc, 0, &mw_base, NULL); 7302 off = position_memwin(sc, 0, 7303 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 7304 if (is_t4(sc)) { 7305 hits = t4_read_reg64(sc, mw_base + off + 16); 7306 hits = be64toh(hits); 7307 } else { 7308 hits = t4_read_reg(sc, mw_base + off + 24); 7309 hits = be32toh(hits); 7310 } 7311 7312 return (hits); 7313 } 7314 7315 static int 7316 get_filter(struct adapter *sc, struct t4_filter *t) 7317 { 7318 int i, rc, nfilters = sc->tids.nftids; 7319 struct filter_entry *f; 7320 7321 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7322 "t4getf"); 7323 if (rc) 7324 return (rc); 7325 7326 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 7327 t->idx >= nfilters) { 7328 t->idx = 0xffffffff; 7329 goto done; 7330 } 7331 7332 f = &sc->tids.ftid_tab[t->idx]; 7333 for (i = t->idx; i < nfilters; i++, f++) { 7334 if (f->valid) { 7335 t->idx = i; 7336 t->l2tidx = f->l2t ? f->l2t->idx : 0; 7337 t->smtidx = f->smtidx; 7338 if (f->fs.hitcnts) 7339 t->hits = get_filter_hits(sc, t->idx); 7340 else 7341 t->hits = UINT64_MAX; 7342 t->fs = f->fs; 7343 7344 goto done; 7345 } 7346 } 7347 7348 t->idx = 0xffffffff; 7349 done: 7350 end_synchronized_op(sc, LOCK_HELD); 7351 return (0); 7352 } 7353 7354 static int 7355 set_filter(struct adapter *sc, struct t4_filter *t) 7356 { 7357 unsigned int nfilters, nports; 7358 struct filter_entry *f; 7359 int i, rc; 7360 7361 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 7362 if (rc) 7363 return (rc); 7364 7365 nfilters = sc->tids.nftids; 7366 nports = sc->params.nports; 7367 7368 if (nfilters == 0) { 7369 rc = ENOTSUP; 7370 goto done; 7371 } 7372 7373 if (!(sc->flags & FULL_INIT_DONE)) { 7374 rc = EAGAIN; 7375 goto done; 7376 } 7377 7378 if (t->idx >= nfilters) { 7379 rc = EINVAL; 7380 goto done; 7381 } 7382 7383 /* Validate against the global filter mode */ 7384 if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) != 7385 sc->params.tp.vlan_pri_map) { 7386 rc = E2BIG; 7387 goto done; 7388 } 7389 7390 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 7391 rc = EINVAL; 7392 goto done; 7393 } 7394 7395 if (t->fs.val.iport >= nports) { 7396 rc = EINVAL; 7397 goto done; 7398 } 7399 7400 /* Can't specify an iq if not steering to it */ 7401 if (!t->fs.dirsteer && t->fs.iq) { 7402 rc = EINVAL; 7403 goto done; 7404 } 7405 7406 /* IPv6 filter idx must be 4 aligned */ 7407 if (t->fs.type == 1 && 7408 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 7409 rc = EINVAL; 7410 goto done; 7411 } 7412 7413 if (sc->tids.ftid_tab == NULL) { 7414 KASSERT(sc->tids.ftids_in_use == 0, 7415 ("%s: no memory allocated but filters_in_use > 0", 7416 __func__)); 7417 7418 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 7419 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 7420 if (sc->tids.ftid_tab == NULL) { 7421 rc = ENOMEM; 7422 goto done; 7423 } 7424 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 7425 } 7426 7427 for (i = 0; i < 4; i++) { 7428 f = &sc->tids.ftid_tab[t->idx + i]; 7429 7430 if (f->pending || f->valid) { 7431 rc = EBUSY; 7432 goto done; 7433 } 7434 if (f->locked) { 7435 rc = EPERM; 7436 goto done; 7437 } 7438 7439 if (t->fs.type == 0) 7440 break; 7441 } 7442 7443 f = &sc->tids.ftid_tab[t->idx]; 7444 f->fs = t->fs; 7445 7446 rc = set_filter_wr(sc, t->idx); 7447 done: 7448 end_synchronized_op(sc, 0); 7449 7450 if (rc == 0) { 7451 mtx_lock(&sc->tids.ftid_lock); 7452 for (;;) { 7453 if (f->pending == 0) { 7454 rc = f->valid ? 0 : EIO; 7455 break; 7456 } 7457 7458 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7459 PCATCH, "t4setfw", 0)) { 7460 rc = EINPROGRESS; 7461 break; 7462 } 7463 } 7464 mtx_unlock(&sc->tids.ftid_lock); 7465 } 7466 return (rc); 7467 } 7468 7469 static int 7470 del_filter(struct adapter *sc, struct t4_filter *t) 7471 { 7472 unsigned int nfilters; 7473 struct filter_entry *f; 7474 int rc; 7475 7476 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 7477 if (rc) 7478 return (rc); 7479 7480 nfilters = sc->tids.nftids; 7481 7482 if (nfilters == 0) { 7483 rc = ENOTSUP; 7484 goto done; 7485 } 7486 7487 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 7488 t->idx >= nfilters) { 7489 rc = EINVAL; 7490 goto done; 7491 } 7492 7493 if (!(sc->flags & FULL_INIT_DONE)) { 7494 rc = EAGAIN; 7495 goto done; 7496 } 7497 7498 f = &sc->tids.ftid_tab[t->idx]; 7499 7500 if (f->pending) { 7501 rc = EBUSY; 7502 goto done; 7503 } 7504 if (f->locked) { 7505 rc = EPERM; 7506 goto done; 7507 } 7508 7509 if (f->valid) { 7510 t->fs = f->fs; /* extra info for the caller */ 7511 rc = del_filter_wr(sc, t->idx); 7512 } 7513 7514 done: 7515 end_synchronized_op(sc, 0); 7516 7517 if (rc == 0) { 7518 mtx_lock(&sc->tids.ftid_lock); 7519 for (;;) { 7520 if (f->pending == 0) { 7521 rc = f->valid ? EIO : 0; 7522 break; 7523 } 7524 7525 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7526 PCATCH, "t4delfw", 0)) { 7527 rc = EINPROGRESS; 7528 break; 7529 } 7530 } 7531 mtx_unlock(&sc->tids.ftid_lock); 7532 } 7533 7534 return (rc); 7535 } 7536 7537 static void 7538 clear_filter(struct filter_entry *f) 7539 { 7540 if (f->l2t) 7541 t4_l2t_release(f->l2t); 7542 7543 bzero(f, sizeof (*f)); 7544 } 7545 7546 static int 7547 set_filter_wr(struct adapter *sc, int fidx) 7548 { 7549 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7550 struct fw_filter_wr *fwr; 7551 unsigned int ftid; 7552 struct wrq_cookie cookie; 7553 7554 ASSERT_SYNCHRONIZED_OP(sc); 7555 7556 if (f->fs.newdmac || f->fs.newvlan) { 7557 /* This filter needs an L2T entry; allocate one. */ 7558 f->l2t = t4_l2t_alloc_switching(sc->l2t); 7559 if (f->l2t == NULL) 7560 return (EAGAIN); 7561 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 7562 f->fs.dmac)) { 7563 t4_l2t_release(f->l2t); 7564 f->l2t = NULL; 7565 return (ENOMEM); 7566 } 7567 } 7568 7569 ftid = sc->tids.ftid_base + fidx; 7570 7571 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 7572 if (fwr == NULL) 7573 return (ENOMEM); 7574 bzero(fwr, sizeof(*fwr)); 7575 7576 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 7577 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 7578 fwr->tid_to_iq = 7579 htobe32(V_FW_FILTER_WR_TID(ftid) | 7580 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 7581 V_FW_FILTER_WR_NOREPLY(0) | 7582 V_FW_FILTER_WR_IQ(f->fs.iq)); 7583 fwr->del_filter_to_l2tix = 7584 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 7585 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 7586 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 7587 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 7588 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 7589 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 7590 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 7591 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 7592 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 7593 f->fs.newvlan == VLAN_REWRITE) | 7594 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 7595 f->fs.newvlan == VLAN_REWRITE) | 7596 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 7597 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 7598 V_FW_FILTER_WR_PRIO(f->fs.prio) | 7599 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 7600 fwr->ethtype = htobe16(f->fs.val.ethtype); 7601 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 7602 fwr->frag_to_ovlan_vldm = 7603 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 7604 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 7605 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 7606 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 7607 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 7608 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 7609 fwr->smac_sel = 0; 7610 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 7611 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 7612 fwr->maci_to_matchtypem = 7613 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 7614 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 7615 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 7616 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 7617 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 7618 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 7619 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 7620 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 7621 fwr->ptcl = f->fs.val.proto; 7622 fwr->ptclm = f->fs.mask.proto; 7623 fwr->ttyp = f->fs.val.tos; 7624 fwr->ttypm = f->fs.mask.tos; 7625 fwr->ivlan = htobe16(f->fs.val.vlan); 7626 fwr->ivlanm = htobe16(f->fs.mask.vlan); 7627 fwr->ovlan = htobe16(f->fs.val.vnic); 7628 fwr->ovlanm = htobe16(f->fs.mask.vnic); 7629 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 7630 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 7631 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 7632 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 7633 fwr->lp = htobe16(f->fs.val.dport); 7634 fwr->lpm = htobe16(f->fs.mask.dport); 7635 fwr->fp = htobe16(f->fs.val.sport); 7636 fwr->fpm = htobe16(f->fs.mask.sport); 7637 if (f->fs.newsmac) 7638 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 7639 7640 f->pending = 1; 7641 sc->tids.ftids_in_use++; 7642 7643 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 7644 return (0); 7645 } 7646 7647 static int 7648 del_filter_wr(struct adapter *sc, int fidx) 7649 { 7650 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7651 struct fw_filter_wr *fwr; 7652 unsigned int ftid; 7653 struct wrq_cookie cookie; 7654 7655 ftid = sc->tids.ftid_base + fidx; 7656 7657 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 7658 if (fwr == NULL) 7659 return (ENOMEM); 7660 bzero(fwr, sizeof (*fwr)); 7661 7662 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 7663 7664 f->pending = 1; 7665 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 7666 return (0); 7667 } 7668 7669 int 7670 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 7671 { 7672 struct adapter *sc = iq->adapter; 7673 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 7674 unsigned int idx = GET_TID(rpl); 7675 unsigned int rc; 7676 struct filter_entry *f; 7677 7678 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 7679 rss->opcode)); 7680 7681 if (is_ftid(sc, idx)) { 7682 7683 idx -= sc->tids.ftid_base; 7684 f = &sc->tids.ftid_tab[idx]; 7685 rc = G_COOKIE(rpl->cookie); 7686 7687 mtx_lock(&sc->tids.ftid_lock); 7688 if (rc == FW_FILTER_WR_FLT_ADDED) { 7689 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 7690 __func__, idx)); 7691 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 7692 f->pending = 0; /* asynchronous setup completed */ 7693 f->valid = 1; 7694 } else { 7695 if (rc != FW_FILTER_WR_FLT_DELETED) { 7696 /* Add or delete failed, display an error */ 7697 log(LOG_ERR, 7698 "filter %u setup failed with error %u\n", 7699 idx, rc); 7700 } 7701 7702 clear_filter(f); 7703 sc->tids.ftids_in_use--; 7704 } 7705 wakeup(&sc->tids.ftid_tab); 7706 mtx_unlock(&sc->tids.ftid_lock); 7707 } 7708 7709 return (0); 7710 } 7711 7712 static int 7713 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 7714 { 7715 int rc; 7716 7717 if (cntxt->cid > M_CTXTQID) 7718 return (EINVAL); 7719 7720 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 7721 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 7722 return (EINVAL); 7723 7724 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 7725 if (rc) 7726 return (rc); 7727 7728 if (sc->flags & FW_OK) { 7729 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 7730 &cntxt->data[0]); 7731 if (rc == 0) 7732 goto done; 7733 } 7734 7735 /* 7736 * Read via firmware failed or wasn't even attempted. Read directly via 7737 * the backdoor. 7738 */ 7739 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 7740 done: 7741 end_synchronized_op(sc, 0); 7742 return (rc); 7743 } 7744 7745 static int 7746 load_fw(struct adapter *sc, struct t4_data *fw) 7747 { 7748 int rc; 7749 uint8_t *fw_data; 7750 7751 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 7752 if (rc) 7753 return (rc); 7754 7755 if (sc->flags & FULL_INIT_DONE) { 7756 rc = EBUSY; 7757 goto done; 7758 } 7759 7760 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 7761 if (fw_data == NULL) { 7762 rc = ENOMEM; 7763 goto done; 7764 } 7765 7766 rc = copyin(fw->data, fw_data, fw->len); 7767 if (rc == 0) 7768 rc = -t4_load_fw(sc, fw_data, fw->len); 7769 7770 free(fw_data, M_CXGBE); 7771 done: 7772 end_synchronized_op(sc, 0); 7773 return (rc); 7774 } 7775 7776 static int 7777 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 7778 { 7779 uint32_t addr, off, remaining, i, n; 7780 uint32_t *buf, *b; 7781 uint32_t mw_base, mw_aperture; 7782 int rc; 7783 uint8_t *dst; 7784 7785 rc = validate_mem_range(sc, mr->addr, mr->len); 7786 if (rc != 0) 7787 return (rc); 7788 7789 memwin_info(sc, win, &mw_base, &mw_aperture); 7790 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 7791 addr = mr->addr; 7792 remaining = mr->len; 7793 dst = (void *)mr->data; 7794 7795 while (remaining) { 7796 off = position_memwin(sc, win, addr); 7797 7798 /* number of bytes that we'll copy in the inner loop */ 7799 n = min(remaining, mw_aperture - off); 7800 for (i = 0; i < n; i += 4) 7801 *b++ = t4_read_reg(sc, mw_base + off + i); 7802 7803 rc = copyout(buf, dst, n); 7804 if (rc != 0) 7805 break; 7806 7807 b = buf; 7808 dst += n; 7809 remaining -= n; 7810 addr += n; 7811 } 7812 7813 free(buf, M_CXGBE); 7814 return (rc); 7815 } 7816 7817 static int 7818 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 7819 { 7820 int rc; 7821 7822 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 7823 return (EINVAL); 7824 7825 if (i2cd->len > sizeof(i2cd->data)) 7826 return (EFBIG); 7827 7828 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 7829 if (rc) 7830 return (rc); 7831 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 7832 i2cd->offset, i2cd->len, &i2cd->data[0]); 7833 end_synchronized_op(sc, 0); 7834 7835 return (rc); 7836 } 7837 7838 static int 7839 in_range(int val, int lo, int hi) 7840 { 7841 7842 return (val < 0 || (val <= hi && val >= lo)); 7843 } 7844 7845 static int 7846 set_sched_class(struct adapter *sc, struct t4_sched_params *p) 7847 { 7848 int fw_subcmd, fw_type, rc; 7849 7850 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc"); 7851 if (rc) 7852 return (rc); 7853 7854 if (!(sc->flags & FULL_INIT_DONE)) { 7855 rc = EAGAIN; 7856 goto done; 7857 } 7858 7859 /* 7860 * Translate the cxgbetool parameters into T4 firmware parameters. (The 7861 * sub-command and type are in common locations.) 7862 */ 7863 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG) 7864 fw_subcmd = FW_SCHED_SC_CONFIG; 7865 else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS) 7866 fw_subcmd = FW_SCHED_SC_PARAMS; 7867 else { 7868 rc = EINVAL; 7869 goto done; 7870 } 7871 if (p->type == SCHED_CLASS_TYPE_PACKET) 7872 fw_type = FW_SCHED_TYPE_PKTSCHED; 7873 else { 7874 rc = EINVAL; 7875 goto done; 7876 } 7877 7878 if (fw_subcmd == FW_SCHED_SC_CONFIG) { 7879 /* Vet our parameters ..*/ 7880 if (p->u.config.minmax < 0) { 7881 rc = EINVAL; 7882 goto done; 7883 } 7884 7885 /* And pass the request to the firmware ...*/ 7886 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1); 7887 goto done; 7888 } 7889 7890 if (fw_subcmd == FW_SCHED_SC_PARAMS) { 7891 int fw_level; 7892 int fw_mode; 7893 int fw_rateunit; 7894 int fw_ratemode; 7895 7896 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL) 7897 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL; 7898 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) 7899 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR; 7900 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) 7901 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL; 7902 else { 7903 rc = EINVAL; 7904 goto done; 7905 } 7906 7907 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS) 7908 fw_mode = FW_SCHED_PARAMS_MODE_CLASS; 7909 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW) 7910 fw_mode = FW_SCHED_PARAMS_MODE_FLOW; 7911 else { 7912 rc = EINVAL; 7913 goto done; 7914 } 7915 7916 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS) 7917 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 7918 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS) 7919 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE; 7920 else { 7921 rc = EINVAL; 7922 goto done; 7923 } 7924 7925 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL) 7926 fw_ratemode = FW_SCHED_PARAMS_RATE_REL; 7927 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS) 7928 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS; 7929 else { 7930 rc = EINVAL; 7931 goto done; 7932 } 7933 7934 /* Vet our parameters ... */ 7935 if (!in_range(p->u.params.channel, 0, 3) || 7936 !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) || 7937 !in_range(p->u.params.minrate, 0, 10000000) || 7938 !in_range(p->u.params.maxrate, 0, 10000000) || 7939 !in_range(p->u.params.weight, 0, 100)) { 7940 rc = ERANGE; 7941 goto done; 7942 } 7943 7944 /* 7945 * Translate any unset parameters into the firmware's 7946 * nomenclature and/or fail the call if the parameters 7947 * are required ... 7948 */ 7949 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 || 7950 p->u.params.channel < 0 || p->u.params.cl < 0) { 7951 rc = EINVAL; 7952 goto done; 7953 } 7954 if (p->u.params.minrate < 0) 7955 p->u.params.minrate = 0; 7956 if (p->u.params.maxrate < 0) { 7957 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7958 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7959 rc = EINVAL; 7960 goto done; 7961 } else 7962 p->u.params.maxrate = 0; 7963 } 7964 if (p->u.params.weight < 0) { 7965 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) { 7966 rc = EINVAL; 7967 goto done; 7968 } else 7969 p->u.params.weight = 0; 7970 } 7971 if (p->u.params.pktsize < 0) { 7972 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7973 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7974 rc = EINVAL; 7975 goto done; 7976 } else 7977 p->u.params.pktsize = 0; 7978 } 7979 7980 /* See what the firmware thinks of the request ... */ 7981 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode, 7982 fw_rateunit, fw_ratemode, p->u.params.channel, 7983 p->u.params.cl, p->u.params.minrate, p->u.params.maxrate, 7984 p->u.params.weight, p->u.params.pktsize, 1); 7985 goto done; 7986 } 7987 7988 rc = EINVAL; 7989 done: 7990 end_synchronized_op(sc, 0); 7991 return (rc); 7992 } 7993 7994 static int 7995 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p) 7996 { 7997 struct port_info *pi = NULL; 7998 struct sge_txq *txq; 7999 uint32_t fw_mnem, fw_queue, fw_class; 8000 int i, rc; 8001 8002 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq"); 8003 if (rc) 8004 return (rc); 8005 8006 if (!(sc->flags & FULL_INIT_DONE)) { 8007 rc = EAGAIN; 8008 goto done; 8009 } 8010 8011 if (p->port >= sc->params.nports) { 8012 rc = EINVAL; 8013 goto done; 8014 } 8015 8016 pi = sc->port[p->port]; 8017 if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) { 8018 rc = EINVAL; 8019 goto done; 8020 } 8021 8022 /* 8023 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX 8024 * Scheduling Class in this case). 8025 */ 8026 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 8027 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH)); 8028 fw_class = p->cl < 0 ? 0xffffffff : p->cl; 8029 8030 /* 8031 * If op.queue is non-negative, then we're only changing the scheduling 8032 * on a single specified TX queue. 8033 */ 8034 if (p->queue >= 0) { 8035 txq = &sc->sge.txq[pi->first_txq + p->queue]; 8036 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 8037 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 8038 &fw_class); 8039 goto done; 8040 } 8041 8042 /* 8043 * Change the scheduling on all the TX queues for the 8044 * interface. 8045 */ 8046 for_each_txq(pi, i, txq) { 8047 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 8048 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 8049 &fw_class); 8050 if (rc) 8051 goto done; 8052 } 8053 8054 rc = 0; 8055 done: 8056 end_synchronized_op(sc, 0); 8057 return (rc); 8058 } 8059 8060 int 8061 t4_os_find_pci_capability(struct adapter *sc, int cap) 8062 { 8063 int i; 8064 8065 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 8066 } 8067 8068 int 8069 t4_os_pci_save_state(struct adapter *sc) 8070 { 8071 device_t dev; 8072 struct pci_devinfo *dinfo; 8073 8074 dev = sc->dev; 8075 dinfo = device_get_ivars(dev); 8076 8077 pci_cfg_save(dev, dinfo, 0); 8078 return (0); 8079 } 8080 8081 int 8082 t4_os_pci_restore_state(struct adapter *sc) 8083 { 8084 device_t dev; 8085 struct pci_devinfo *dinfo; 8086 8087 dev = sc->dev; 8088 dinfo = device_get_ivars(dev); 8089 8090 pci_cfg_restore(dev, dinfo); 8091 return (0); 8092 } 8093 8094 void 8095 t4_os_portmod_changed(const struct adapter *sc, int idx) 8096 { 8097 struct port_info *pi = sc->port[idx]; 8098 static const char *mod_str[] = { 8099 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 8100 }; 8101 8102 build_medialist(pi, &pi->media); 8103 #ifdef DEV_NETMAP 8104 build_medialist(pi, &pi->nm_media); 8105 #endif 8106 8107 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 8108 if_printf(pi->ifp, "transceiver unplugged.\n"); 8109 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 8110 if_printf(pi->ifp, "unknown transceiver inserted.\n"); 8111 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 8112 if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 8113 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 8114 if_printf(pi->ifp, "%s transceiver inserted.\n", 8115 mod_str[pi->mod_type]); 8116 } else { 8117 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 8118 pi->mod_type); 8119 } 8120 } 8121 8122 void 8123 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason) 8124 { 8125 struct port_info *pi = sc->port[idx]; 8126 struct ifnet *ifp = pi->ifp; 8127 8128 if (link_stat) { 8129 pi->linkdnrc = -1; 8130 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 8131 if_link_state_change(ifp, LINK_STATE_UP); 8132 } else { 8133 if (reason >= 0) 8134 pi->linkdnrc = reason; 8135 if_link_state_change(ifp, LINK_STATE_DOWN); 8136 } 8137 } 8138 8139 void 8140 t4_iterate(void (*func)(struct adapter *, void *), void *arg) 8141 { 8142 struct adapter *sc; 8143 8144 sx_slock(&t4_list_lock); 8145 SLIST_FOREACH(sc, &t4_list, link) { 8146 /* 8147 * func should not make any assumptions about what state sc is 8148 * in - the only guarantee is that sc->sc_lock is a valid lock. 8149 */ 8150 func(sc, arg); 8151 } 8152 sx_sunlock(&t4_list_lock); 8153 } 8154 8155 static int 8156 t4_open(struct cdev *dev, int flags, int type, struct thread *td) 8157 { 8158 return (0); 8159 } 8160 8161 static int 8162 t4_close(struct cdev *dev, int flags, int type, struct thread *td) 8163 { 8164 return (0); 8165 } 8166 8167 static int 8168 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 8169 struct thread *td) 8170 { 8171 int rc; 8172 struct adapter *sc = dev->si_drv1; 8173 8174 rc = priv_check(td, PRIV_DRIVER); 8175 if (rc != 0) 8176 return (rc); 8177 8178 switch (cmd) { 8179 case CHELSIO_T4_GETREG: { 8180 struct t4_reg *edata = (struct t4_reg *)data; 8181 8182 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8183 return (EFAULT); 8184 8185 if (edata->size == 4) 8186 edata->val = t4_read_reg(sc, edata->addr); 8187 else if (edata->size == 8) 8188 edata->val = t4_read_reg64(sc, edata->addr); 8189 else 8190 return (EINVAL); 8191 8192 break; 8193 } 8194 case CHELSIO_T4_SETREG: { 8195 struct t4_reg *edata = (struct t4_reg *)data; 8196 8197 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8198 return (EFAULT); 8199 8200 if (edata->size == 4) { 8201 if (edata->val & 0xffffffff00000000) 8202 return (EINVAL); 8203 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 8204 } else if (edata->size == 8) 8205 t4_write_reg64(sc, edata->addr, edata->val); 8206 else 8207 return (EINVAL); 8208 break; 8209 } 8210 case CHELSIO_T4_REGDUMP: { 8211 struct t4_regdump *regs = (struct t4_regdump *)data; 8212 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 8213 uint8_t *buf; 8214 8215 if (regs->len < reglen) { 8216 regs->len = reglen; /* hint to the caller */ 8217 return (ENOBUFS); 8218 } 8219 8220 regs->len = reglen; 8221 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 8222 t4_get_regs(sc, regs, buf); 8223 rc = copyout(buf, regs->data, reglen); 8224 free(buf, M_CXGBE); 8225 break; 8226 } 8227 case CHELSIO_T4_GET_FILTER_MODE: 8228 rc = get_filter_mode(sc, (uint32_t *)data); 8229 break; 8230 case CHELSIO_T4_SET_FILTER_MODE: 8231 rc = set_filter_mode(sc, *(uint32_t *)data); 8232 break; 8233 case CHELSIO_T4_GET_FILTER: 8234 rc = get_filter(sc, (struct t4_filter *)data); 8235 break; 8236 case CHELSIO_T4_SET_FILTER: 8237 rc = set_filter(sc, (struct t4_filter *)data); 8238 break; 8239 case CHELSIO_T4_DEL_FILTER: 8240 rc = del_filter(sc, (struct t4_filter *)data); 8241 break; 8242 case CHELSIO_T4_GET_SGE_CONTEXT: 8243 rc = get_sge_context(sc, (struct t4_sge_context *)data); 8244 break; 8245 case CHELSIO_T4_LOAD_FW: 8246 rc = load_fw(sc, (struct t4_data *)data); 8247 break; 8248 case CHELSIO_T4_GET_MEM: 8249 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 8250 break; 8251 case CHELSIO_T4_GET_I2C: 8252 rc = read_i2c(sc, (struct t4_i2c_data *)data); 8253 break; 8254 case CHELSIO_T4_CLEAR_STATS: { 8255 int i; 8256 u_int port_id = *(uint32_t *)data; 8257 struct port_info *pi; 8258 8259 if (port_id >= sc->params.nports) 8260 return (EINVAL); 8261 pi = sc->port[port_id]; 8262 8263 /* MAC stats */ 8264 t4_clr_port_stats(sc, pi->tx_chan); 8265 pi->tx_parse_error = 0; 8266 8267 if (pi->flags & PORT_INIT_DONE) { 8268 struct sge_rxq *rxq; 8269 struct sge_txq *txq; 8270 struct sge_wrq *wrq; 8271 8272 for_each_rxq(pi, i, rxq) { 8273 #if defined(INET) || defined(INET6) 8274 rxq->lro.lro_queued = 0; 8275 rxq->lro.lro_flushed = 0; 8276 #endif 8277 rxq->rxcsum = 0; 8278 rxq->vlan_extraction = 0; 8279 } 8280 8281 for_each_txq(pi, i, txq) { 8282 txq->txcsum = 0; 8283 txq->tso_wrs = 0; 8284 txq->vlan_insertion = 0; 8285 txq->imm_wrs = 0; 8286 txq->sgl_wrs = 0; 8287 txq->txpkt_wrs = 0; 8288 txq->txpkts0_wrs = 0; 8289 txq->txpkts1_wrs = 0; 8290 txq->txpkts0_pkts = 0; 8291 txq->txpkts1_pkts = 0; 8292 mp_ring_reset_stats(txq->r); 8293 } 8294 8295 #ifdef TCP_OFFLOAD 8296 /* nothing to clear for each ofld_rxq */ 8297 8298 for_each_ofld_txq(pi, i, wrq) { 8299 wrq->tx_wrs_direct = 0; 8300 wrq->tx_wrs_copied = 0; 8301 } 8302 #endif 8303 wrq = &sc->sge.ctrlq[pi->port_id]; 8304 wrq->tx_wrs_direct = 0; 8305 wrq->tx_wrs_copied = 0; 8306 } 8307 break; 8308 } 8309 case CHELSIO_T4_SCHED_CLASS: 8310 rc = set_sched_class(sc, (struct t4_sched_params *)data); 8311 break; 8312 case CHELSIO_T4_SCHED_QUEUE: 8313 rc = set_sched_queue(sc, (struct t4_sched_queue *)data); 8314 break; 8315 case CHELSIO_T4_GET_TRACER: 8316 rc = t4_get_tracer(sc, (struct t4_tracer *)data); 8317 break; 8318 case CHELSIO_T4_SET_TRACER: 8319 rc = t4_set_tracer(sc, (struct t4_tracer *)data); 8320 break; 8321 default: 8322 rc = EINVAL; 8323 } 8324 8325 return (rc); 8326 } 8327 8328 #ifdef TCP_OFFLOAD 8329 void 8330 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask, 8331 const unsigned int *pgsz_order) 8332 { 8333 struct port_info *pi = ifp->if_softc; 8334 struct adapter *sc = pi->adapter; 8335 8336 t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask); 8337 t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) | 8338 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) | 8339 V_HPZ3(pgsz_order[3])); 8340 } 8341 8342 static int 8343 toe_capability(struct port_info *pi, int enable) 8344 { 8345 int rc; 8346 struct adapter *sc = pi->adapter; 8347 8348 ASSERT_SYNCHRONIZED_OP(sc); 8349 8350 if (!is_offload(sc)) 8351 return (ENODEV); 8352 8353 if (enable) { 8354 /* 8355 * We need the port's queues around so that we're able to send 8356 * and receive CPLs to/from the TOE even if the ifnet for this 8357 * port has never been UP'd administratively. 8358 */ 8359 if (!(pi->flags & PORT_INIT_DONE)) { 8360 rc = cxgbe_init_synchronized(pi); 8361 if (rc) 8362 return (rc); 8363 } 8364 8365 if (isset(&sc->offload_map, pi->port_id)) 8366 return (0); 8367 8368 if (!uld_active(sc, ULD_TOM)) { 8369 rc = t4_activate_uld(sc, ULD_TOM); 8370 if (rc == EAGAIN) { 8371 log(LOG_WARNING, 8372 "You must kldload t4_tom.ko before trying " 8373 "to enable TOE on a cxgbe interface.\n"); 8374 } 8375 if (rc != 0) 8376 return (rc); 8377 KASSERT(sc->tom_softc != NULL, 8378 ("%s: TOM activated but softc NULL", __func__)); 8379 KASSERT(uld_active(sc, ULD_TOM), 8380 ("%s: TOM activated but flag not set", __func__)); 8381 } 8382 8383 /* Activate iWARP and iSCSI too, if the modules are loaded. */ 8384 if (!uld_active(sc, ULD_IWARP)) 8385 (void) t4_activate_uld(sc, ULD_IWARP); 8386 if (!uld_active(sc, ULD_ISCSI)) 8387 (void) t4_activate_uld(sc, ULD_ISCSI); 8388 8389 setbit(&sc->offload_map, pi->port_id); 8390 } else { 8391 if (!isset(&sc->offload_map, pi->port_id)) 8392 return (0); 8393 8394 KASSERT(uld_active(sc, ULD_TOM), 8395 ("%s: TOM never initialized?", __func__)); 8396 clrbit(&sc->offload_map, pi->port_id); 8397 } 8398 8399 return (0); 8400 } 8401 8402 /* 8403 * Add an upper layer driver to the global list. 8404 */ 8405 int 8406 t4_register_uld(struct uld_info *ui) 8407 { 8408 int rc = 0; 8409 struct uld_info *u; 8410 8411 sx_xlock(&t4_uld_list_lock); 8412 SLIST_FOREACH(u, &t4_uld_list, link) { 8413 if (u->uld_id == ui->uld_id) { 8414 rc = EEXIST; 8415 goto done; 8416 } 8417 } 8418 8419 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 8420 ui->refcount = 0; 8421 done: 8422 sx_xunlock(&t4_uld_list_lock); 8423 return (rc); 8424 } 8425 8426 int 8427 t4_unregister_uld(struct uld_info *ui) 8428 { 8429 int rc = EINVAL; 8430 struct uld_info *u; 8431 8432 sx_xlock(&t4_uld_list_lock); 8433 8434 SLIST_FOREACH(u, &t4_uld_list, link) { 8435 if (u == ui) { 8436 if (ui->refcount > 0) { 8437 rc = EBUSY; 8438 goto done; 8439 } 8440 8441 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 8442 rc = 0; 8443 goto done; 8444 } 8445 } 8446 done: 8447 sx_xunlock(&t4_uld_list_lock); 8448 return (rc); 8449 } 8450 8451 int 8452 t4_activate_uld(struct adapter *sc, int id) 8453 { 8454 int rc; 8455 struct uld_info *ui; 8456 8457 ASSERT_SYNCHRONIZED_OP(sc); 8458 8459 if (id < 0 || id > ULD_MAX) 8460 return (EINVAL); 8461 rc = EAGAIN; /* kldoad the module with this ULD and try again. */ 8462 8463 sx_slock(&t4_uld_list_lock); 8464 8465 SLIST_FOREACH(ui, &t4_uld_list, link) { 8466 if (ui->uld_id == id) { 8467 if (!(sc->flags & FULL_INIT_DONE)) { 8468 rc = adapter_full_init(sc); 8469 if (rc != 0) 8470 break; 8471 } 8472 8473 rc = ui->activate(sc); 8474 if (rc == 0) { 8475 setbit(&sc->active_ulds, id); 8476 ui->refcount++; 8477 } 8478 break; 8479 } 8480 } 8481 8482 sx_sunlock(&t4_uld_list_lock); 8483 8484 return (rc); 8485 } 8486 8487 int 8488 t4_deactivate_uld(struct adapter *sc, int id) 8489 { 8490 int rc; 8491 struct uld_info *ui; 8492 8493 ASSERT_SYNCHRONIZED_OP(sc); 8494 8495 if (id < 0 || id > ULD_MAX) 8496 return (EINVAL); 8497 rc = ENXIO; 8498 8499 sx_slock(&t4_uld_list_lock); 8500 8501 SLIST_FOREACH(ui, &t4_uld_list, link) { 8502 if (ui->uld_id == id) { 8503 rc = ui->deactivate(sc); 8504 if (rc == 0) { 8505 clrbit(&sc->active_ulds, id); 8506 ui->refcount--; 8507 } 8508 break; 8509 } 8510 } 8511 8512 sx_sunlock(&t4_uld_list_lock); 8513 8514 return (rc); 8515 } 8516 8517 int 8518 uld_active(struct adapter *sc, int uld_id) 8519 { 8520 8521 MPASS(uld_id >= 0 && uld_id <= ULD_MAX); 8522 8523 return (isset(&sc->active_ulds, uld_id)); 8524 } 8525 #endif 8526 8527 /* 8528 * Come up with reasonable defaults for some of the tunables, provided they're 8529 * not set by the user (in which case we'll use the values as is). 8530 */ 8531 static void 8532 tweak_tunables(void) 8533 { 8534 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 8535 8536 if (t4_ntxq10g < 1) { 8537 #ifdef RSS 8538 t4_ntxq10g = rss_getnumbuckets(); 8539 #else 8540 t4_ntxq10g = min(nc, NTXQ_10G); 8541 #endif 8542 } 8543 8544 if (t4_ntxq1g < 1) { 8545 #ifdef RSS 8546 /* XXX: way too many for 1GbE? */ 8547 t4_ntxq1g = rss_getnumbuckets(); 8548 #else 8549 t4_ntxq1g = min(nc, NTXQ_1G); 8550 #endif 8551 } 8552 8553 if (t4_nrxq10g < 1) { 8554 #ifdef RSS 8555 t4_nrxq10g = rss_getnumbuckets(); 8556 #else 8557 t4_nrxq10g = min(nc, NRXQ_10G); 8558 #endif 8559 } 8560 8561 if (t4_nrxq1g < 1) { 8562 #ifdef RSS 8563 /* XXX: way too many for 1GbE? */ 8564 t4_nrxq1g = rss_getnumbuckets(); 8565 #else 8566 t4_nrxq1g = min(nc, NRXQ_1G); 8567 #endif 8568 } 8569 8570 #ifdef TCP_OFFLOAD 8571 if (t4_nofldtxq10g < 1) 8572 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 8573 8574 if (t4_nofldtxq1g < 1) 8575 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 8576 8577 if (t4_nofldrxq10g < 1) 8578 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 8579 8580 if (t4_nofldrxq1g < 1) 8581 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 8582 8583 if (t4_toecaps_allowed == -1) 8584 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 8585 #else 8586 if (t4_toecaps_allowed == -1) 8587 t4_toecaps_allowed = 0; 8588 #endif 8589 8590 #ifdef DEV_NETMAP 8591 if (t4_nnmtxq10g < 1) 8592 t4_nnmtxq10g = min(nc, NNMTXQ_10G); 8593 8594 if (t4_nnmtxq1g < 1) 8595 t4_nnmtxq1g = min(nc, NNMTXQ_1G); 8596 8597 if (t4_nnmrxq10g < 1) 8598 t4_nnmrxq10g = min(nc, NNMRXQ_10G); 8599 8600 if (t4_nnmrxq1g < 1) 8601 t4_nnmrxq1g = min(nc, NNMRXQ_1G); 8602 #endif 8603 8604 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 8605 t4_tmr_idx_10g = TMR_IDX_10G; 8606 8607 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 8608 t4_pktc_idx_10g = PKTC_IDX_10G; 8609 8610 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 8611 t4_tmr_idx_1g = TMR_IDX_1G; 8612 8613 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 8614 t4_pktc_idx_1g = PKTC_IDX_1G; 8615 8616 if (t4_qsize_txq < 128) 8617 t4_qsize_txq = 128; 8618 8619 if (t4_qsize_rxq < 128) 8620 t4_qsize_rxq = 128; 8621 while (t4_qsize_rxq & 7) 8622 t4_qsize_rxq++; 8623 8624 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 8625 } 8626 8627 static struct sx mlu; /* mod load unload */ 8628 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload"); 8629 8630 static int 8631 mod_event(module_t mod, int cmd, void *arg) 8632 { 8633 int rc = 0; 8634 static int loaded = 0; 8635 8636 switch (cmd) { 8637 case MOD_LOAD: 8638 sx_xlock(&mlu); 8639 if (loaded++ == 0) { 8640 t4_sge_modload(); 8641 sx_init(&t4_list_lock, "T4/T5 adapters"); 8642 SLIST_INIT(&t4_list); 8643 #ifdef TCP_OFFLOAD 8644 sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); 8645 SLIST_INIT(&t4_uld_list); 8646 #endif 8647 t4_tracer_modload(); 8648 tweak_tunables(); 8649 } 8650 sx_xunlock(&mlu); 8651 break; 8652 8653 case MOD_UNLOAD: 8654 sx_xlock(&mlu); 8655 if (--loaded == 0) { 8656 int tries; 8657 8658 sx_slock(&t4_list_lock); 8659 if (!SLIST_EMPTY(&t4_list)) { 8660 rc = EBUSY; 8661 sx_sunlock(&t4_list_lock); 8662 goto done_unload; 8663 } 8664 #ifdef TCP_OFFLOAD 8665 sx_slock(&t4_uld_list_lock); 8666 if (!SLIST_EMPTY(&t4_uld_list)) { 8667 rc = EBUSY; 8668 sx_sunlock(&t4_uld_list_lock); 8669 sx_sunlock(&t4_list_lock); 8670 goto done_unload; 8671 } 8672 #endif 8673 tries = 0; 8674 while (tries++ < 5 && t4_sge_extfree_refs() != 0) { 8675 uprintf("%ju clusters with custom free routine " 8676 "still is use.\n", t4_sge_extfree_refs()); 8677 pause("t4unload", 2 * hz); 8678 } 8679 #ifdef TCP_OFFLOAD 8680 sx_sunlock(&t4_uld_list_lock); 8681 #endif 8682 sx_sunlock(&t4_list_lock); 8683 8684 if (t4_sge_extfree_refs() == 0) { 8685 t4_tracer_modunload(); 8686 #ifdef TCP_OFFLOAD 8687 sx_destroy(&t4_uld_list_lock); 8688 #endif 8689 sx_destroy(&t4_list_lock); 8690 t4_sge_modunload(); 8691 loaded = 0; 8692 } else { 8693 rc = EBUSY; 8694 loaded++; /* undo earlier decrement */ 8695 } 8696 } 8697 done_unload: 8698 sx_xunlock(&mlu); 8699 break; 8700 } 8701 8702 return (rc); 8703 } 8704 8705 static devclass_t t4_devclass, t5_devclass; 8706 static devclass_t cxgbe_devclass, cxl_devclass; 8707 8708 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 8709 MODULE_VERSION(t4nex, 1); 8710 MODULE_DEPEND(t4nex, firmware, 1, 1, 1); 8711 #ifdef DEV_NETMAP 8712 MODULE_DEPEND(t4nex, netmap, 1, 1, 1); 8713 #endif /* DEV_NETMAP */ 8714 8715 8716 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 8717 MODULE_VERSION(t5nex, 1); 8718 MODULE_DEPEND(t5nex, firmware, 1, 1, 1); 8719 #ifdef DEV_NETMAP 8720 MODULE_DEPEND(t5nex, netmap, 1, 1, 1); 8721 #endif /* DEV_NETMAP */ 8722 8723 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 8724 MODULE_VERSION(cxgbe, 1); 8725 8726 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 8727 MODULE_VERSION(cxl, 1); 8728