1 /*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_inet.h" 32 #include "opt_inet6.h" 33 #include "opt_rss.h" 34 35 #include <sys/param.h> 36 #include <sys/conf.h> 37 #include <sys/priv.h> 38 #include <sys/kernel.h> 39 #include <sys/bus.h> 40 #include <sys/module.h> 41 #include <sys/malloc.h> 42 #include <sys/queue.h> 43 #include <sys/taskqueue.h> 44 #include <sys/pciio.h> 45 #include <dev/pci/pcireg.h> 46 #include <dev/pci/pcivar.h> 47 #include <dev/pci/pci_private.h> 48 #include <sys/firmware.h> 49 #include <sys/sbuf.h> 50 #include <sys/smp.h> 51 #include <sys/socket.h> 52 #include <sys/sockio.h> 53 #include <sys/sysctl.h> 54 #include <net/ethernet.h> 55 #include <net/if.h> 56 #include <net/if_types.h> 57 #include <net/if_dl.h> 58 #include <net/if_vlan_var.h> 59 #ifdef RSS 60 #include <net/rss_config.h> 61 #endif 62 #if defined(__i386__) || defined(__amd64__) 63 #include <vm/vm.h> 64 #include <vm/pmap.h> 65 #endif 66 67 #include "common/common.h" 68 #include "common/t4_msg.h" 69 #include "common/t4_regs.h" 70 #include "common/t4_regs_values.h" 71 #include "t4_ioctl.h" 72 #include "t4_l2t.h" 73 #include "t4_mp_ring.h" 74 75 /* T4 bus driver interface */ 76 static int t4_probe(device_t); 77 static int t4_attach(device_t); 78 static int t4_detach(device_t); 79 static device_method_t t4_methods[] = { 80 DEVMETHOD(device_probe, t4_probe), 81 DEVMETHOD(device_attach, t4_attach), 82 DEVMETHOD(device_detach, t4_detach), 83 84 DEVMETHOD_END 85 }; 86 static driver_t t4_driver = { 87 "t4nex", 88 t4_methods, 89 sizeof(struct adapter) 90 }; 91 92 93 /* T4 port (cxgbe) interface */ 94 static int cxgbe_probe(device_t); 95 static int cxgbe_attach(device_t); 96 static int cxgbe_detach(device_t); 97 static device_method_t cxgbe_methods[] = { 98 DEVMETHOD(device_probe, cxgbe_probe), 99 DEVMETHOD(device_attach, cxgbe_attach), 100 DEVMETHOD(device_detach, cxgbe_detach), 101 { 0, 0 } 102 }; 103 static driver_t cxgbe_driver = { 104 "cxgbe", 105 cxgbe_methods, 106 sizeof(struct port_info) 107 }; 108 109 static d_ioctl_t t4_ioctl; 110 static d_open_t t4_open; 111 static d_close_t t4_close; 112 113 static struct cdevsw t4_cdevsw = { 114 .d_version = D_VERSION, 115 .d_flags = 0, 116 .d_open = t4_open, 117 .d_close = t4_close, 118 .d_ioctl = t4_ioctl, 119 .d_name = "t4nex", 120 }; 121 122 /* T5 bus driver interface */ 123 static int t5_probe(device_t); 124 static device_method_t t5_methods[] = { 125 DEVMETHOD(device_probe, t5_probe), 126 DEVMETHOD(device_attach, t4_attach), 127 DEVMETHOD(device_detach, t4_detach), 128 129 DEVMETHOD_END 130 }; 131 static driver_t t5_driver = { 132 "t5nex", 133 t5_methods, 134 sizeof(struct adapter) 135 }; 136 137 138 /* T5 port (cxl) interface */ 139 static driver_t cxl_driver = { 140 "cxl", 141 cxgbe_methods, 142 sizeof(struct port_info) 143 }; 144 145 static struct cdevsw t5_cdevsw = { 146 .d_version = D_VERSION, 147 .d_flags = 0, 148 .d_open = t4_open, 149 .d_close = t4_close, 150 .d_ioctl = t4_ioctl, 151 .d_name = "t5nex", 152 }; 153 154 /* ifnet + media interface */ 155 static void cxgbe_init(void *); 156 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 157 static int cxgbe_transmit(struct ifnet *, struct mbuf *); 158 static void cxgbe_qflush(struct ifnet *); 159 static uint64_t cxgbe_get_counter(struct ifnet *, ift_counter); 160 static int cxgbe_media_change(struct ifnet *); 161 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 162 163 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 164 165 /* 166 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 167 * then ADAPTER_LOCK, then t4_uld_list_lock. 168 */ 169 static struct sx t4_list_lock; 170 SLIST_HEAD(, adapter) t4_list; 171 #ifdef TCP_OFFLOAD 172 static struct sx t4_uld_list_lock; 173 SLIST_HEAD(, uld_info) t4_uld_list; 174 #endif 175 176 /* 177 * Tunables. See tweak_tunables() too. 178 * 179 * Each tunable is set to a default value here if it's known at compile-time. 180 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 181 * provide a reasonable default when the driver is loaded. 182 * 183 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 184 * T5 are under hw.cxl. 185 */ 186 187 /* 188 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 189 */ 190 #define NTXQ_10G 16 191 static int t4_ntxq10g = -1; 192 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 193 194 #define NRXQ_10G 8 195 static int t4_nrxq10g = -1; 196 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 197 198 #define NTXQ_1G 4 199 static int t4_ntxq1g = -1; 200 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 201 202 #define NRXQ_1G 2 203 static int t4_nrxq1g = -1; 204 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 205 206 static int t4_rsrv_noflowq = 0; 207 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq); 208 209 #ifdef TCP_OFFLOAD 210 #define NOFLDTXQ_10G 8 211 static int t4_nofldtxq10g = -1; 212 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 213 214 #define NOFLDRXQ_10G 2 215 static int t4_nofldrxq10g = -1; 216 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 217 218 #define NOFLDTXQ_1G 2 219 static int t4_nofldtxq1g = -1; 220 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 221 222 #define NOFLDRXQ_1G 1 223 static int t4_nofldrxq1g = -1; 224 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 225 #endif 226 227 #ifdef DEV_NETMAP 228 #define NNMTXQ_10G 2 229 static int t4_nnmtxq10g = -1; 230 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g); 231 232 #define NNMRXQ_10G 2 233 static int t4_nnmrxq10g = -1; 234 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g); 235 236 #define NNMTXQ_1G 1 237 static int t4_nnmtxq1g = -1; 238 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g); 239 240 #define NNMRXQ_1G 1 241 static int t4_nnmrxq1g = -1; 242 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g); 243 #endif 244 245 /* 246 * Holdoff parameters for 10G and 1G ports. 247 */ 248 #define TMR_IDX_10G 1 249 static int t4_tmr_idx_10g = TMR_IDX_10G; 250 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 251 252 #define PKTC_IDX_10G (-1) 253 static int t4_pktc_idx_10g = PKTC_IDX_10G; 254 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 255 256 #define TMR_IDX_1G 1 257 static int t4_tmr_idx_1g = TMR_IDX_1G; 258 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 259 260 #define PKTC_IDX_1G (-1) 261 static int t4_pktc_idx_1g = PKTC_IDX_1G; 262 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 263 264 /* 265 * Size (# of entries) of each tx and rx queue. 266 */ 267 static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 268 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 269 270 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 271 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 272 273 /* 274 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 275 */ 276 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 277 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 278 279 /* 280 * Configuration file. 281 */ 282 #define DEFAULT_CF "default" 283 #define FLASH_CF "flash" 284 #define UWIRE_CF "uwire" 285 #define FPGA_CF "fpga" 286 static char t4_cfg_file[32] = DEFAULT_CF; 287 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 288 289 /* 290 * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively). 291 * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them. 292 * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water 293 * mark or when signalled to do so, 0 to never emit PAUSE. 294 */ 295 static int t4_pause_settings = PAUSE_TX | PAUSE_RX; 296 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings); 297 298 /* 299 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 300 * encouraged respectively). 301 */ 302 static unsigned int t4_fw_install = 1; 303 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 304 305 /* 306 * ASIC features that will be used. Disable the ones you don't want so that the 307 * chip resources aren't wasted on features that will not be used. 308 */ 309 static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 310 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 311 312 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 313 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 314 315 static int t4_toecaps_allowed = -1; 316 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 317 318 static int t4_rdmacaps_allowed = 0; 319 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 320 321 static int t4_iscsicaps_allowed = 0; 322 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 323 324 static int t4_fcoecaps_allowed = 0; 325 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 326 327 static int t5_write_combine = 0; 328 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 329 330 struct intrs_and_queues { 331 uint16_t intr_type; /* INTx, MSI, or MSI-X */ 332 uint16_t nirq; /* Total # of vectors */ 333 uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */ 334 uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */ 335 uint16_t ntxq10g; /* # of NIC txq's for each 10G port */ 336 uint16_t nrxq10g; /* # of NIC rxq's for each 10G port */ 337 uint16_t ntxq1g; /* # of NIC txq's for each 1G port */ 338 uint16_t nrxq1g; /* # of NIC rxq's for each 1G port */ 339 uint16_t rsrv_noflowq; /* Flag whether to reserve queue 0 */ 340 #ifdef TCP_OFFLOAD 341 uint16_t nofldtxq10g; /* # of TOE txq's for each 10G port */ 342 uint16_t nofldrxq10g; /* # of TOE rxq's for each 10G port */ 343 uint16_t nofldtxq1g; /* # of TOE txq's for each 1G port */ 344 uint16_t nofldrxq1g; /* # of TOE rxq's for each 1G port */ 345 #endif 346 #ifdef DEV_NETMAP 347 uint16_t nnmtxq10g; /* # of netmap txq's for each 10G port */ 348 uint16_t nnmrxq10g; /* # of netmap rxq's for each 10G port */ 349 uint16_t nnmtxq1g; /* # of netmap txq's for each 1G port */ 350 uint16_t nnmrxq1g; /* # of netmap rxq's for each 1G port */ 351 #endif 352 }; 353 354 struct filter_entry { 355 uint32_t valid:1; /* filter allocated and valid */ 356 uint32_t locked:1; /* filter is administratively locked */ 357 uint32_t pending:1; /* filter action is pending firmware reply */ 358 uint32_t smtidx:8; /* Source MAC Table index for smac */ 359 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 360 361 struct t4_filter_specification fs; 362 }; 363 364 static int map_bars_0_and_4(struct adapter *); 365 static int map_bar_2(struct adapter *); 366 static void setup_memwin(struct adapter *); 367 static int validate_mem_range(struct adapter *, uint32_t, int); 368 static int fwmtype_to_hwmtype(int); 369 static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 370 uint32_t *); 371 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 372 static uint32_t position_memwin(struct adapter *, int, uint32_t); 373 static int cfg_itype_and_nqueues(struct adapter *, int, int, 374 struct intrs_and_queues *); 375 static int prep_firmware(struct adapter *); 376 static int partition_resources(struct adapter *, const struct firmware *, 377 const char *); 378 static int get_params__pre_init(struct adapter *); 379 static int get_params__post_init(struct adapter *); 380 static int set_params__post_init(struct adapter *); 381 static void t4_set_desc(struct adapter *); 382 static void build_medialist(struct port_info *, struct ifmedia *); 383 static int cxgbe_init_synchronized(struct port_info *); 384 static int cxgbe_uninit_synchronized(struct port_info *); 385 static int setup_intr_handlers(struct adapter *); 386 static void quiesce_txq(struct adapter *, struct sge_txq *); 387 static void quiesce_wrq(struct adapter *, struct sge_wrq *); 388 static void quiesce_iq(struct adapter *, struct sge_iq *); 389 static void quiesce_fl(struct adapter *, struct sge_fl *); 390 static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 391 driver_intr_t *, void *, char *); 392 static int t4_free_irq(struct adapter *, struct irq *); 393 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 394 unsigned int); 395 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 396 static void cxgbe_refresh_stats(struct adapter *, struct port_info *); 397 static void cxgbe_tick(void *); 398 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 399 static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 400 struct mbuf *); 401 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 402 static int fw_msg_not_handled(struct adapter *, const __be64 *); 403 static int t4_sysctls(struct adapter *); 404 static int cxgbe_sysctls(struct port_info *); 405 static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 406 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 407 static int sysctl_btphy(SYSCTL_HANDLER_ARGS); 408 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); 409 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 410 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 411 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 412 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 413 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS); 414 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 415 static int sysctl_temperature(SYSCTL_HANDLER_ARGS); 416 #ifdef SBUF_DRAIN 417 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 418 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 419 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 420 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); 421 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); 422 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 423 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 424 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 425 static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 426 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 427 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 428 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 429 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); 430 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 431 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); 432 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 433 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 434 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 435 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 436 static int sysctl_tids(SYSCTL_HANDLER_ARGS); 437 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 438 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 439 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 440 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 441 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 442 #endif 443 static uint32_t fconf_to_mode(uint32_t); 444 static uint32_t mode_to_fconf(uint32_t); 445 static uint32_t fspec_to_fconf(struct t4_filter_specification *); 446 static int get_filter_mode(struct adapter *, uint32_t *); 447 static int set_filter_mode(struct adapter *, uint32_t); 448 static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 449 static int get_filter(struct adapter *, struct t4_filter *); 450 static int set_filter(struct adapter *, struct t4_filter *); 451 static int del_filter(struct adapter *, struct t4_filter *); 452 static void clear_filter(struct filter_entry *); 453 static int set_filter_wr(struct adapter *, int); 454 static int del_filter_wr(struct adapter *, int); 455 static int get_sge_context(struct adapter *, struct t4_sge_context *); 456 static int load_fw(struct adapter *, struct t4_data *); 457 static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 458 static int read_i2c(struct adapter *, struct t4_i2c_data *); 459 static int set_sched_class(struct adapter *, struct t4_sched_params *); 460 static int set_sched_queue(struct adapter *, struct t4_sched_queue *); 461 #ifdef TCP_OFFLOAD 462 static int toe_capability(struct port_info *, int); 463 #endif 464 static int mod_event(module_t, int, void *); 465 466 struct { 467 uint16_t device; 468 char *desc; 469 } t4_pciids[] = { 470 {0xa000, "Chelsio Terminator 4 FPGA"}, 471 {0x4400, "Chelsio T440-dbg"}, 472 {0x4401, "Chelsio T420-CR"}, 473 {0x4402, "Chelsio T422-CR"}, 474 {0x4403, "Chelsio T440-CR"}, 475 {0x4404, "Chelsio T420-BCH"}, 476 {0x4405, "Chelsio T440-BCH"}, 477 {0x4406, "Chelsio T440-CH"}, 478 {0x4407, "Chelsio T420-SO"}, 479 {0x4408, "Chelsio T420-CX"}, 480 {0x4409, "Chelsio T420-BT"}, 481 {0x440a, "Chelsio T404-BT"}, 482 {0x440e, "Chelsio T440-LP-CR"}, 483 }, t5_pciids[] = { 484 {0xb000, "Chelsio Terminator 5 FPGA"}, 485 {0x5400, "Chelsio T580-dbg"}, 486 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ 487 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ 488 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ 489 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ 490 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ 491 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ 492 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ 493 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ 494 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 495 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ 496 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ 497 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ 498 {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */ 499 #ifdef notyet 500 {0x5404, "Chelsio T520-BCH"}, 501 {0x5405, "Chelsio T540-BCH"}, 502 {0x5406, "Chelsio T540-CH"}, 503 {0x5408, "Chelsio T520-CX"}, 504 {0x540b, "Chelsio B520-SR"}, 505 {0x540c, "Chelsio B504-BT"}, 506 {0x540f, "Chelsio Amsterdam"}, 507 {0x5413, "Chelsio T580-CHR"}, 508 #endif 509 }; 510 511 #ifdef TCP_OFFLOAD 512 /* 513 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 514 * exactly the same for both rxq and ofld_rxq. 515 */ 516 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 517 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 518 #endif 519 520 /* No easy way to include t4_msg.h before adapter.h so we check this way */ 521 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 522 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 523 524 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); 525 526 static int 527 t4_probe(device_t dev) 528 { 529 int i; 530 uint16_t v = pci_get_vendor(dev); 531 uint16_t d = pci_get_device(dev); 532 uint8_t f = pci_get_function(dev); 533 534 if (v != PCI_VENDOR_ID_CHELSIO) 535 return (ENXIO); 536 537 /* Attach only to PF0 of the FPGA */ 538 if (d == 0xa000 && f != 0) 539 return (ENXIO); 540 541 for (i = 0; i < nitems(t4_pciids); i++) { 542 if (d == t4_pciids[i].device) { 543 device_set_desc(dev, t4_pciids[i].desc); 544 return (BUS_PROBE_DEFAULT); 545 } 546 } 547 548 return (ENXIO); 549 } 550 551 static int 552 t5_probe(device_t dev) 553 { 554 int i; 555 uint16_t v = pci_get_vendor(dev); 556 uint16_t d = pci_get_device(dev); 557 uint8_t f = pci_get_function(dev); 558 559 if (v != PCI_VENDOR_ID_CHELSIO) 560 return (ENXIO); 561 562 /* Attach only to PF0 of the FPGA */ 563 if (d == 0xb000 && f != 0) 564 return (ENXIO); 565 566 for (i = 0; i < nitems(t5_pciids); i++) { 567 if (d == t5_pciids[i].device) { 568 device_set_desc(dev, t5_pciids[i].desc); 569 return (BUS_PROBE_DEFAULT); 570 } 571 } 572 573 return (ENXIO); 574 } 575 576 static void 577 t5_attribute_workaround(device_t dev) 578 { 579 device_t root_port; 580 uint32_t v; 581 582 /* 583 * The T5 chips do not properly echo the No Snoop and Relaxed 584 * Ordering attributes when replying to a TLP from a Root 585 * Port. As a workaround, find the parent Root Port and 586 * disable No Snoop and Relaxed Ordering. Note that this 587 * affects all devices under this root port. 588 */ 589 root_port = pci_find_pcie_root_port(dev); 590 if (root_port == NULL) { 591 device_printf(dev, "Unable to find parent root port\n"); 592 return; 593 } 594 595 v = pcie_adjust_config(root_port, PCIER_DEVICE_CTL, 596 PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE, 0, 2); 597 if ((v & (PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE)) != 598 0) 599 device_printf(dev, "Disabled No Snoop/Relaxed Ordering on %s\n", 600 device_get_nameunit(root_port)); 601 } 602 603 static int 604 t4_attach(device_t dev) 605 { 606 struct adapter *sc; 607 int rc = 0, i, n10g, n1g, rqidx, tqidx; 608 struct intrs_and_queues iaq; 609 struct sge *s; 610 #ifdef TCP_OFFLOAD 611 int ofld_rqidx, ofld_tqidx; 612 #endif 613 #ifdef DEV_NETMAP 614 int nm_rqidx, nm_tqidx; 615 #endif 616 617 sc = device_get_softc(dev); 618 sc->dev = dev; 619 TUNABLE_INT_FETCH("hw.cxgbe.debug_flags", &sc->debug_flags); 620 621 if ((pci_get_device(dev) & 0xff00) == 0x5400) 622 t5_attribute_workaround(dev); 623 pci_enable_busmaster(dev); 624 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 625 uint32_t v; 626 627 pci_set_max_read_req(dev, 4096); 628 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 629 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 630 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 631 632 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5); 633 } 634 635 sc->traceq = -1; 636 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); 637 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", 638 device_get_nameunit(dev)); 639 640 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 641 device_get_nameunit(dev)); 642 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 643 sx_xlock(&t4_list_lock); 644 SLIST_INSERT_HEAD(&t4_list, sc, link); 645 sx_xunlock(&t4_list_lock); 646 647 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 648 TAILQ_INIT(&sc->sfl); 649 callout_init(&sc->sfl_callout, 1); 650 651 mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF); 652 653 rc = map_bars_0_and_4(sc); 654 if (rc != 0) 655 goto done; /* error message displayed already */ 656 657 /* 658 * This is the real PF# to which we're attaching. Works from within PCI 659 * passthrough environments too, where pci_get_function() could return a 660 * different PF# depending on the passthrough configuration. We need to 661 * use the real PF# in all our communication with the firmware. 662 */ 663 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 664 sc->mbox = sc->pf; 665 666 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 667 sc->an_handler = an_not_handled; 668 for (i = 0; i < nitems(sc->cpl_handler); i++) 669 sc->cpl_handler[i] = cpl_not_handled; 670 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 671 sc->fw_msg_handler[i] = fw_msg_not_handled; 672 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 673 t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt); 674 t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt); 675 t4_init_sge_cpl_handlers(sc); 676 677 /* Prepare the adapter for operation */ 678 rc = -t4_prep_adapter(sc); 679 if (rc != 0) { 680 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 681 goto done; 682 } 683 684 /* 685 * Do this really early, with the memory windows set up even before the 686 * character device. The userland tool's register i/o and mem read 687 * will work even in "recovery mode". 688 */ 689 setup_memwin(sc); 690 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 691 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 692 device_get_nameunit(dev)); 693 if (sc->cdev == NULL) 694 device_printf(dev, "failed to create nexus char device.\n"); 695 else 696 sc->cdev->si_drv1 = sc; 697 698 /* Go no further if recovery mode has been requested. */ 699 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 700 device_printf(dev, "recovery mode.\n"); 701 goto done; 702 } 703 704 #if defined(__i386__) 705 if ((cpu_feature & CPUID_CX8) == 0) { 706 device_printf(dev, "64 bit atomics not available.\n"); 707 rc = ENOTSUP; 708 goto done; 709 } 710 #endif 711 712 /* Prepare the firmware for operation */ 713 rc = prep_firmware(sc); 714 if (rc != 0) 715 goto done; /* error message displayed already */ 716 717 rc = get_params__post_init(sc); 718 if (rc != 0) 719 goto done; /* error message displayed already */ 720 721 rc = set_params__post_init(sc); 722 if (rc != 0) 723 goto done; /* error message displayed already */ 724 725 rc = map_bar_2(sc); 726 if (rc != 0) 727 goto done; /* error message displayed already */ 728 729 rc = t4_create_dma_tag(sc); 730 if (rc != 0) 731 goto done; /* error message displayed already */ 732 733 /* 734 * First pass over all the ports - allocate VIs and initialize some 735 * basic parameters like mac address, port type, etc. We also figure 736 * out whether a port is 10G or 1G and use that information when 737 * calculating how many interrupts to attempt to allocate. 738 */ 739 n10g = n1g = 0; 740 for_each_port(sc, i) { 741 struct port_info *pi; 742 743 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 744 sc->port[i] = pi; 745 746 /* These must be set before t4_port_init */ 747 pi->adapter = sc; 748 pi->port_id = i; 749 750 /* Allocate the vi and initialize parameters like mac addr */ 751 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 752 if (rc != 0) { 753 device_printf(dev, "unable to initialize port %d: %d\n", 754 i, rc); 755 free(pi, M_CXGBE); 756 sc->port[i] = NULL; 757 goto done; 758 } 759 760 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX); 761 pi->link_cfg.requested_fc |= t4_pause_settings; 762 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX); 763 pi->link_cfg.fc |= t4_pause_settings; 764 765 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 766 if (rc != 0) { 767 device_printf(dev, "port %d l1cfg failed: %d\n", i, rc); 768 free(pi, M_CXGBE); 769 sc->port[i] = NULL; 770 goto done; 771 } 772 773 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 774 device_get_nameunit(dev), i); 775 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 776 sc->chan_map[pi->tx_chan] = i; 777 778 if (is_10G_port(pi) || is_40G_port(pi)) { 779 n10g++; 780 pi->tmr_idx = t4_tmr_idx_10g; 781 pi->pktc_idx = t4_pktc_idx_10g; 782 } else { 783 n1g++; 784 pi->tmr_idx = t4_tmr_idx_1g; 785 pi->pktc_idx = t4_pktc_idx_1g; 786 } 787 788 pi->xact_addr_filt = -1; 789 pi->linkdnrc = -1; 790 791 pi->qsize_rxq = t4_qsize_rxq; 792 pi->qsize_txq = t4_qsize_txq; 793 794 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 795 if (pi->dev == NULL) { 796 device_printf(dev, 797 "failed to add device for port %d.\n", i); 798 rc = ENXIO; 799 goto done; 800 } 801 device_set_softc(pi->dev, pi); 802 } 803 804 /* 805 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 806 */ 807 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 808 if (rc != 0) 809 goto done; /* error message displayed already */ 810 811 sc->intr_type = iaq.intr_type; 812 sc->intr_count = iaq.nirq; 813 814 s = &sc->sge; 815 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 816 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 817 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 818 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 819 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 820 #ifdef TCP_OFFLOAD 821 if (is_offload(sc)) { 822 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 823 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 824 s->neq += s->nofldtxq + s->nofldrxq; 825 s->niq += s->nofldrxq; 826 827 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 828 M_CXGBE, M_ZERO | M_WAITOK); 829 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 830 M_CXGBE, M_ZERO | M_WAITOK); 831 } 832 #endif 833 #ifdef DEV_NETMAP 834 s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g; 835 s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g; 836 s->neq += s->nnmtxq + s->nnmrxq; 837 s->niq += s->nnmrxq; 838 839 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq), 840 M_CXGBE, M_ZERO | M_WAITOK); 841 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq), 842 M_CXGBE, M_ZERO | M_WAITOK); 843 #endif 844 845 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 846 M_ZERO | M_WAITOK); 847 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 848 M_ZERO | M_WAITOK); 849 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 850 M_ZERO | M_WAITOK); 851 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 852 M_ZERO | M_WAITOK); 853 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 854 M_ZERO | M_WAITOK); 855 856 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 857 M_ZERO | M_WAITOK); 858 859 t4_init_l2t(sc, M_WAITOK); 860 861 /* 862 * Second pass over the ports. This time we know the number of rx and 863 * tx queues that each port should get. 864 */ 865 rqidx = tqidx = 0; 866 #ifdef TCP_OFFLOAD 867 ofld_rqidx = ofld_tqidx = 0; 868 #endif 869 #ifdef DEV_NETMAP 870 nm_rqidx = nm_tqidx = 0; 871 #endif 872 for_each_port(sc, i) { 873 struct port_info *pi = sc->port[i]; 874 875 if (pi == NULL) 876 continue; 877 878 pi->first_rxq = rqidx; 879 pi->first_txq = tqidx; 880 if (is_10G_port(pi) || is_40G_port(pi)) { 881 pi->flags |= iaq.intr_flags_10g; 882 pi->nrxq = iaq.nrxq10g; 883 pi->ntxq = iaq.ntxq10g; 884 } else { 885 pi->flags |= iaq.intr_flags_1g; 886 pi->nrxq = iaq.nrxq1g; 887 pi->ntxq = iaq.ntxq1g; 888 } 889 890 if (pi->ntxq > 1) 891 pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0; 892 else 893 pi->rsrv_noflowq = 0; 894 895 rqidx += pi->nrxq; 896 tqidx += pi->ntxq; 897 #ifdef TCP_OFFLOAD 898 if (is_offload(sc)) { 899 pi->first_ofld_rxq = ofld_rqidx; 900 pi->first_ofld_txq = ofld_tqidx; 901 if (is_10G_port(pi) || is_40G_port(pi)) { 902 pi->nofldrxq = iaq.nofldrxq10g; 903 pi->nofldtxq = iaq.nofldtxq10g; 904 } else { 905 pi->nofldrxq = iaq.nofldrxq1g; 906 pi->nofldtxq = iaq.nofldtxq1g; 907 } 908 ofld_rqidx += pi->nofldrxq; 909 ofld_tqidx += pi->nofldtxq; 910 } 911 #endif 912 #ifdef DEV_NETMAP 913 pi->first_nm_rxq = nm_rqidx; 914 pi->first_nm_txq = nm_tqidx; 915 if (is_10G_port(pi) || is_40G_port(pi)) { 916 pi->nnmrxq = iaq.nnmrxq10g; 917 pi->nnmtxq = iaq.nnmtxq10g; 918 } else { 919 pi->nnmrxq = iaq.nnmrxq1g; 920 pi->nnmtxq = iaq.nnmtxq1g; 921 } 922 nm_rqidx += pi->nnmrxq; 923 nm_tqidx += pi->nnmtxq; 924 #endif 925 } 926 927 rc = setup_intr_handlers(sc); 928 if (rc != 0) { 929 device_printf(dev, 930 "failed to setup interrupt handlers: %d\n", rc); 931 goto done; 932 } 933 934 rc = bus_generic_attach(dev); 935 if (rc != 0) { 936 device_printf(dev, 937 "failed to attach all child ports: %d\n", rc); 938 goto done; 939 } 940 941 device_printf(dev, 942 "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 943 sc->params.pci.speed, sc->params.pci.width, sc->params.nports, 944 sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" : 945 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 946 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 947 948 t4_set_desc(sc); 949 950 done: 951 if (rc != 0 && sc->cdev) { 952 /* cdev was created and so cxgbetool works; recover that way. */ 953 device_printf(dev, 954 "error during attach, adapter is now in recovery mode.\n"); 955 rc = 0; 956 } 957 958 if (rc != 0) 959 t4_detach(dev); 960 else 961 t4_sysctls(sc); 962 963 return (rc); 964 } 965 966 /* 967 * Idempotent 968 */ 969 static int 970 t4_detach(device_t dev) 971 { 972 struct adapter *sc; 973 struct port_info *pi; 974 int i, rc; 975 976 sc = device_get_softc(dev); 977 978 if (sc->flags & FULL_INIT_DONE) 979 t4_intr_disable(sc); 980 981 if (sc->cdev) { 982 destroy_dev(sc->cdev); 983 sc->cdev = NULL; 984 } 985 986 rc = bus_generic_detach(dev); 987 if (rc) { 988 device_printf(dev, 989 "failed to detach child devices: %d\n", rc); 990 return (rc); 991 } 992 993 for (i = 0; i < sc->intr_count; i++) 994 t4_free_irq(sc, &sc->irq[i]); 995 996 for (i = 0; i < MAX_NPORTS; i++) { 997 pi = sc->port[i]; 998 if (pi) { 999 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid); 1000 if (pi->dev) 1001 device_delete_child(dev, pi->dev); 1002 1003 mtx_destroy(&pi->pi_lock); 1004 free(pi, M_CXGBE); 1005 } 1006 } 1007 1008 if (sc->flags & FULL_INIT_DONE) 1009 adapter_full_uninit(sc); 1010 1011 if (sc->flags & FW_OK) 1012 t4_fw_bye(sc, sc->mbox); 1013 1014 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 1015 pci_release_msi(dev); 1016 1017 if (sc->regs_res) 1018 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 1019 sc->regs_res); 1020 1021 if (sc->udbs_res) 1022 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 1023 sc->udbs_res); 1024 1025 if (sc->msix_res) 1026 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 1027 sc->msix_res); 1028 1029 if (sc->l2t) 1030 t4_free_l2t(sc->l2t); 1031 1032 #ifdef TCP_OFFLOAD 1033 free(sc->sge.ofld_rxq, M_CXGBE); 1034 free(sc->sge.ofld_txq, M_CXGBE); 1035 #endif 1036 #ifdef DEV_NETMAP 1037 free(sc->sge.nm_rxq, M_CXGBE); 1038 free(sc->sge.nm_txq, M_CXGBE); 1039 #endif 1040 free(sc->irq, M_CXGBE); 1041 free(sc->sge.rxq, M_CXGBE); 1042 free(sc->sge.txq, M_CXGBE); 1043 free(sc->sge.ctrlq, M_CXGBE); 1044 free(sc->sge.iqmap, M_CXGBE); 1045 free(sc->sge.eqmap, M_CXGBE); 1046 free(sc->tids.ftid_tab, M_CXGBE); 1047 t4_destroy_dma_tag(sc); 1048 if (mtx_initialized(&sc->sc_lock)) { 1049 sx_xlock(&t4_list_lock); 1050 SLIST_REMOVE(&t4_list, sc, adapter, link); 1051 sx_xunlock(&t4_list_lock); 1052 mtx_destroy(&sc->sc_lock); 1053 } 1054 1055 if (mtx_initialized(&sc->tids.ftid_lock)) 1056 mtx_destroy(&sc->tids.ftid_lock); 1057 if (mtx_initialized(&sc->sfl_lock)) 1058 mtx_destroy(&sc->sfl_lock); 1059 if (mtx_initialized(&sc->ifp_lock)) 1060 mtx_destroy(&sc->ifp_lock); 1061 if (mtx_initialized(&sc->regwin_lock)) 1062 mtx_destroy(&sc->regwin_lock); 1063 1064 bzero(sc, sizeof(*sc)); 1065 1066 return (0); 1067 } 1068 1069 static int 1070 cxgbe_probe(device_t dev) 1071 { 1072 char buf[128]; 1073 struct port_info *pi = device_get_softc(dev); 1074 1075 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 1076 device_set_desc_copy(dev, buf); 1077 1078 return (BUS_PROBE_DEFAULT); 1079 } 1080 1081 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 1082 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 1083 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS) 1084 #define T4_CAP_ENABLE (T4_CAP) 1085 1086 static int 1087 cxgbe_attach(device_t dev) 1088 { 1089 struct port_info *pi = device_get_softc(dev); 1090 struct ifnet *ifp; 1091 char *s; 1092 int n, o; 1093 1094 /* Allocate an ifnet and set it up */ 1095 ifp = if_alloc(IFT_ETHER); 1096 if (ifp == NULL) { 1097 device_printf(dev, "Cannot allocate ifnet\n"); 1098 return (ENOMEM); 1099 } 1100 pi->ifp = ifp; 1101 ifp->if_softc = pi; 1102 1103 callout_init(&pi->tick, 1); 1104 1105 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1106 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1107 1108 ifp->if_init = cxgbe_init; 1109 ifp->if_ioctl = cxgbe_ioctl; 1110 ifp->if_transmit = cxgbe_transmit; 1111 ifp->if_qflush = cxgbe_qflush; 1112 ifp->if_get_counter = cxgbe_get_counter; 1113 1114 ifp->if_capabilities = T4_CAP; 1115 #ifdef TCP_OFFLOAD 1116 if (is_offload(pi->adapter)) 1117 ifp->if_capabilities |= IFCAP_TOE; 1118 #endif 1119 ifp->if_capenable = T4_CAP_ENABLE; 1120 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1121 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1122 1123 ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1124 ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS; 1125 ifp->if_hw_tsomaxsegsize = 65536; 1126 1127 /* Initialize ifmedia for this port */ 1128 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 1129 cxgbe_media_status); 1130 build_medialist(pi, &pi->media); 1131 1132 pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 1133 EVENTHANDLER_PRI_ANY); 1134 1135 ether_ifattach(ifp, pi->hw_addr); 1136 1137 n = 128; 1138 s = malloc(n, M_CXGBE, M_WAITOK); 1139 o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq); 1140 MPASS(n > o); 1141 #ifdef TCP_OFFLOAD 1142 if (is_offload(pi->adapter)) { 1143 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)", 1144 pi->nofldtxq, pi->nofldrxq); 1145 MPASS(n > o); 1146 } 1147 #endif 1148 #ifdef DEV_NETMAP 1149 o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq, 1150 pi->nnmrxq); 1151 MPASS(n > o); 1152 #endif 1153 device_printf(dev, "%s\n", s); 1154 free(s, M_CXGBE); 1155 1156 #ifdef DEV_NETMAP 1157 /* nm_media handled here to keep implementation private to this file */ 1158 ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change, 1159 cxgbe_media_status); 1160 build_medialist(pi, &pi->nm_media); 1161 create_netmap_ifnet(pi); /* logs errors it something fails */ 1162 #endif 1163 cxgbe_sysctls(pi); 1164 1165 return (0); 1166 } 1167 1168 static int 1169 cxgbe_detach(device_t dev) 1170 { 1171 struct port_info *pi = device_get_softc(dev); 1172 struct adapter *sc = pi->adapter; 1173 struct ifnet *ifp = pi->ifp; 1174 1175 /* Tell if_ioctl and if_init that the port is going away */ 1176 ADAPTER_LOCK(sc); 1177 SET_DOOMED(pi); 1178 wakeup(&sc->flags); 1179 while (IS_BUSY(sc)) 1180 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 1181 SET_BUSY(sc); 1182 #ifdef INVARIANTS 1183 sc->last_op = "t4detach"; 1184 sc->last_op_thr = curthread; 1185 sc->last_op_flags = 0; 1186 #endif 1187 ADAPTER_UNLOCK(sc); 1188 1189 if (pi->flags & HAS_TRACEQ) { 1190 sc->traceq = -1; /* cloner should not create ifnet */ 1191 t4_tracer_port_detach(sc); 1192 } 1193 1194 if (pi->vlan_c) 1195 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c); 1196 1197 PORT_LOCK(pi); 1198 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1199 callout_stop(&pi->tick); 1200 PORT_UNLOCK(pi); 1201 callout_drain(&pi->tick); 1202 1203 /* Let detach proceed even if these fail. */ 1204 cxgbe_uninit_synchronized(pi); 1205 port_full_uninit(pi); 1206 1207 ifmedia_removeall(&pi->media); 1208 ether_ifdetach(pi->ifp); 1209 if_free(pi->ifp); 1210 1211 #ifdef DEV_NETMAP 1212 /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */ 1213 destroy_netmap_ifnet(pi); 1214 #endif 1215 1216 ADAPTER_LOCK(sc); 1217 CLR_BUSY(sc); 1218 wakeup(&sc->flags); 1219 ADAPTER_UNLOCK(sc); 1220 1221 return (0); 1222 } 1223 1224 static void 1225 cxgbe_init(void *arg) 1226 { 1227 struct port_info *pi = arg; 1228 struct adapter *sc = pi->adapter; 1229 1230 if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0) 1231 return; 1232 cxgbe_init_synchronized(pi); 1233 end_synchronized_op(sc, 0); 1234 } 1235 1236 static int 1237 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1238 { 1239 int rc = 0, mtu, flags, can_sleep; 1240 struct port_info *pi = ifp->if_softc; 1241 struct adapter *sc = pi->adapter; 1242 struct ifreq *ifr = (struct ifreq *)data; 1243 uint32_t mask; 1244 1245 switch (cmd) { 1246 case SIOCSIFMTU: 1247 mtu = ifr->ifr_mtu; 1248 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1249 return (EINVAL); 1250 1251 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu"); 1252 if (rc) 1253 return (rc); 1254 ifp->if_mtu = mtu; 1255 if (pi->flags & PORT_INIT_DONE) { 1256 t4_update_fl_bufsize(ifp); 1257 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1258 rc = update_mac_settings(ifp, XGMAC_MTU); 1259 } 1260 end_synchronized_op(sc, 0); 1261 break; 1262 1263 case SIOCSIFFLAGS: 1264 can_sleep = 0; 1265 redo_sifflags: 1266 rc = begin_synchronized_op(sc, pi, 1267 can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg"); 1268 if (rc) 1269 return (rc); 1270 1271 if (ifp->if_flags & IFF_UP) { 1272 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1273 flags = pi->if_flags; 1274 if ((ifp->if_flags ^ flags) & 1275 (IFF_PROMISC | IFF_ALLMULTI)) { 1276 if (can_sleep == 1) { 1277 end_synchronized_op(sc, 0); 1278 can_sleep = 0; 1279 goto redo_sifflags; 1280 } 1281 rc = update_mac_settings(ifp, 1282 XGMAC_PROMISC | XGMAC_ALLMULTI); 1283 } 1284 } else { 1285 if (can_sleep == 0) { 1286 end_synchronized_op(sc, LOCK_HELD); 1287 can_sleep = 1; 1288 goto redo_sifflags; 1289 } 1290 rc = cxgbe_init_synchronized(pi); 1291 } 1292 pi->if_flags = ifp->if_flags; 1293 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1294 if (can_sleep == 0) { 1295 end_synchronized_op(sc, LOCK_HELD); 1296 can_sleep = 1; 1297 goto redo_sifflags; 1298 } 1299 rc = cxgbe_uninit_synchronized(pi); 1300 } 1301 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD); 1302 break; 1303 1304 case SIOCADDMULTI: 1305 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1306 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi"); 1307 if (rc) 1308 return (rc); 1309 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1310 rc = update_mac_settings(ifp, XGMAC_MCADDRS); 1311 end_synchronized_op(sc, LOCK_HELD); 1312 break; 1313 1314 case SIOCSIFCAP: 1315 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap"); 1316 if (rc) 1317 return (rc); 1318 1319 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1320 if (mask & IFCAP_TXCSUM) { 1321 ifp->if_capenable ^= IFCAP_TXCSUM; 1322 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1323 1324 if (IFCAP_TSO4 & ifp->if_capenable && 1325 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1326 ifp->if_capenable &= ~IFCAP_TSO4; 1327 if_printf(ifp, 1328 "tso4 disabled due to -txcsum.\n"); 1329 } 1330 } 1331 if (mask & IFCAP_TXCSUM_IPV6) { 1332 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1333 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1334 1335 if (IFCAP_TSO6 & ifp->if_capenable && 1336 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1337 ifp->if_capenable &= ~IFCAP_TSO6; 1338 if_printf(ifp, 1339 "tso6 disabled due to -txcsum6.\n"); 1340 } 1341 } 1342 if (mask & IFCAP_RXCSUM) 1343 ifp->if_capenable ^= IFCAP_RXCSUM; 1344 if (mask & IFCAP_RXCSUM_IPV6) 1345 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1346 1347 /* 1348 * Note that we leave CSUM_TSO alone (it is always set). The 1349 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1350 * sending a TSO request our way, so it's sufficient to toggle 1351 * IFCAP_TSOx only. 1352 */ 1353 if (mask & IFCAP_TSO4) { 1354 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1355 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1356 if_printf(ifp, "enable txcsum first.\n"); 1357 rc = EAGAIN; 1358 goto fail; 1359 } 1360 ifp->if_capenable ^= IFCAP_TSO4; 1361 } 1362 if (mask & IFCAP_TSO6) { 1363 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1364 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1365 if_printf(ifp, "enable txcsum6 first.\n"); 1366 rc = EAGAIN; 1367 goto fail; 1368 } 1369 ifp->if_capenable ^= IFCAP_TSO6; 1370 } 1371 if (mask & IFCAP_LRO) { 1372 #if defined(INET) || defined(INET6) 1373 int i; 1374 struct sge_rxq *rxq; 1375 1376 ifp->if_capenable ^= IFCAP_LRO; 1377 for_each_rxq(pi, i, rxq) { 1378 if (ifp->if_capenable & IFCAP_LRO) 1379 rxq->iq.flags |= IQ_LRO_ENABLED; 1380 else 1381 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1382 } 1383 #endif 1384 } 1385 #ifdef TCP_OFFLOAD 1386 if (mask & IFCAP_TOE) { 1387 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1388 1389 rc = toe_capability(pi, enable); 1390 if (rc != 0) 1391 goto fail; 1392 1393 ifp->if_capenable ^= mask; 1394 } 1395 #endif 1396 if (mask & IFCAP_VLAN_HWTAGGING) { 1397 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1398 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1399 rc = update_mac_settings(ifp, XGMAC_VLANEX); 1400 } 1401 if (mask & IFCAP_VLAN_MTU) { 1402 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1403 1404 /* Need to find out how to disable auto-mtu-inflation */ 1405 } 1406 if (mask & IFCAP_VLAN_HWTSO) 1407 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1408 if (mask & IFCAP_VLAN_HWCSUM) 1409 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1410 1411 #ifdef VLAN_CAPABILITIES 1412 VLAN_CAPABILITIES(ifp); 1413 #endif 1414 fail: 1415 end_synchronized_op(sc, 0); 1416 break; 1417 1418 case SIOCSIFMEDIA: 1419 case SIOCGIFMEDIA: 1420 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1421 break; 1422 1423 case SIOCGI2C: { 1424 struct ifi2creq i2c; 1425 1426 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); 1427 if (rc != 0) 1428 break; 1429 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) { 1430 rc = EPERM; 1431 break; 1432 } 1433 if (i2c.len > sizeof(i2c.data)) { 1434 rc = EINVAL; 1435 break; 1436 } 1437 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4i2c"); 1438 if (rc) 1439 return (rc); 1440 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr, 1441 i2c.offset, i2c.len, &i2c.data[0]); 1442 end_synchronized_op(sc, 0); 1443 if (rc == 0) 1444 rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); 1445 break; 1446 } 1447 1448 default: 1449 rc = ether_ioctl(ifp, cmd, data); 1450 } 1451 1452 return (rc); 1453 } 1454 1455 static int 1456 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1457 { 1458 struct port_info *pi = ifp->if_softc; 1459 struct adapter *sc = pi->adapter; 1460 struct sge_txq *txq; 1461 void *items[1]; 1462 int rc; 1463 1464 M_ASSERTPKTHDR(m); 1465 MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */ 1466 1467 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1468 m_freem(m); 1469 return (ENETDOWN); 1470 } 1471 1472 rc = parse_pkt(&m); 1473 if (__predict_false(rc != 0)) { 1474 MPASS(m == NULL); /* was freed already */ 1475 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */ 1476 return (rc); 1477 } 1478 1479 /* Select a txq. */ 1480 txq = &sc->sge.txq[pi->first_txq]; 1481 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 1482 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) + 1483 pi->rsrv_noflowq); 1484 1485 items[0] = m; 1486 rc = mp_ring_enqueue(txq->r, items, 1, 4096); 1487 if (__predict_false(rc != 0)) 1488 m_freem(m); 1489 1490 return (rc); 1491 } 1492 1493 static void 1494 cxgbe_qflush(struct ifnet *ifp) 1495 { 1496 struct port_info *pi = ifp->if_softc; 1497 struct sge_txq *txq; 1498 int i; 1499 1500 /* queues do not exist if !PORT_INIT_DONE. */ 1501 if (pi->flags & PORT_INIT_DONE) { 1502 for_each_txq(pi, i, txq) { 1503 TXQ_LOCK(txq); 1504 txq->eq.flags &= ~EQ_ENABLED; 1505 TXQ_UNLOCK(txq); 1506 while (!mp_ring_is_idle(txq->r)) { 1507 mp_ring_check_drainage(txq->r, 0); 1508 pause("qflush", 1); 1509 } 1510 } 1511 } 1512 if_qflush(ifp); 1513 } 1514 1515 static uint64_t 1516 cxgbe_get_counter(struct ifnet *ifp, ift_counter c) 1517 { 1518 struct port_info *pi = ifp->if_softc; 1519 struct adapter *sc = pi->adapter; 1520 struct port_stats *s = &pi->stats; 1521 1522 cxgbe_refresh_stats(sc, pi); 1523 1524 switch (c) { 1525 case IFCOUNTER_IPACKETS: 1526 return (s->rx_frames - s->rx_pause); 1527 1528 case IFCOUNTER_IERRORS: 1529 return (s->rx_jabber + s->rx_runt + s->rx_too_long + 1530 s->rx_fcs_err + s->rx_len_err); 1531 1532 case IFCOUNTER_OPACKETS: 1533 return (s->tx_frames - s->tx_pause); 1534 1535 case IFCOUNTER_OERRORS: 1536 return (s->tx_error_frames); 1537 1538 case IFCOUNTER_IBYTES: 1539 return (s->rx_octets - s->rx_pause * 64); 1540 1541 case IFCOUNTER_OBYTES: 1542 return (s->tx_octets - s->tx_pause * 64); 1543 1544 case IFCOUNTER_IMCASTS: 1545 return (s->rx_mcast_frames - s->rx_pause); 1546 1547 case IFCOUNTER_OMCASTS: 1548 return (s->tx_mcast_frames - s->tx_pause); 1549 1550 case IFCOUNTER_IQDROPS: 1551 return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 1552 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 1553 s->rx_trunc3 + pi->tnl_cong_drops); 1554 1555 case IFCOUNTER_OQDROPS: { 1556 uint64_t drops; 1557 1558 drops = s->tx_drop; 1559 if (pi->flags & PORT_INIT_DONE) { 1560 int i; 1561 struct sge_txq *txq; 1562 1563 for_each_txq(pi, i, txq) 1564 drops += counter_u64_fetch(txq->r->drops); 1565 } 1566 1567 return (drops); 1568 1569 } 1570 1571 default: 1572 return (if_get_counter_default(ifp, c)); 1573 } 1574 } 1575 1576 static int 1577 cxgbe_media_change(struct ifnet *ifp) 1578 { 1579 struct port_info *pi = ifp->if_softc; 1580 1581 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1582 1583 return (EOPNOTSUPP); 1584 } 1585 1586 static void 1587 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1588 { 1589 struct port_info *pi = ifp->if_softc; 1590 struct ifmedia *media = NULL; 1591 struct ifmedia_entry *cur; 1592 int speed = pi->link_cfg.speed; 1593 1594 if (ifp == pi->ifp) 1595 media = &pi->media; 1596 #ifdef DEV_NETMAP 1597 else if (ifp == pi->nm_ifp) 1598 media = &pi->nm_media; 1599 #endif 1600 MPASS(media != NULL); 1601 1602 cur = media->ifm_cur; 1603 1604 ifmr->ifm_status = IFM_AVALID; 1605 if (!pi->link_cfg.link_ok) 1606 return; 1607 1608 ifmr->ifm_status |= IFM_ACTIVE; 1609 1610 /* active and current will differ iff current media is autoselect. */ 1611 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1612 return; 1613 1614 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1615 if (speed == SPEED_10000) 1616 ifmr->ifm_active |= IFM_10G_T; 1617 else if (speed == SPEED_1000) 1618 ifmr->ifm_active |= IFM_1000_T; 1619 else if (speed == SPEED_100) 1620 ifmr->ifm_active |= IFM_100_TX; 1621 else if (speed == SPEED_10) 1622 ifmr->ifm_active |= IFM_10_T; 1623 else 1624 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1625 speed)); 1626 } 1627 1628 void 1629 t4_fatal_err(struct adapter *sc) 1630 { 1631 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1632 t4_intr_disable(sc); 1633 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1634 device_get_nameunit(sc->dev)); 1635 } 1636 1637 static int 1638 map_bars_0_and_4(struct adapter *sc) 1639 { 1640 sc->regs_rid = PCIR_BAR(0); 1641 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1642 &sc->regs_rid, RF_ACTIVE); 1643 if (sc->regs_res == NULL) { 1644 device_printf(sc->dev, "cannot map registers.\n"); 1645 return (ENXIO); 1646 } 1647 sc->bt = rman_get_bustag(sc->regs_res); 1648 sc->bh = rman_get_bushandle(sc->regs_res); 1649 sc->mmio_len = rman_get_size(sc->regs_res); 1650 setbit(&sc->doorbells, DOORBELL_KDB); 1651 1652 sc->msix_rid = PCIR_BAR(4); 1653 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1654 &sc->msix_rid, RF_ACTIVE); 1655 if (sc->msix_res == NULL) { 1656 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1657 return (ENXIO); 1658 } 1659 1660 return (0); 1661 } 1662 1663 static int 1664 map_bar_2(struct adapter *sc) 1665 { 1666 1667 /* 1668 * T4: only iWARP driver uses the userspace doorbells. There is no need 1669 * to map it if RDMA is disabled. 1670 */ 1671 if (is_t4(sc) && sc->rdmacaps == 0) 1672 return (0); 1673 1674 sc->udbs_rid = PCIR_BAR(2); 1675 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1676 &sc->udbs_rid, RF_ACTIVE); 1677 if (sc->udbs_res == NULL) { 1678 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1679 return (ENXIO); 1680 } 1681 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1682 1683 if (is_t5(sc)) { 1684 setbit(&sc->doorbells, DOORBELL_UDB); 1685 #if defined(__i386__) || defined(__amd64__) 1686 if (t5_write_combine) { 1687 int rc; 1688 1689 /* 1690 * Enable write combining on BAR2. This is the 1691 * userspace doorbell BAR and is split into 128B 1692 * (UDBS_SEG_SIZE) doorbell regions, each associated 1693 * with an egress queue. The first 64B has the doorbell 1694 * and the second 64B can be used to submit a tx work 1695 * request with an implicit doorbell. 1696 */ 1697 1698 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1699 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1700 if (rc == 0) { 1701 clrbit(&sc->doorbells, DOORBELL_UDB); 1702 setbit(&sc->doorbells, DOORBELL_WCWR); 1703 setbit(&sc->doorbells, DOORBELL_UDBWC); 1704 } else { 1705 device_printf(sc->dev, 1706 "couldn't enable write combining: %d\n", 1707 rc); 1708 } 1709 1710 t4_write_reg(sc, A_SGE_STAT_CFG, 1711 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1712 } 1713 #endif 1714 } 1715 1716 return (0); 1717 } 1718 1719 static const struct memwin t4_memwin[] = { 1720 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1721 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1722 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1723 }; 1724 1725 static const struct memwin t5_memwin[] = { 1726 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1727 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1728 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1729 }; 1730 1731 static void 1732 setup_memwin(struct adapter *sc) 1733 { 1734 const struct memwin *mw; 1735 int i, n; 1736 uint32_t bar0; 1737 1738 if (is_t4(sc)) { 1739 /* 1740 * Read low 32b of bar0 indirectly via the hardware backdoor 1741 * mechanism. Works from within PCI passthrough environments 1742 * too, where rman_get_start() can return a different value. We 1743 * need to program the T4 memory window decoders with the actual 1744 * addresses that will be coming across the PCIe link. 1745 */ 1746 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1747 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1748 1749 mw = &t4_memwin[0]; 1750 n = nitems(t4_memwin); 1751 } else { 1752 /* T5 uses the relative offset inside the PCIe BAR */ 1753 bar0 = 0; 1754 1755 mw = &t5_memwin[0]; 1756 n = nitems(t5_memwin); 1757 } 1758 1759 for (i = 0; i < n; i++, mw++) { 1760 t4_write_reg(sc, 1761 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 1762 (mw->base + bar0) | V_BIR(0) | 1763 V_WINDOW(ilog2(mw->aperture) - 10)); 1764 } 1765 1766 /* flush */ 1767 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1768 } 1769 1770 /* 1771 * Verify that the memory range specified by the addr/len pair is valid and lies 1772 * entirely within a single region (EDCx or MCx). 1773 */ 1774 static int 1775 validate_mem_range(struct adapter *sc, uint32_t addr, int len) 1776 { 1777 uint32_t em, addr_len, maddr, mlen; 1778 1779 /* Memory can only be accessed in naturally aligned 4 byte units */ 1780 if (addr & 3 || len & 3 || len == 0) 1781 return (EINVAL); 1782 1783 /* Enabled memories */ 1784 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1785 if (em & F_EDRAM0_ENABLE) { 1786 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1787 maddr = G_EDRAM0_BASE(addr_len) << 20; 1788 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1789 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1790 addr + len <= maddr + mlen) 1791 return (0); 1792 } 1793 if (em & F_EDRAM1_ENABLE) { 1794 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1795 maddr = G_EDRAM1_BASE(addr_len) << 20; 1796 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1797 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1798 addr + len <= maddr + mlen) 1799 return (0); 1800 } 1801 if (em & F_EXT_MEM_ENABLE) { 1802 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1803 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1804 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1805 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1806 addr + len <= maddr + mlen) 1807 return (0); 1808 } 1809 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 1810 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1811 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1812 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1813 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1814 addr + len <= maddr + mlen) 1815 return (0); 1816 } 1817 1818 return (EFAULT); 1819 } 1820 1821 static int 1822 fwmtype_to_hwmtype(int mtype) 1823 { 1824 1825 switch (mtype) { 1826 case FW_MEMTYPE_EDC0: 1827 return (MEM_EDC0); 1828 case FW_MEMTYPE_EDC1: 1829 return (MEM_EDC1); 1830 case FW_MEMTYPE_EXTMEM: 1831 return (MEM_MC0); 1832 case FW_MEMTYPE_EXTMEM1: 1833 return (MEM_MC1); 1834 default: 1835 panic("%s: cannot translate fw mtype %d.", __func__, mtype); 1836 } 1837 } 1838 1839 /* 1840 * Verify that the memory range specified by the memtype/offset/len pair is 1841 * valid and lies entirely within the memtype specified. The global address of 1842 * the start of the range is returned in addr. 1843 */ 1844 static int 1845 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 1846 uint32_t *addr) 1847 { 1848 uint32_t em, addr_len, maddr, mlen; 1849 1850 /* Memory can only be accessed in naturally aligned 4 byte units */ 1851 if (off & 3 || len & 3 || len == 0) 1852 return (EINVAL); 1853 1854 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1855 switch (fwmtype_to_hwmtype(mtype)) { 1856 case MEM_EDC0: 1857 if (!(em & F_EDRAM0_ENABLE)) 1858 return (EINVAL); 1859 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1860 maddr = G_EDRAM0_BASE(addr_len) << 20; 1861 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1862 break; 1863 case MEM_EDC1: 1864 if (!(em & F_EDRAM1_ENABLE)) 1865 return (EINVAL); 1866 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1867 maddr = G_EDRAM1_BASE(addr_len) << 20; 1868 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1869 break; 1870 case MEM_MC: 1871 if (!(em & F_EXT_MEM_ENABLE)) 1872 return (EINVAL); 1873 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1874 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1875 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1876 break; 1877 case MEM_MC1: 1878 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 1879 return (EINVAL); 1880 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1881 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1882 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1883 break; 1884 default: 1885 return (EINVAL); 1886 } 1887 1888 if (mlen > 0 && off < mlen && off + len <= mlen) { 1889 *addr = maddr + off; /* global address */ 1890 return (0); 1891 } 1892 1893 return (EFAULT); 1894 } 1895 1896 static void 1897 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 1898 { 1899 const struct memwin *mw; 1900 1901 if (is_t4(sc)) { 1902 KASSERT(win >= 0 && win < nitems(t4_memwin), 1903 ("%s: incorrect memwin# (%d)", __func__, win)); 1904 mw = &t4_memwin[win]; 1905 } else { 1906 KASSERT(win >= 0 && win < nitems(t5_memwin), 1907 ("%s: incorrect memwin# (%d)", __func__, win)); 1908 mw = &t5_memwin[win]; 1909 } 1910 1911 if (base != NULL) 1912 *base = mw->base; 1913 if (aperture != NULL) 1914 *aperture = mw->aperture; 1915 } 1916 1917 /* 1918 * Positions the memory window such that it can be used to access the specified 1919 * address in the chip's address space. The return value is the offset of addr 1920 * from the start of the window. 1921 */ 1922 static uint32_t 1923 position_memwin(struct adapter *sc, int n, uint32_t addr) 1924 { 1925 uint32_t start, pf; 1926 uint32_t reg; 1927 1928 KASSERT(n >= 0 && n <= 3, 1929 ("%s: invalid window %d.", __func__, n)); 1930 KASSERT((addr & 3) == 0, 1931 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 1932 1933 if (is_t4(sc)) { 1934 pf = 0; 1935 start = addr & ~0xf; /* start must be 16B aligned */ 1936 } else { 1937 pf = V_PFNUM(sc->pf); 1938 start = addr & ~0x7f; /* start must be 128B aligned */ 1939 } 1940 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 1941 1942 t4_write_reg(sc, reg, start | pf); 1943 t4_read_reg(sc, reg); 1944 1945 return (addr - start); 1946 } 1947 1948 static int 1949 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1950 struct intrs_and_queues *iaq) 1951 { 1952 int rc, itype, navail, nrxq10g, nrxq1g, n; 1953 int nofldrxq10g = 0, nofldrxq1g = 0; 1954 int nnmrxq10g = 0, nnmrxq1g = 0; 1955 1956 bzero(iaq, sizeof(*iaq)); 1957 1958 iaq->ntxq10g = t4_ntxq10g; 1959 iaq->ntxq1g = t4_ntxq1g; 1960 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1961 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1962 iaq->rsrv_noflowq = t4_rsrv_noflowq; 1963 #ifdef TCP_OFFLOAD 1964 if (is_offload(sc)) { 1965 iaq->nofldtxq10g = t4_nofldtxq10g; 1966 iaq->nofldtxq1g = t4_nofldtxq1g; 1967 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1968 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1969 } 1970 #endif 1971 #ifdef DEV_NETMAP 1972 iaq->nnmtxq10g = t4_nnmtxq10g; 1973 iaq->nnmtxq1g = t4_nnmtxq1g; 1974 iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g; 1975 iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g; 1976 #endif 1977 1978 for (itype = INTR_MSIX; itype; itype >>= 1) { 1979 1980 if ((itype & t4_intr_types) == 0) 1981 continue; /* not allowed */ 1982 1983 if (itype == INTR_MSIX) 1984 navail = pci_msix_count(sc->dev); 1985 else if (itype == INTR_MSI) 1986 navail = pci_msi_count(sc->dev); 1987 else 1988 navail = 1; 1989 restart: 1990 if (navail == 0) 1991 continue; 1992 1993 iaq->intr_type = itype; 1994 iaq->intr_flags_10g = 0; 1995 iaq->intr_flags_1g = 0; 1996 1997 /* 1998 * Best option: an interrupt vector for errors, one for the 1999 * firmware event queue, and one for every rxq (NIC, TOE, and 2000 * netmap). 2001 */ 2002 iaq->nirq = T4_EXTRA_INTR; 2003 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g); 2004 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g); 2005 if (iaq->nirq <= navail && 2006 (itype != INTR_MSI || powerof2(iaq->nirq))) { 2007 iaq->intr_flags_10g = INTR_ALL; 2008 iaq->intr_flags_1g = INTR_ALL; 2009 goto allocate; 2010 } 2011 2012 /* 2013 * Second best option: a vector for errors, one for the firmware 2014 * event queue, and vectors for either all the NIC rx queues or 2015 * all the TOE rx queues. The queues that don't get vectors 2016 * will forward their interrupts to those that do. 2017 * 2018 * Note: netmap rx queues cannot be created early and so they 2019 * can't be setup to receive forwarded interrupts for others. 2020 */ 2021 iaq->nirq = T4_EXTRA_INTR; 2022 if (nrxq10g >= nofldrxq10g) { 2023 iaq->intr_flags_10g = INTR_RXQ; 2024 iaq->nirq += n10g * nrxq10g; 2025 #ifdef DEV_NETMAP 2026 iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g); 2027 #endif 2028 } else { 2029 iaq->intr_flags_10g = INTR_OFLD_RXQ; 2030 iaq->nirq += n10g * nofldrxq10g; 2031 #ifdef DEV_NETMAP 2032 iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g); 2033 #endif 2034 } 2035 if (nrxq1g >= nofldrxq1g) { 2036 iaq->intr_flags_1g = INTR_RXQ; 2037 iaq->nirq += n1g * nrxq1g; 2038 #ifdef DEV_NETMAP 2039 iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g); 2040 #endif 2041 } else { 2042 iaq->intr_flags_1g = INTR_OFLD_RXQ; 2043 iaq->nirq += n1g * nofldrxq1g; 2044 #ifdef DEV_NETMAP 2045 iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g); 2046 #endif 2047 } 2048 if (iaq->nirq <= navail && 2049 (itype != INTR_MSI || powerof2(iaq->nirq))) 2050 goto allocate; 2051 2052 /* 2053 * Next best option: an interrupt vector for errors, one for the 2054 * firmware event queue, and at least one per port. At this 2055 * point we know we'll have to downsize nrxq and/or nofldrxq 2056 * and/or nnmrxq to fit what's available to us. 2057 */ 2058 iaq->nirq = T4_EXTRA_INTR; 2059 iaq->nirq += n10g + n1g; 2060 if (iaq->nirq <= navail) { 2061 int leftover = navail - iaq->nirq; 2062 2063 if (n10g > 0) { 2064 int target = max(nrxq10g, nofldrxq10g); 2065 2066 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ? 2067 INTR_RXQ : INTR_OFLD_RXQ; 2068 2069 n = 1; 2070 while (n < target && leftover >= n10g) { 2071 leftover -= n10g; 2072 iaq->nirq += n10g; 2073 n++; 2074 } 2075 iaq->nrxq10g = min(n, nrxq10g); 2076 #ifdef TCP_OFFLOAD 2077 iaq->nofldrxq10g = min(n, nofldrxq10g); 2078 #endif 2079 #ifdef DEV_NETMAP 2080 iaq->nnmrxq10g = min(n, nnmrxq10g); 2081 #endif 2082 } 2083 2084 if (n1g > 0) { 2085 int target = max(nrxq1g, nofldrxq1g); 2086 2087 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ? 2088 INTR_RXQ : INTR_OFLD_RXQ; 2089 2090 n = 1; 2091 while (n < target && leftover >= n1g) { 2092 leftover -= n1g; 2093 iaq->nirq += n1g; 2094 n++; 2095 } 2096 iaq->nrxq1g = min(n, nrxq1g); 2097 #ifdef TCP_OFFLOAD 2098 iaq->nofldrxq1g = min(n, nofldrxq1g); 2099 #endif 2100 #ifdef DEV_NETMAP 2101 iaq->nnmrxq1g = min(n, nnmrxq1g); 2102 #endif 2103 } 2104 2105 if (itype != INTR_MSI || powerof2(iaq->nirq)) 2106 goto allocate; 2107 } 2108 2109 /* 2110 * Least desirable option: one interrupt vector for everything. 2111 */ 2112 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 2113 iaq->intr_flags_10g = iaq->intr_flags_1g = 0; 2114 #ifdef TCP_OFFLOAD 2115 if (is_offload(sc)) 2116 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 2117 #endif 2118 #ifdef DEV_NETMAP 2119 iaq->nnmrxq10g = iaq->nnmrxq1g = 1; 2120 #endif 2121 2122 allocate: 2123 navail = iaq->nirq; 2124 rc = 0; 2125 if (itype == INTR_MSIX) 2126 rc = pci_alloc_msix(sc->dev, &navail); 2127 else if (itype == INTR_MSI) 2128 rc = pci_alloc_msi(sc->dev, &navail); 2129 2130 if (rc == 0) { 2131 if (navail == iaq->nirq) 2132 return (0); 2133 2134 /* 2135 * Didn't get the number requested. Use whatever number 2136 * the kernel is willing to allocate (it's in navail). 2137 */ 2138 device_printf(sc->dev, "fewer vectors than requested, " 2139 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 2140 itype, iaq->nirq, navail); 2141 pci_release_msi(sc->dev); 2142 goto restart; 2143 } 2144 2145 device_printf(sc->dev, 2146 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 2147 itype, rc, iaq->nirq, navail); 2148 } 2149 2150 device_printf(sc->dev, 2151 "failed to find a usable interrupt type. " 2152 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 2153 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 2154 2155 return (ENXIO); 2156 } 2157 2158 #define FW_VERSION(chip) ( \ 2159 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ 2160 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ 2161 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ 2162 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) 2163 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) 2164 2165 struct fw_info { 2166 uint8_t chip; 2167 char *kld_name; 2168 char *fw_mod_name; 2169 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 2170 } fw_info[] = { 2171 { 2172 .chip = CHELSIO_T4, 2173 .kld_name = "t4fw_cfg", 2174 .fw_mod_name = "t4fw", 2175 .fw_hdr = { 2176 .chip = FW_HDR_CHIP_T4, 2177 .fw_ver = htobe32_const(FW_VERSION(T4)), 2178 .intfver_nic = FW_INTFVER(T4, NIC), 2179 .intfver_vnic = FW_INTFVER(T4, VNIC), 2180 .intfver_ofld = FW_INTFVER(T4, OFLD), 2181 .intfver_ri = FW_INTFVER(T4, RI), 2182 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 2183 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 2184 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 2185 .intfver_fcoe = FW_INTFVER(T4, FCOE), 2186 }, 2187 }, { 2188 .chip = CHELSIO_T5, 2189 .kld_name = "t5fw_cfg", 2190 .fw_mod_name = "t5fw", 2191 .fw_hdr = { 2192 .chip = FW_HDR_CHIP_T5, 2193 .fw_ver = htobe32_const(FW_VERSION(T5)), 2194 .intfver_nic = FW_INTFVER(T5, NIC), 2195 .intfver_vnic = FW_INTFVER(T5, VNIC), 2196 .intfver_ofld = FW_INTFVER(T5, OFLD), 2197 .intfver_ri = FW_INTFVER(T5, RI), 2198 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 2199 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 2200 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 2201 .intfver_fcoe = FW_INTFVER(T5, FCOE), 2202 }, 2203 } 2204 }; 2205 2206 static struct fw_info * 2207 find_fw_info(int chip) 2208 { 2209 int i; 2210 2211 for (i = 0; i < nitems(fw_info); i++) { 2212 if (fw_info[i].chip == chip) 2213 return (&fw_info[i]); 2214 } 2215 return (NULL); 2216 } 2217 2218 /* 2219 * Is the given firmware API compatible with the one the driver was compiled 2220 * with? 2221 */ 2222 static int 2223 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 2224 { 2225 2226 /* short circuit if it's the exact same firmware version */ 2227 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 2228 return (1); 2229 2230 /* 2231 * XXX: Is this too conservative? Perhaps I should limit this to the 2232 * features that are supported in the driver. 2233 */ 2234 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 2235 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 2236 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 2237 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 2238 return (1); 2239 #undef SAME_INTF 2240 2241 return (0); 2242 } 2243 2244 /* 2245 * The firmware in the KLD is usable, but should it be installed? This routine 2246 * explains itself in detail if it indicates the KLD firmware should be 2247 * installed. 2248 */ 2249 static int 2250 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 2251 { 2252 const char *reason; 2253 2254 if (!card_fw_usable) { 2255 reason = "incompatible or unusable"; 2256 goto install; 2257 } 2258 2259 if (k > c) { 2260 reason = "older than the version bundled with this driver"; 2261 goto install; 2262 } 2263 2264 if (t4_fw_install == 2 && k != c) { 2265 reason = "different than the version bundled with this driver"; 2266 goto install; 2267 } 2268 2269 return (0); 2270 2271 install: 2272 if (t4_fw_install == 0) { 2273 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2274 "but the driver is prohibited from installing a different " 2275 "firmware on the card.\n", 2276 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2277 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 2278 2279 return (0); 2280 } 2281 2282 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2283 "installing firmware %u.%u.%u.%u on card.\n", 2284 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2285 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 2286 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2287 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2288 2289 return (1); 2290 } 2291 /* 2292 * Establish contact with the firmware and determine if we are the master driver 2293 * or not, and whether we are responsible for chip initialization. 2294 */ 2295 static int 2296 prep_firmware(struct adapter *sc) 2297 { 2298 const struct firmware *fw = NULL, *default_cfg; 2299 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 2300 enum dev_state state; 2301 struct fw_info *fw_info; 2302 struct fw_hdr *card_fw; /* fw on the card */ 2303 const struct fw_hdr *kld_fw; /* fw in the KLD */ 2304 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 2305 against */ 2306 2307 /* Contact firmware. */ 2308 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 2309 if (rc < 0 || state == DEV_STATE_ERR) { 2310 rc = -rc; 2311 device_printf(sc->dev, 2312 "failed to connect to the firmware: %d, %d.\n", rc, state); 2313 return (rc); 2314 } 2315 pf = rc; 2316 if (pf == sc->mbox) 2317 sc->flags |= MASTER_PF; 2318 else if (state == DEV_STATE_UNINIT) { 2319 /* 2320 * We didn't get to be the master so we definitely won't be 2321 * configuring the chip. It's a bug if someone else hasn't 2322 * configured it already. 2323 */ 2324 device_printf(sc->dev, "couldn't be master(%d), " 2325 "device not already initialized either(%d).\n", rc, state); 2326 return (EDOOFUS); 2327 } 2328 2329 /* This is the firmware whose headers the driver was compiled against */ 2330 fw_info = find_fw_info(chip_id(sc)); 2331 if (fw_info == NULL) { 2332 device_printf(sc->dev, 2333 "unable to look up firmware information for chip %d.\n", 2334 chip_id(sc)); 2335 return (EINVAL); 2336 } 2337 drv_fw = &fw_info->fw_hdr; 2338 2339 /* 2340 * The firmware KLD contains many modules. The KLD name is also the 2341 * name of the module that contains the default config file. 2342 */ 2343 default_cfg = firmware_get(fw_info->kld_name); 2344 2345 /* Read the header of the firmware on the card */ 2346 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 2347 rc = -t4_read_flash(sc, FLASH_FW_START, 2348 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 2349 if (rc == 0) 2350 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 2351 else { 2352 device_printf(sc->dev, 2353 "Unable to read card's firmware header: %d\n", rc); 2354 card_fw_usable = 0; 2355 } 2356 2357 /* This is the firmware in the KLD */ 2358 fw = firmware_get(fw_info->fw_mod_name); 2359 if (fw != NULL) { 2360 kld_fw = (const void *)fw->data; 2361 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 2362 } else { 2363 kld_fw = NULL; 2364 kld_fw_usable = 0; 2365 } 2366 2367 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2368 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) { 2369 /* 2370 * Common case: the firmware on the card is an exact match and 2371 * the KLD is an exact match too, or the KLD is 2372 * absent/incompatible. Note that t4_fw_install = 2 is ignored 2373 * here -- use cxgbetool loadfw if you want to reinstall the 2374 * same firmware as the one on the card. 2375 */ 2376 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2377 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2378 be32toh(card_fw->fw_ver))) { 2379 2380 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 2381 if (rc != 0) { 2382 device_printf(sc->dev, 2383 "failed to install firmware: %d\n", rc); 2384 goto done; 2385 } 2386 2387 /* Installed successfully, update the cached header too. */ 2388 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2389 card_fw_usable = 1; 2390 need_fw_reset = 0; /* already reset as part of load_fw */ 2391 } 2392 2393 if (!card_fw_usable) { 2394 uint32_t d, c, k; 2395 2396 d = ntohl(drv_fw->fw_ver); 2397 c = ntohl(card_fw->fw_ver); 2398 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2399 2400 device_printf(sc->dev, "Cannot find a usable firmware: " 2401 "fw_install %d, chip state %d, " 2402 "driver compiled with %d.%d.%d.%d, " 2403 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2404 t4_fw_install, state, 2405 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2406 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2407 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2408 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2409 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2410 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2411 rc = EINVAL; 2412 goto done; 2413 } 2414 2415 /* We're using whatever's on the card and it's known to be good. */ 2416 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2417 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2418 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2419 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2420 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2421 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2422 t4_get_tp_version(sc, &sc->params.tp_vers); 2423 2424 /* Reset device */ 2425 if (need_fw_reset && 2426 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2427 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2428 if (rc != ETIMEDOUT && rc != EIO) 2429 t4_fw_bye(sc, sc->mbox); 2430 goto done; 2431 } 2432 sc->flags |= FW_OK; 2433 2434 rc = get_params__pre_init(sc); 2435 if (rc != 0) 2436 goto done; /* error message displayed already */ 2437 2438 /* Partition adapter resources as specified in the config file. */ 2439 if (state == DEV_STATE_UNINIT) { 2440 2441 KASSERT(sc->flags & MASTER_PF, 2442 ("%s: trying to change chip settings when not master.", 2443 __func__)); 2444 2445 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2446 if (rc != 0) 2447 goto done; /* error message displayed already */ 2448 2449 t4_tweak_chip_settings(sc); 2450 2451 /* get basic stuff going */ 2452 rc = -t4_fw_initialize(sc, sc->mbox); 2453 if (rc != 0) { 2454 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2455 goto done; 2456 } 2457 } else { 2458 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2459 sc->cfcsum = 0; 2460 } 2461 2462 done: 2463 free(card_fw, M_CXGBE); 2464 if (fw != NULL) 2465 firmware_put(fw, FIRMWARE_UNLOAD); 2466 if (default_cfg != NULL) 2467 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2468 2469 return (rc); 2470 } 2471 2472 #define FW_PARAM_DEV(param) \ 2473 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2474 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2475 #define FW_PARAM_PFVF(param) \ 2476 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2477 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2478 2479 /* 2480 * Partition chip resources for use between various PFs, VFs, etc. 2481 */ 2482 static int 2483 partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2484 const char *name_prefix) 2485 { 2486 const struct firmware *cfg = NULL; 2487 int rc = 0; 2488 struct fw_caps_config_cmd caps; 2489 uint32_t mtype, moff, finicsum, cfcsum; 2490 2491 /* 2492 * Figure out what configuration file to use. Pick the default config 2493 * file for the card if the user hasn't specified one explicitly. 2494 */ 2495 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2496 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2497 /* Card specific overrides go here. */ 2498 if (pci_get_device(sc->dev) == 0x440a) 2499 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2500 if (is_fpga(sc)) 2501 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2502 } 2503 2504 /* 2505 * We need to load another module if the profile is anything except 2506 * "default" or "flash". 2507 */ 2508 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2509 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2510 char s[32]; 2511 2512 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2513 cfg = firmware_get(s); 2514 if (cfg == NULL) { 2515 if (default_cfg != NULL) { 2516 device_printf(sc->dev, 2517 "unable to load module \"%s\" for " 2518 "configuration profile \"%s\", will use " 2519 "the default config file instead.\n", 2520 s, sc->cfg_file); 2521 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2522 "%s", DEFAULT_CF); 2523 } else { 2524 device_printf(sc->dev, 2525 "unable to load module \"%s\" for " 2526 "configuration profile \"%s\", will use " 2527 "the config file on the card's flash " 2528 "instead.\n", s, sc->cfg_file); 2529 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2530 "%s", FLASH_CF); 2531 } 2532 } 2533 } 2534 2535 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2536 default_cfg == NULL) { 2537 device_printf(sc->dev, 2538 "default config file not available, will use the config " 2539 "file on the card's flash instead.\n"); 2540 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2541 } 2542 2543 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2544 u_int cflen, i, n; 2545 const uint32_t *cfdata; 2546 uint32_t param, val, addr, off, mw_base, mw_aperture; 2547 2548 KASSERT(cfg != NULL || default_cfg != NULL, 2549 ("%s: no config to upload", __func__)); 2550 2551 /* 2552 * Ask the firmware where it wants us to upload the config file. 2553 */ 2554 param = FW_PARAM_DEV(CF); 2555 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2556 if (rc != 0) { 2557 /* No support for config file? Shouldn't happen. */ 2558 device_printf(sc->dev, 2559 "failed to query config file location: %d.\n", rc); 2560 goto done; 2561 } 2562 mtype = G_FW_PARAMS_PARAM_Y(val); 2563 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2564 2565 /* 2566 * XXX: sheer laziness. We deliberately added 4 bytes of 2567 * useless stuffing/comments at the end of the config file so 2568 * it's ok to simply throw away the last remaining bytes when 2569 * the config file is not an exact multiple of 4. This also 2570 * helps with the validate_mt_off_len check. 2571 */ 2572 if (cfg != NULL) { 2573 cflen = cfg->datasize & ~3; 2574 cfdata = cfg->data; 2575 } else { 2576 cflen = default_cfg->datasize & ~3; 2577 cfdata = default_cfg->data; 2578 } 2579 2580 if (cflen > FLASH_CFG_MAX_SIZE) { 2581 device_printf(sc->dev, 2582 "config file too long (%d, max allowed is %d). " 2583 "Will try to use the config on the card, if any.\n", 2584 cflen, FLASH_CFG_MAX_SIZE); 2585 goto use_config_on_flash; 2586 } 2587 2588 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2589 if (rc != 0) { 2590 device_printf(sc->dev, 2591 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2592 "Will try to use the config on the card, if any.\n", 2593 __func__, mtype, moff, cflen, rc); 2594 goto use_config_on_flash; 2595 } 2596 2597 memwin_info(sc, 2, &mw_base, &mw_aperture); 2598 while (cflen) { 2599 off = position_memwin(sc, 2, addr); 2600 n = min(cflen, mw_aperture - off); 2601 for (i = 0; i < n; i += 4) 2602 t4_write_reg(sc, mw_base + off + i, *cfdata++); 2603 cflen -= n; 2604 addr += n; 2605 } 2606 } else { 2607 use_config_on_flash: 2608 mtype = FW_MEMTYPE_FLASH; 2609 moff = t4_flash_cfg_addr(sc); 2610 } 2611 2612 bzero(&caps, sizeof(caps)); 2613 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2614 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2615 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2616 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2617 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2618 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2619 if (rc != 0) { 2620 device_printf(sc->dev, 2621 "failed to pre-process config file: %d " 2622 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2623 goto done; 2624 } 2625 2626 finicsum = be32toh(caps.finicsum); 2627 cfcsum = be32toh(caps.cfcsum); 2628 if (finicsum != cfcsum) { 2629 device_printf(sc->dev, 2630 "WARNING: config file checksum mismatch: %08x %08x\n", 2631 finicsum, cfcsum); 2632 } 2633 sc->cfcsum = cfcsum; 2634 2635 #define LIMIT_CAPS(x) do { \ 2636 caps.x &= htobe16(t4_##x##_allowed); \ 2637 } while (0) 2638 2639 /* 2640 * Let the firmware know what features will (not) be used so it can tune 2641 * things accordingly. 2642 */ 2643 LIMIT_CAPS(linkcaps); 2644 LIMIT_CAPS(niccaps); 2645 LIMIT_CAPS(toecaps); 2646 LIMIT_CAPS(rdmacaps); 2647 LIMIT_CAPS(iscsicaps); 2648 LIMIT_CAPS(fcoecaps); 2649 #undef LIMIT_CAPS 2650 2651 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2652 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2653 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2654 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2655 if (rc != 0) { 2656 device_printf(sc->dev, 2657 "failed to process config file: %d.\n", rc); 2658 } 2659 done: 2660 if (cfg != NULL) 2661 firmware_put(cfg, FIRMWARE_UNLOAD); 2662 return (rc); 2663 } 2664 2665 /* 2666 * Retrieve parameters that are needed (or nice to have) very early. 2667 */ 2668 static int 2669 get_params__pre_init(struct adapter *sc) 2670 { 2671 int rc; 2672 uint32_t param[2], val[2]; 2673 struct fw_devlog_cmd cmd; 2674 struct devlog_params *dlog = &sc->params.devlog; 2675 2676 param[0] = FW_PARAM_DEV(PORTVEC); 2677 param[1] = FW_PARAM_DEV(CCLK); 2678 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2679 if (rc != 0) { 2680 device_printf(sc->dev, 2681 "failed to query parameters (pre_init): %d.\n", rc); 2682 return (rc); 2683 } 2684 2685 sc->params.portvec = val[0]; 2686 sc->params.nports = bitcount32(val[0]); 2687 sc->params.vpd.cclk = val[1]; 2688 2689 /* Read device log parameters. */ 2690 bzero(&cmd, sizeof(cmd)); 2691 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2692 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2693 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2694 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2695 if (rc != 0) { 2696 device_printf(sc->dev, 2697 "failed to get devlog parameters: %d.\n", rc); 2698 bzero(dlog, sizeof (*dlog)); 2699 rc = 0; /* devlog isn't critical for device operation */ 2700 } else { 2701 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2702 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2703 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2704 dlog->size = be32toh(cmd.memsize_devlog); 2705 } 2706 2707 return (rc); 2708 } 2709 2710 /* 2711 * Retrieve various parameters that are of interest to the driver. The device 2712 * has been initialized by the firmware at this point. 2713 */ 2714 static int 2715 get_params__post_init(struct adapter *sc) 2716 { 2717 int rc; 2718 uint32_t param[7], val[7]; 2719 struct fw_caps_config_cmd caps; 2720 2721 param[0] = FW_PARAM_PFVF(IQFLINT_START); 2722 param[1] = FW_PARAM_PFVF(EQ_START); 2723 param[2] = FW_PARAM_PFVF(FILTER_START); 2724 param[3] = FW_PARAM_PFVF(FILTER_END); 2725 param[4] = FW_PARAM_PFVF(L2T_START); 2726 param[5] = FW_PARAM_PFVF(L2T_END); 2727 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2728 if (rc != 0) { 2729 device_printf(sc->dev, 2730 "failed to query parameters (post_init): %d.\n", rc); 2731 return (rc); 2732 } 2733 2734 sc->sge.iq_start = val[0]; 2735 sc->sge.eq_start = val[1]; 2736 sc->tids.ftid_base = val[2]; 2737 sc->tids.nftids = val[3] - val[2] + 1; 2738 sc->params.ftid_min = val[2]; 2739 sc->params.ftid_max = val[3]; 2740 sc->vres.l2t.start = val[4]; 2741 sc->vres.l2t.size = val[5] - val[4] + 1; 2742 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2743 ("%s: L2 table size (%u) larger than expected (%u)", 2744 __func__, sc->vres.l2t.size, L2T_SIZE)); 2745 2746 /* get capabilites */ 2747 bzero(&caps, sizeof(caps)); 2748 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2749 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2750 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2751 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2752 if (rc != 0) { 2753 device_printf(sc->dev, 2754 "failed to get card capabilities: %d.\n", rc); 2755 return (rc); 2756 } 2757 2758 #define READ_CAPS(x) do { \ 2759 sc->x = htobe16(caps.x); \ 2760 } while (0) 2761 READ_CAPS(linkcaps); 2762 READ_CAPS(niccaps); 2763 READ_CAPS(toecaps); 2764 READ_CAPS(rdmacaps); 2765 READ_CAPS(iscsicaps); 2766 READ_CAPS(fcoecaps); 2767 2768 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { 2769 param[0] = FW_PARAM_PFVF(ETHOFLD_START); 2770 param[1] = FW_PARAM_PFVF(ETHOFLD_END); 2771 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2772 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); 2773 if (rc != 0) { 2774 device_printf(sc->dev, 2775 "failed to query NIC parameters: %d.\n", rc); 2776 return (rc); 2777 } 2778 sc->tids.etid_base = val[0]; 2779 sc->params.etid_min = val[0]; 2780 sc->tids.netids = val[1] - val[0] + 1; 2781 sc->params.netids = sc->tids.netids; 2782 sc->params.eo_wr_cred = val[2]; 2783 sc->params.ethoffload = 1; 2784 } 2785 2786 if (sc->toecaps) { 2787 /* query offload-related parameters */ 2788 param[0] = FW_PARAM_DEV(NTID); 2789 param[1] = FW_PARAM_PFVF(SERVER_START); 2790 param[2] = FW_PARAM_PFVF(SERVER_END); 2791 param[3] = FW_PARAM_PFVF(TDDP_START); 2792 param[4] = FW_PARAM_PFVF(TDDP_END); 2793 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2794 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2795 if (rc != 0) { 2796 device_printf(sc->dev, 2797 "failed to query TOE parameters: %d.\n", rc); 2798 return (rc); 2799 } 2800 sc->tids.ntids = val[0]; 2801 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 2802 sc->tids.stid_base = val[1]; 2803 sc->tids.nstids = val[2] - val[1] + 1; 2804 sc->vres.ddp.start = val[3]; 2805 sc->vres.ddp.size = val[4] - val[3] + 1; 2806 sc->params.ofldq_wr_cred = val[5]; 2807 sc->params.offload = 1; 2808 } 2809 if (sc->rdmacaps) { 2810 param[0] = FW_PARAM_PFVF(STAG_START); 2811 param[1] = FW_PARAM_PFVF(STAG_END); 2812 param[2] = FW_PARAM_PFVF(RQ_START); 2813 param[3] = FW_PARAM_PFVF(RQ_END); 2814 param[4] = FW_PARAM_PFVF(PBL_START); 2815 param[5] = FW_PARAM_PFVF(PBL_END); 2816 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2817 if (rc != 0) { 2818 device_printf(sc->dev, 2819 "failed to query RDMA parameters(1): %d.\n", rc); 2820 return (rc); 2821 } 2822 sc->vres.stag.start = val[0]; 2823 sc->vres.stag.size = val[1] - val[0] + 1; 2824 sc->vres.rq.start = val[2]; 2825 sc->vres.rq.size = val[3] - val[2] + 1; 2826 sc->vres.pbl.start = val[4]; 2827 sc->vres.pbl.size = val[5] - val[4] + 1; 2828 2829 param[0] = FW_PARAM_PFVF(SQRQ_START); 2830 param[1] = FW_PARAM_PFVF(SQRQ_END); 2831 param[2] = FW_PARAM_PFVF(CQ_START); 2832 param[3] = FW_PARAM_PFVF(CQ_END); 2833 param[4] = FW_PARAM_PFVF(OCQ_START); 2834 param[5] = FW_PARAM_PFVF(OCQ_END); 2835 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2836 if (rc != 0) { 2837 device_printf(sc->dev, 2838 "failed to query RDMA parameters(2): %d.\n", rc); 2839 return (rc); 2840 } 2841 sc->vres.qp.start = val[0]; 2842 sc->vres.qp.size = val[1] - val[0] + 1; 2843 sc->vres.cq.start = val[2]; 2844 sc->vres.cq.size = val[3] - val[2] + 1; 2845 sc->vres.ocq.start = val[4]; 2846 sc->vres.ocq.size = val[5] - val[4] + 1; 2847 } 2848 if (sc->iscsicaps) { 2849 param[0] = FW_PARAM_PFVF(ISCSI_START); 2850 param[1] = FW_PARAM_PFVF(ISCSI_END); 2851 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2852 if (rc != 0) { 2853 device_printf(sc->dev, 2854 "failed to query iSCSI parameters: %d.\n", rc); 2855 return (rc); 2856 } 2857 sc->vres.iscsi.start = val[0]; 2858 sc->vres.iscsi.size = val[1] - val[0] + 1; 2859 } 2860 2861 /* 2862 * We've got the params we wanted to query via the firmware. Now grab 2863 * some others directly from the chip. 2864 */ 2865 rc = t4_read_chip_settings(sc); 2866 2867 return (rc); 2868 } 2869 2870 static int 2871 set_params__post_init(struct adapter *sc) 2872 { 2873 uint32_t param, val; 2874 2875 /* ask for encapsulated CPLs */ 2876 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 2877 val = 1; 2878 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2879 2880 return (0); 2881 } 2882 2883 #undef FW_PARAM_PFVF 2884 #undef FW_PARAM_DEV 2885 2886 static void 2887 t4_set_desc(struct adapter *sc) 2888 { 2889 char buf[128]; 2890 struct adapter_params *p = &sc->params; 2891 2892 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, " 2893 "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "", 2894 chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec); 2895 2896 device_set_desc_copy(sc->dev, buf); 2897 } 2898 2899 static void 2900 build_medialist(struct port_info *pi, struct ifmedia *media) 2901 { 2902 int m; 2903 2904 PORT_LOCK(pi); 2905 2906 ifmedia_removeall(media); 2907 2908 m = IFM_ETHER | IFM_FDX; 2909 2910 switch(pi->port_type) { 2911 case FW_PORT_TYPE_BT_XFI: 2912 case FW_PORT_TYPE_BT_XAUI: 2913 ifmedia_add(media, m | IFM_10G_T, 0, NULL); 2914 /* fall through */ 2915 2916 case FW_PORT_TYPE_BT_SGMII: 2917 ifmedia_add(media, m | IFM_1000_T, 0, NULL); 2918 ifmedia_add(media, m | IFM_100_TX, 0, NULL); 2919 ifmedia_add(media, IFM_ETHER | IFM_AUTO, 0, NULL); 2920 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2921 break; 2922 2923 case FW_PORT_TYPE_CX4: 2924 ifmedia_add(media, m | IFM_10G_CX4, 0, NULL); 2925 ifmedia_set(media, m | IFM_10G_CX4); 2926 break; 2927 2928 case FW_PORT_TYPE_QSFP_10G: 2929 case FW_PORT_TYPE_SFP: 2930 case FW_PORT_TYPE_FIBER_XFI: 2931 case FW_PORT_TYPE_FIBER_XAUI: 2932 switch (pi->mod_type) { 2933 2934 case FW_PORT_MOD_TYPE_LR: 2935 ifmedia_add(media, m | IFM_10G_LR, 0, NULL); 2936 ifmedia_set(media, m | IFM_10G_LR); 2937 break; 2938 2939 case FW_PORT_MOD_TYPE_SR: 2940 ifmedia_add(media, m | IFM_10G_SR, 0, NULL); 2941 ifmedia_set(media, m | IFM_10G_SR); 2942 break; 2943 2944 case FW_PORT_MOD_TYPE_LRM: 2945 ifmedia_add(media, m | IFM_10G_LRM, 0, NULL); 2946 ifmedia_set(media, m | IFM_10G_LRM); 2947 break; 2948 2949 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2950 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2951 ifmedia_add(media, m | IFM_10G_TWINAX, 0, NULL); 2952 ifmedia_set(media, m | IFM_10G_TWINAX); 2953 break; 2954 2955 case FW_PORT_MOD_TYPE_NONE: 2956 m &= ~IFM_FDX; 2957 ifmedia_add(media, m | IFM_NONE, 0, NULL); 2958 ifmedia_set(media, m | IFM_NONE); 2959 break; 2960 2961 case FW_PORT_MOD_TYPE_NA: 2962 case FW_PORT_MOD_TYPE_ER: 2963 default: 2964 device_printf(pi->dev, 2965 "unknown port_type (%d), mod_type (%d)\n", 2966 pi->port_type, pi->mod_type); 2967 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 2968 ifmedia_set(media, m | IFM_UNKNOWN); 2969 break; 2970 } 2971 break; 2972 2973 case FW_PORT_TYPE_QSFP: 2974 switch (pi->mod_type) { 2975 2976 case FW_PORT_MOD_TYPE_LR: 2977 ifmedia_add(media, m | IFM_40G_LR4, 0, NULL); 2978 ifmedia_set(media, m | IFM_40G_LR4); 2979 break; 2980 2981 case FW_PORT_MOD_TYPE_SR: 2982 ifmedia_add(media, m | IFM_40G_SR4, 0, NULL); 2983 ifmedia_set(media, m | IFM_40G_SR4); 2984 break; 2985 2986 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2987 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2988 ifmedia_add(media, m | IFM_40G_CR4, 0, NULL); 2989 ifmedia_set(media, m | IFM_40G_CR4); 2990 break; 2991 2992 case FW_PORT_MOD_TYPE_NONE: 2993 m &= ~IFM_FDX; 2994 ifmedia_add(media, m | IFM_NONE, 0, NULL); 2995 ifmedia_set(media, m | IFM_NONE); 2996 break; 2997 2998 default: 2999 device_printf(pi->dev, 3000 "unknown port_type (%d), mod_type (%d)\n", 3001 pi->port_type, pi->mod_type); 3002 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 3003 ifmedia_set(media, m | IFM_UNKNOWN); 3004 break; 3005 } 3006 break; 3007 3008 default: 3009 device_printf(pi->dev, 3010 "unknown port_type (%d), mod_type (%d)\n", pi->port_type, 3011 pi->mod_type); 3012 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 3013 ifmedia_set(media, m | IFM_UNKNOWN); 3014 break; 3015 } 3016 3017 PORT_UNLOCK(pi); 3018 } 3019 3020 #define FW_MAC_EXACT_CHUNK 7 3021 3022 /* 3023 * Program the port's XGMAC based on parameters in ifnet. The caller also 3024 * indicates which parameters should be programmed (the rest are left alone). 3025 */ 3026 int 3027 update_mac_settings(struct ifnet *ifp, int flags) 3028 { 3029 int rc = 0; 3030 struct port_info *pi = ifp->if_softc; 3031 struct adapter *sc = pi->adapter; 3032 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 3033 uint16_t viid = 0xffff; 3034 int16_t *xact_addr_filt = NULL; 3035 3036 ASSERT_SYNCHRONIZED_OP(sc); 3037 KASSERT(flags, ("%s: not told what to update.", __func__)); 3038 3039 if (ifp == pi->ifp) { 3040 viid = pi->viid; 3041 xact_addr_filt = &pi->xact_addr_filt; 3042 } 3043 #ifdef DEV_NETMAP 3044 else if (ifp == pi->nm_ifp) { 3045 viid = pi->nm_viid; 3046 xact_addr_filt = &pi->nm_xact_addr_filt; 3047 } 3048 #endif 3049 if (flags & XGMAC_MTU) 3050 mtu = ifp->if_mtu; 3051 3052 if (flags & XGMAC_PROMISC) 3053 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 3054 3055 if (flags & XGMAC_ALLMULTI) 3056 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 3057 3058 if (flags & XGMAC_VLANEX) 3059 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 3060 3061 if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) { 3062 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti, 3063 1, vlanex, false); 3064 if (rc) { 3065 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, 3066 rc); 3067 return (rc); 3068 } 3069 } 3070 3071 if (flags & XGMAC_UCADDR) { 3072 uint8_t ucaddr[ETHER_ADDR_LEN]; 3073 3074 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 3075 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr, 3076 true, true); 3077 if (rc < 0) { 3078 rc = -rc; 3079 if_printf(ifp, "change_mac failed: %d\n", rc); 3080 return (rc); 3081 } else { 3082 *xact_addr_filt = rc; 3083 rc = 0; 3084 } 3085 } 3086 3087 if (flags & XGMAC_MCADDRS) { 3088 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 3089 int del = 1; 3090 uint64_t hash = 0; 3091 struct ifmultiaddr *ifma; 3092 int i = 0, j; 3093 3094 if_maddr_rlock(ifp); 3095 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3096 if (ifma->ifma_addr->sa_family != AF_LINK) 3097 continue; 3098 mcaddr[i] = 3099 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 3100 MPASS(ETHER_IS_MULTICAST(mcaddr[i])); 3101 i++; 3102 3103 if (i == FW_MAC_EXACT_CHUNK) { 3104 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, 3105 i, mcaddr, NULL, &hash, 0); 3106 if (rc < 0) { 3107 rc = -rc; 3108 for (j = 0; j < i; j++) { 3109 if_printf(ifp, 3110 "failed to add mc address" 3111 " %02x:%02x:%02x:" 3112 "%02x:%02x:%02x rc=%d\n", 3113 mcaddr[j][0], mcaddr[j][1], 3114 mcaddr[j][2], mcaddr[j][3], 3115 mcaddr[j][4], mcaddr[j][5], 3116 rc); 3117 } 3118 goto mcfail; 3119 } 3120 del = 0; 3121 i = 0; 3122 } 3123 } 3124 if (i > 0) { 3125 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i, 3126 mcaddr, NULL, &hash, 0); 3127 if (rc < 0) { 3128 rc = -rc; 3129 for (j = 0; j < i; j++) { 3130 if_printf(ifp, 3131 "failed to add mc address" 3132 " %02x:%02x:%02x:" 3133 "%02x:%02x:%02x rc=%d\n", 3134 mcaddr[j][0], mcaddr[j][1], 3135 mcaddr[j][2], mcaddr[j][3], 3136 mcaddr[j][4], mcaddr[j][5], 3137 rc); 3138 } 3139 goto mcfail; 3140 } 3141 } 3142 3143 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0); 3144 if (rc != 0) 3145 if_printf(ifp, "failed to set mc address hash: %d", rc); 3146 mcfail: 3147 if_maddr_runlock(ifp); 3148 } 3149 3150 return (rc); 3151 } 3152 3153 /* 3154 * {begin|end}_synchronized_op must be called from the same thread. 3155 */ 3156 int 3157 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags, 3158 char *wmesg) 3159 { 3160 int rc, pri; 3161 3162 #ifdef WITNESS 3163 /* the caller thinks it's ok to sleep, but is it really? */ 3164 if (flags & SLEEP_OK) 3165 pause("t4slptst", 1); 3166 #endif 3167 3168 if (INTR_OK) 3169 pri = PCATCH; 3170 else 3171 pri = 0; 3172 3173 ADAPTER_LOCK(sc); 3174 for (;;) { 3175 3176 if (pi && IS_DOOMED(pi)) { 3177 rc = ENXIO; 3178 goto done; 3179 } 3180 3181 if (!IS_BUSY(sc)) { 3182 rc = 0; 3183 break; 3184 } 3185 3186 if (!(flags & SLEEP_OK)) { 3187 rc = EBUSY; 3188 goto done; 3189 } 3190 3191 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 3192 rc = EINTR; 3193 goto done; 3194 } 3195 } 3196 3197 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 3198 SET_BUSY(sc); 3199 #ifdef INVARIANTS 3200 sc->last_op = wmesg; 3201 sc->last_op_thr = curthread; 3202 sc->last_op_flags = flags; 3203 #endif 3204 3205 done: 3206 if (!(flags & HOLD_LOCK) || rc) 3207 ADAPTER_UNLOCK(sc); 3208 3209 return (rc); 3210 } 3211 3212 /* 3213 * {begin|end}_synchronized_op must be called from the same thread. 3214 */ 3215 void 3216 end_synchronized_op(struct adapter *sc, int flags) 3217 { 3218 3219 if (flags & LOCK_HELD) 3220 ADAPTER_LOCK_ASSERT_OWNED(sc); 3221 else 3222 ADAPTER_LOCK(sc); 3223 3224 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 3225 CLR_BUSY(sc); 3226 wakeup(&sc->flags); 3227 ADAPTER_UNLOCK(sc); 3228 } 3229 3230 static int 3231 cxgbe_init_synchronized(struct port_info *pi) 3232 { 3233 struct adapter *sc = pi->adapter; 3234 struct ifnet *ifp = pi->ifp; 3235 int rc = 0, i; 3236 struct sge_txq *txq; 3237 3238 ASSERT_SYNCHRONIZED_OP(sc); 3239 3240 if (isset(&sc->open_device_map, pi->port_id)) { 3241 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 3242 ("mismatch between open_device_map and if_drv_flags")); 3243 return (0); /* already running */ 3244 } 3245 3246 if (!(sc->flags & FULL_INIT_DONE) && 3247 ((rc = adapter_full_init(sc)) != 0)) 3248 return (rc); /* error message displayed already */ 3249 3250 if (!(pi->flags & PORT_INIT_DONE) && 3251 ((rc = port_full_init(pi)) != 0)) 3252 return (rc); /* error message displayed already */ 3253 3254 rc = update_mac_settings(ifp, XGMAC_ALL); 3255 if (rc) 3256 goto done; /* error message displayed already */ 3257 3258 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 3259 if (rc != 0) { 3260 if_printf(ifp, "enable_vi failed: %d\n", rc); 3261 goto done; 3262 } 3263 3264 /* 3265 * Can't fail from this point onwards. Review cxgbe_uninit_synchronized 3266 * if this changes. 3267 */ 3268 3269 for_each_txq(pi, i, txq) { 3270 TXQ_LOCK(txq); 3271 txq->eq.flags |= EQ_ENABLED; 3272 TXQ_UNLOCK(txq); 3273 } 3274 3275 /* 3276 * The first iq of the first port to come up is used for tracing. 3277 */ 3278 if (sc->traceq < 0) { 3279 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id; 3280 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : 3281 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | 3282 V_QUEUENUMBER(sc->traceq)); 3283 pi->flags |= HAS_TRACEQ; 3284 } 3285 3286 /* all ok */ 3287 setbit(&sc->open_device_map, pi->port_id); 3288 PORT_LOCK(pi); 3289 ifp->if_drv_flags |= IFF_DRV_RUNNING; 3290 PORT_UNLOCK(pi); 3291 3292 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 3293 done: 3294 if (rc != 0) 3295 cxgbe_uninit_synchronized(pi); 3296 3297 return (rc); 3298 } 3299 3300 /* 3301 * Idempotent. 3302 */ 3303 static int 3304 cxgbe_uninit_synchronized(struct port_info *pi) 3305 { 3306 struct adapter *sc = pi->adapter; 3307 struct ifnet *ifp = pi->ifp; 3308 int rc, i; 3309 struct sge_txq *txq; 3310 3311 ASSERT_SYNCHRONIZED_OP(sc); 3312 3313 if (!(pi->flags & PORT_INIT_DONE)) { 3314 KASSERT(!(ifp->if_drv_flags & IFF_DRV_RUNNING), 3315 ("uninited port is running")); 3316 return (0); 3317 } 3318 3319 /* 3320 * Disable the VI so that all its data in either direction is discarded 3321 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 3322 * tick) intact as the TP can deliver negative advice or data that it's 3323 * holding in its RAM (for an offloaded connection) even after the VI is 3324 * disabled. 3325 */ 3326 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 3327 if (rc) { 3328 if_printf(ifp, "disable_vi failed: %d\n", rc); 3329 return (rc); 3330 } 3331 3332 for_each_txq(pi, i, txq) { 3333 TXQ_LOCK(txq); 3334 txq->eq.flags &= ~EQ_ENABLED; 3335 TXQ_UNLOCK(txq); 3336 } 3337 3338 clrbit(&sc->open_device_map, pi->port_id); 3339 PORT_LOCK(pi); 3340 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3341 PORT_UNLOCK(pi); 3342 3343 pi->link_cfg.link_ok = 0; 3344 pi->link_cfg.speed = 0; 3345 pi->linkdnrc = -1; 3346 t4_os_link_changed(sc, pi->port_id, 0, -1); 3347 3348 return (0); 3349 } 3350 3351 /* 3352 * It is ok for this function to fail midway and return right away. t4_detach 3353 * will walk the entire sc->irq list and clean up whatever is valid. 3354 */ 3355 static int 3356 setup_intr_handlers(struct adapter *sc) 3357 { 3358 int rc, rid, p, q; 3359 char s[8]; 3360 struct irq *irq; 3361 struct port_info *pi; 3362 struct sge_rxq *rxq; 3363 #ifdef TCP_OFFLOAD 3364 struct sge_ofld_rxq *ofld_rxq; 3365 #endif 3366 #ifdef DEV_NETMAP 3367 struct sge_nm_rxq *nm_rxq; 3368 #endif 3369 3370 /* 3371 * Setup interrupts. 3372 */ 3373 irq = &sc->irq[0]; 3374 rid = sc->intr_type == INTR_INTX ? 0 : 1; 3375 if (sc->intr_count == 1) 3376 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all")); 3377 3378 /* Multiple interrupts. */ 3379 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 3380 ("%s: too few intr.", __func__)); 3381 3382 /* The first one is always error intr */ 3383 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 3384 if (rc != 0) 3385 return (rc); 3386 irq++; 3387 rid++; 3388 3389 /* The second one is always the firmware event queue */ 3390 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt"); 3391 if (rc != 0) 3392 return (rc); 3393 irq++; 3394 rid++; 3395 3396 for_each_port(sc, p) { 3397 pi = sc->port[p]; 3398 3399 if (pi->flags & INTR_RXQ) { 3400 for_each_rxq(pi, q, rxq) { 3401 snprintf(s, sizeof(s), "%d.%d", p, q); 3402 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq, 3403 s); 3404 if (rc != 0) 3405 return (rc); 3406 irq++; 3407 rid++; 3408 } 3409 } 3410 #ifdef TCP_OFFLOAD 3411 if (pi->flags & INTR_OFLD_RXQ) { 3412 for_each_ofld_rxq(pi, q, ofld_rxq) { 3413 snprintf(s, sizeof(s), "%d,%d", p, q); 3414 rc = t4_alloc_irq(sc, irq, rid, t4_intr, 3415 ofld_rxq, s); 3416 if (rc != 0) 3417 return (rc); 3418 irq++; 3419 rid++; 3420 } 3421 } 3422 #endif 3423 #ifdef DEV_NETMAP 3424 if (pi->flags & INTR_NM_RXQ) { 3425 for_each_nm_rxq(pi, q, nm_rxq) { 3426 snprintf(s, sizeof(s), "%d-%d", p, q); 3427 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr, 3428 nm_rxq, s); 3429 if (rc != 0) 3430 return (rc); 3431 irq++; 3432 rid++; 3433 } 3434 } 3435 #endif 3436 } 3437 MPASS(irq == &sc->irq[sc->intr_count]); 3438 3439 return (0); 3440 } 3441 3442 int 3443 adapter_full_init(struct adapter *sc) 3444 { 3445 int rc, i; 3446 3447 ASSERT_SYNCHRONIZED_OP(sc); 3448 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3449 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 3450 ("%s: FULL_INIT_DONE already", __func__)); 3451 3452 /* 3453 * queues that belong to the adapter (not any particular port). 3454 */ 3455 rc = t4_setup_adapter_queues(sc); 3456 if (rc != 0) 3457 goto done; 3458 3459 for (i = 0; i < nitems(sc->tq); i++) { 3460 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 3461 taskqueue_thread_enqueue, &sc->tq[i]); 3462 if (sc->tq[i] == NULL) { 3463 device_printf(sc->dev, 3464 "failed to allocate task queue %d\n", i); 3465 rc = ENOMEM; 3466 goto done; 3467 } 3468 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3469 device_get_nameunit(sc->dev), i); 3470 } 3471 3472 t4_intr_enable(sc); 3473 sc->flags |= FULL_INIT_DONE; 3474 done: 3475 if (rc != 0) 3476 adapter_full_uninit(sc); 3477 3478 return (rc); 3479 } 3480 3481 int 3482 adapter_full_uninit(struct adapter *sc) 3483 { 3484 int i; 3485 3486 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3487 3488 t4_teardown_adapter_queues(sc); 3489 3490 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3491 taskqueue_free(sc->tq[i]); 3492 sc->tq[i] = NULL; 3493 } 3494 3495 sc->flags &= ~FULL_INIT_DONE; 3496 3497 return (0); 3498 } 3499 3500 #ifdef RSS 3501 #define SUPPORTED_RSS_HASHTYPES (RSS_HASHTYPE_RSS_IPV4 | \ 3502 RSS_HASHTYPE_RSS_TCP_IPV4 | RSS_HASHTYPE_RSS_IPV6 | \ 3503 RSS_HASHTYPE_RSS_TCP_IPV6 | RSS_HASHTYPE_RSS_UDP_IPV4 | \ 3504 RSS_HASHTYPE_RSS_UDP_IPV6) 3505 3506 /* Translates kernel hash types to hardware. */ 3507 static int 3508 hashconfig_to_hashen(int hashconfig) 3509 { 3510 int hashen = 0; 3511 3512 if (hashconfig & RSS_HASHTYPE_RSS_IPV4) 3513 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN; 3514 if (hashconfig & RSS_HASHTYPE_RSS_IPV6) 3515 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN; 3516 if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV4) { 3517 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN | 3518 F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN; 3519 } 3520 if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV6) { 3521 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN | 3522 F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN; 3523 } 3524 if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV4) 3525 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN; 3526 if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV6) 3527 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN; 3528 3529 return (hashen); 3530 } 3531 3532 /* Translates hardware hash types to kernel. */ 3533 static int 3534 hashen_to_hashconfig(int hashen) 3535 { 3536 int hashconfig = 0; 3537 3538 if (hashen & F_FW_RSS_VI_CONFIG_CMD_UDPEN) { 3539 /* 3540 * If UDP hashing was enabled it must have been enabled for 3541 * either IPv4 or IPv6 (inclusive or). Enabling UDP without 3542 * enabling any 4-tuple hash is nonsense configuration. 3543 */ 3544 MPASS(hashen & (F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN | 3545 F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)); 3546 3547 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) 3548 hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV4; 3549 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) 3550 hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV6; 3551 } 3552 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) 3553 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV4; 3554 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) 3555 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV6; 3556 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN) 3557 hashconfig |= RSS_HASHTYPE_RSS_IPV4; 3558 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN) 3559 hashconfig |= RSS_HASHTYPE_RSS_IPV6; 3560 3561 return (hashconfig); 3562 } 3563 #endif 3564 3565 int 3566 port_full_init(struct port_info *pi) 3567 { 3568 struct adapter *sc = pi->adapter; 3569 struct ifnet *ifp = pi->ifp; 3570 uint16_t *rss; 3571 struct sge_rxq *rxq; 3572 int rc, i, j, hashen; 3573 #ifdef RSS 3574 int nbuckets = rss_getnumbuckets(); 3575 int hashconfig = rss_gethashconfig(); 3576 int extra; 3577 uint32_t raw_rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; 3578 uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; 3579 #endif 3580 3581 ASSERT_SYNCHRONIZED_OP(sc); 3582 KASSERT((pi->flags & PORT_INIT_DONE) == 0, 3583 ("%s: PORT_INIT_DONE already", __func__)); 3584 3585 sysctl_ctx_init(&pi->ctx); 3586 pi->flags |= PORT_SYSCTL_CTX; 3587 3588 /* 3589 * Allocate tx/rx/fl queues for this port. 3590 */ 3591 rc = t4_setup_port_queues(pi); 3592 if (rc != 0) 3593 goto done; /* error message displayed already */ 3594 3595 /* 3596 * Setup RSS for this port. Save a copy of the RSS table for later use. 3597 */ 3598 if (pi->nrxq > pi->rss_size) { 3599 if_printf(ifp, "nrxq (%d) > hw RSS table size (%d); " 3600 "some queues will never receive traffic.\n", pi->nrxq, 3601 pi->rss_size); 3602 } else if (pi->rss_size % pi->nrxq) { 3603 if_printf(ifp, "nrxq (%d), hw RSS table size (%d); " 3604 "expect uneven traffic distribution.\n", pi->nrxq, 3605 pi->rss_size); 3606 } 3607 #ifdef RSS 3608 MPASS(RSS_KEYSIZE == 40); 3609 if (pi->nrxq != nbuckets) { 3610 if_printf(ifp, "nrxq (%d) != kernel RSS buckets (%d);" 3611 "performance will be impacted.\n", pi->nrxq, nbuckets); 3612 } 3613 3614 rss_getkey((void *)&raw_rss_key[0]); 3615 for (i = 0; i < nitems(rss_key); i++) { 3616 rss_key[i] = htobe32(raw_rss_key[nitems(rss_key) - 1 - i]); 3617 } 3618 t4_write_rss_key(sc, (void *)&rss_key[0], -1); 3619 #endif 3620 rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); 3621 for (i = 0; i < pi->rss_size;) { 3622 #ifdef RSS 3623 j = rss_get_indirection_to_bucket(i); 3624 j %= pi->nrxq; 3625 rxq = &sc->sge.rxq[pi->first_rxq + j]; 3626 rss[i++] = rxq->iq.abs_id; 3627 #else 3628 for_each_rxq(pi, j, rxq) { 3629 rss[i++] = rxq->iq.abs_id; 3630 if (i == pi->rss_size) 3631 break; 3632 } 3633 #endif 3634 } 3635 3636 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss, 3637 pi->rss_size); 3638 if (rc != 0) { 3639 if_printf(ifp, "rss_config failed: %d\n", rc); 3640 goto done; 3641 } 3642 3643 #ifdef RSS 3644 hashen = hashconfig_to_hashen(hashconfig); 3645 3646 /* 3647 * We may have had to enable some hashes even though the global config 3648 * wants them disabled. This is a potential problem that must be 3649 * reported to the user. 3650 */ 3651 extra = hashen_to_hashconfig(hashen) ^ hashconfig; 3652 3653 /* 3654 * If we consider only the supported hash types, then the enabled hashes 3655 * are a superset of the requested hashes. In other words, there cannot 3656 * be any supported hash that was requested but not enabled, but there 3657 * can be hashes that were not requested but had to be enabled. 3658 */ 3659 extra &= SUPPORTED_RSS_HASHTYPES; 3660 MPASS((extra & hashconfig) == 0); 3661 3662 if (extra) { 3663 if_printf(ifp, 3664 "global RSS config (0x%x) cannot be accomodated.\n", 3665 hashconfig); 3666 } 3667 if (extra & RSS_HASHTYPE_RSS_IPV4) 3668 if_printf(ifp, "IPv4 2-tuple hashing forced on.\n"); 3669 if (extra & RSS_HASHTYPE_RSS_TCP_IPV4) 3670 if_printf(ifp, "TCP/IPv4 4-tuple hashing forced on.\n"); 3671 if (extra & RSS_HASHTYPE_RSS_IPV6) 3672 if_printf(ifp, "IPv6 2-tuple hashing forced on.\n"); 3673 if (extra & RSS_HASHTYPE_RSS_TCP_IPV6) 3674 if_printf(ifp, "TCP/IPv6 4-tuple hashing forced on.\n"); 3675 if (extra & RSS_HASHTYPE_RSS_UDP_IPV4) 3676 if_printf(ifp, "UDP/IPv4 4-tuple hashing forced on.\n"); 3677 if (extra & RSS_HASHTYPE_RSS_UDP_IPV6) 3678 if_printf(ifp, "UDP/IPv6 4-tuple hashing forced on.\n"); 3679 #else 3680 hashen = F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN | 3681 F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN | 3682 F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN | 3683 F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN | F_FW_RSS_VI_CONFIG_CMD_UDPEN; 3684 #endif 3685 rc = -t4_config_vi_rss(sc, sc->mbox, pi->viid, hashen, rss[0]); 3686 if (rc != 0) { 3687 if_printf(ifp, "rss hash/defaultq config failed: %d\n", rc); 3688 goto done; 3689 } 3690 3691 pi->rss = rss; 3692 pi->flags |= PORT_INIT_DONE; 3693 done: 3694 if (rc != 0) 3695 port_full_uninit(pi); 3696 3697 return (rc); 3698 } 3699 3700 /* 3701 * Idempotent. 3702 */ 3703 int 3704 port_full_uninit(struct port_info *pi) 3705 { 3706 struct adapter *sc = pi->adapter; 3707 int i; 3708 struct sge_rxq *rxq; 3709 struct sge_txq *txq; 3710 #ifdef TCP_OFFLOAD 3711 struct sge_ofld_rxq *ofld_rxq; 3712 struct sge_wrq *ofld_txq; 3713 #endif 3714 3715 if (pi->flags & PORT_INIT_DONE) { 3716 3717 /* Need to quiesce queues. */ 3718 3719 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]); 3720 3721 for_each_txq(pi, i, txq) { 3722 quiesce_txq(sc, txq); 3723 } 3724 3725 #ifdef TCP_OFFLOAD 3726 for_each_ofld_txq(pi, i, ofld_txq) { 3727 quiesce_wrq(sc, ofld_txq); 3728 } 3729 #endif 3730 3731 for_each_rxq(pi, i, rxq) { 3732 quiesce_iq(sc, &rxq->iq); 3733 quiesce_fl(sc, &rxq->fl); 3734 } 3735 3736 #ifdef TCP_OFFLOAD 3737 for_each_ofld_rxq(pi, i, ofld_rxq) { 3738 quiesce_iq(sc, &ofld_rxq->iq); 3739 quiesce_fl(sc, &ofld_rxq->fl); 3740 } 3741 #endif 3742 free(pi->rss, M_CXGBE); 3743 } 3744 3745 t4_teardown_port_queues(pi); 3746 pi->flags &= ~PORT_INIT_DONE; 3747 3748 return (0); 3749 } 3750 3751 static void 3752 quiesce_txq(struct adapter *sc, struct sge_txq *txq) 3753 { 3754 struct sge_eq *eq = &txq->eq; 3755 struct sge_qstat *spg = (void *)&eq->desc[eq->sidx]; 3756 3757 (void) sc; /* unused */ 3758 3759 #ifdef INVARIANTS 3760 TXQ_LOCK(txq); 3761 MPASS((eq->flags & EQ_ENABLED) == 0); 3762 TXQ_UNLOCK(txq); 3763 #endif 3764 3765 /* Wait for the mp_ring to empty. */ 3766 while (!mp_ring_is_idle(txq->r)) { 3767 mp_ring_check_drainage(txq->r, 0); 3768 pause("rquiesce", 1); 3769 } 3770 3771 /* Then wait for the hardware to finish. */ 3772 while (spg->cidx != htobe16(eq->pidx)) 3773 pause("equiesce", 1); 3774 3775 /* Finally, wait for the driver to reclaim all descriptors. */ 3776 while (eq->cidx != eq->pidx) 3777 pause("dquiesce", 1); 3778 } 3779 3780 static void 3781 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq) 3782 { 3783 3784 /* XXXTX */ 3785 } 3786 3787 static void 3788 quiesce_iq(struct adapter *sc, struct sge_iq *iq) 3789 { 3790 (void) sc; /* unused */ 3791 3792 /* Synchronize with the interrupt handler */ 3793 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 3794 pause("iqfree", 1); 3795 } 3796 3797 static void 3798 quiesce_fl(struct adapter *sc, struct sge_fl *fl) 3799 { 3800 mtx_lock(&sc->sfl_lock); 3801 FL_LOCK(fl); 3802 fl->flags |= FL_DOOMED; 3803 FL_UNLOCK(fl); 3804 mtx_unlock(&sc->sfl_lock); 3805 3806 callout_drain(&sc->sfl_callout); 3807 KASSERT((fl->flags & FL_STARVING) == 0, 3808 ("%s: still starving", __func__)); 3809 } 3810 3811 static int 3812 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 3813 driver_intr_t *handler, void *arg, char *name) 3814 { 3815 int rc; 3816 3817 irq->rid = rid; 3818 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 3819 RF_SHAREABLE | RF_ACTIVE); 3820 if (irq->res == NULL) { 3821 device_printf(sc->dev, 3822 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 3823 return (ENOMEM); 3824 } 3825 3826 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 3827 NULL, handler, arg, &irq->tag); 3828 if (rc != 0) { 3829 device_printf(sc->dev, 3830 "failed to setup interrupt for rid %d, name %s: %d\n", 3831 rid, name, rc); 3832 } else if (name) 3833 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 3834 3835 return (rc); 3836 } 3837 3838 static int 3839 t4_free_irq(struct adapter *sc, struct irq *irq) 3840 { 3841 if (irq->tag) 3842 bus_teardown_intr(sc->dev, irq->res, irq->tag); 3843 if (irq->res) 3844 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 3845 3846 bzero(irq, sizeof(*irq)); 3847 3848 return (0); 3849 } 3850 3851 static void 3852 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 3853 unsigned int end) 3854 { 3855 uint32_t *p = (uint32_t *)(buf + start); 3856 3857 for ( ; start <= end; start += sizeof(uint32_t)) 3858 *p++ = t4_read_reg(sc, start); 3859 } 3860 3861 static void 3862 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 3863 { 3864 int i, n; 3865 const unsigned int *reg_ranges; 3866 static const unsigned int t4_reg_ranges[] = { 3867 0x1008, 0x1108, 3868 0x1180, 0x11b4, 3869 0x11fc, 0x123c, 3870 0x1300, 0x173c, 3871 0x1800, 0x18fc, 3872 0x3000, 0x30d8, 3873 0x30e0, 0x5924, 3874 0x5960, 0x59d4, 3875 0x5a00, 0x5af8, 3876 0x6000, 0x6098, 3877 0x6100, 0x6150, 3878 0x6200, 0x6208, 3879 0x6240, 0x6248, 3880 0x6280, 0x6338, 3881 0x6370, 0x638c, 3882 0x6400, 0x643c, 3883 0x6500, 0x6524, 3884 0x6a00, 0x6a38, 3885 0x6a60, 0x6a78, 3886 0x6b00, 0x6b84, 3887 0x6bf0, 0x6c84, 3888 0x6cf0, 0x6d84, 3889 0x6df0, 0x6e84, 3890 0x6ef0, 0x6f84, 3891 0x6ff0, 0x7084, 3892 0x70f0, 0x7184, 3893 0x71f0, 0x7284, 3894 0x72f0, 0x7384, 3895 0x73f0, 0x7450, 3896 0x7500, 0x7530, 3897 0x7600, 0x761c, 3898 0x7680, 0x76cc, 3899 0x7700, 0x7798, 3900 0x77c0, 0x77fc, 3901 0x7900, 0x79fc, 3902 0x7b00, 0x7c38, 3903 0x7d00, 0x7efc, 3904 0x8dc0, 0x8e1c, 3905 0x8e30, 0x8e78, 3906 0x8ea0, 0x8f6c, 3907 0x8fc0, 0x9074, 3908 0x90fc, 0x90fc, 3909 0x9400, 0x9458, 3910 0x9600, 0x96bc, 3911 0x9800, 0x9808, 3912 0x9820, 0x983c, 3913 0x9850, 0x9864, 3914 0x9c00, 0x9c6c, 3915 0x9c80, 0x9cec, 3916 0x9d00, 0x9d6c, 3917 0x9d80, 0x9dec, 3918 0x9e00, 0x9e6c, 3919 0x9e80, 0x9eec, 3920 0x9f00, 0x9f6c, 3921 0x9f80, 0x9fec, 3922 0xd004, 0xd03c, 3923 0xdfc0, 0xdfe0, 3924 0xe000, 0xea7c, 3925 0xf000, 0x11110, 3926 0x11118, 0x11190, 3927 0x19040, 0x1906c, 3928 0x19078, 0x19080, 3929 0x1908c, 0x19124, 3930 0x19150, 0x191b0, 3931 0x191d0, 0x191e8, 3932 0x19238, 0x1924c, 3933 0x193f8, 0x19474, 3934 0x19490, 0x194f8, 3935 0x19800, 0x19f30, 3936 0x1a000, 0x1a06c, 3937 0x1a0b0, 0x1a120, 3938 0x1a128, 0x1a138, 3939 0x1a190, 0x1a1c4, 3940 0x1a1fc, 0x1a1fc, 3941 0x1e040, 0x1e04c, 3942 0x1e284, 0x1e28c, 3943 0x1e2c0, 0x1e2c0, 3944 0x1e2e0, 0x1e2e0, 3945 0x1e300, 0x1e384, 3946 0x1e3c0, 0x1e3c8, 3947 0x1e440, 0x1e44c, 3948 0x1e684, 0x1e68c, 3949 0x1e6c0, 0x1e6c0, 3950 0x1e6e0, 0x1e6e0, 3951 0x1e700, 0x1e784, 3952 0x1e7c0, 0x1e7c8, 3953 0x1e840, 0x1e84c, 3954 0x1ea84, 0x1ea8c, 3955 0x1eac0, 0x1eac0, 3956 0x1eae0, 0x1eae0, 3957 0x1eb00, 0x1eb84, 3958 0x1ebc0, 0x1ebc8, 3959 0x1ec40, 0x1ec4c, 3960 0x1ee84, 0x1ee8c, 3961 0x1eec0, 0x1eec0, 3962 0x1eee0, 0x1eee0, 3963 0x1ef00, 0x1ef84, 3964 0x1efc0, 0x1efc8, 3965 0x1f040, 0x1f04c, 3966 0x1f284, 0x1f28c, 3967 0x1f2c0, 0x1f2c0, 3968 0x1f2e0, 0x1f2e0, 3969 0x1f300, 0x1f384, 3970 0x1f3c0, 0x1f3c8, 3971 0x1f440, 0x1f44c, 3972 0x1f684, 0x1f68c, 3973 0x1f6c0, 0x1f6c0, 3974 0x1f6e0, 0x1f6e0, 3975 0x1f700, 0x1f784, 3976 0x1f7c0, 0x1f7c8, 3977 0x1f840, 0x1f84c, 3978 0x1fa84, 0x1fa8c, 3979 0x1fac0, 0x1fac0, 3980 0x1fae0, 0x1fae0, 3981 0x1fb00, 0x1fb84, 3982 0x1fbc0, 0x1fbc8, 3983 0x1fc40, 0x1fc4c, 3984 0x1fe84, 0x1fe8c, 3985 0x1fec0, 0x1fec0, 3986 0x1fee0, 0x1fee0, 3987 0x1ff00, 0x1ff84, 3988 0x1ffc0, 0x1ffc8, 3989 0x20000, 0x2002c, 3990 0x20100, 0x2013c, 3991 0x20190, 0x201c8, 3992 0x20200, 0x20318, 3993 0x20400, 0x20528, 3994 0x20540, 0x20614, 3995 0x21000, 0x21040, 3996 0x2104c, 0x21060, 3997 0x210c0, 0x210ec, 3998 0x21200, 0x21268, 3999 0x21270, 0x21284, 4000 0x212fc, 0x21388, 4001 0x21400, 0x21404, 4002 0x21500, 0x21518, 4003 0x2152c, 0x2153c, 4004 0x21550, 0x21554, 4005 0x21600, 0x21600, 4006 0x21608, 0x21628, 4007 0x21630, 0x2163c, 4008 0x21700, 0x2171c, 4009 0x21780, 0x2178c, 4010 0x21800, 0x21c38, 4011 0x21c80, 0x21d7c, 4012 0x21e00, 0x21e04, 4013 0x22000, 0x2202c, 4014 0x22100, 0x2213c, 4015 0x22190, 0x221c8, 4016 0x22200, 0x22318, 4017 0x22400, 0x22528, 4018 0x22540, 0x22614, 4019 0x23000, 0x23040, 4020 0x2304c, 0x23060, 4021 0x230c0, 0x230ec, 4022 0x23200, 0x23268, 4023 0x23270, 0x23284, 4024 0x232fc, 0x23388, 4025 0x23400, 0x23404, 4026 0x23500, 0x23518, 4027 0x2352c, 0x2353c, 4028 0x23550, 0x23554, 4029 0x23600, 0x23600, 4030 0x23608, 0x23628, 4031 0x23630, 0x2363c, 4032 0x23700, 0x2371c, 4033 0x23780, 0x2378c, 4034 0x23800, 0x23c38, 4035 0x23c80, 0x23d7c, 4036 0x23e00, 0x23e04, 4037 0x24000, 0x2402c, 4038 0x24100, 0x2413c, 4039 0x24190, 0x241c8, 4040 0x24200, 0x24318, 4041 0x24400, 0x24528, 4042 0x24540, 0x24614, 4043 0x25000, 0x25040, 4044 0x2504c, 0x25060, 4045 0x250c0, 0x250ec, 4046 0x25200, 0x25268, 4047 0x25270, 0x25284, 4048 0x252fc, 0x25388, 4049 0x25400, 0x25404, 4050 0x25500, 0x25518, 4051 0x2552c, 0x2553c, 4052 0x25550, 0x25554, 4053 0x25600, 0x25600, 4054 0x25608, 0x25628, 4055 0x25630, 0x2563c, 4056 0x25700, 0x2571c, 4057 0x25780, 0x2578c, 4058 0x25800, 0x25c38, 4059 0x25c80, 0x25d7c, 4060 0x25e00, 0x25e04, 4061 0x26000, 0x2602c, 4062 0x26100, 0x2613c, 4063 0x26190, 0x261c8, 4064 0x26200, 0x26318, 4065 0x26400, 0x26528, 4066 0x26540, 0x26614, 4067 0x27000, 0x27040, 4068 0x2704c, 0x27060, 4069 0x270c0, 0x270ec, 4070 0x27200, 0x27268, 4071 0x27270, 0x27284, 4072 0x272fc, 0x27388, 4073 0x27400, 0x27404, 4074 0x27500, 0x27518, 4075 0x2752c, 0x2753c, 4076 0x27550, 0x27554, 4077 0x27600, 0x27600, 4078 0x27608, 0x27628, 4079 0x27630, 0x2763c, 4080 0x27700, 0x2771c, 4081 0x27780, 0x2778c, 4082 0x27800, 0x27c38, 4083 0x27c80, 0x27d7c, 4084 0x27e00, 0x27e04 4085 }; 4086 static const unsigned int t5_reg_ranges[] = { 4087 0x1008, 0x1148, 4088 0x1180, 0x11b4, 4089 0x11fc, 0x123c, 4090 0x1280, 0x173c, 4091 0x1800, 0x18fc, 4092 0x3000, 0x3028, 4093 0x3060, 0x30d8, 4094 0x30e0, 0x30fc, 4095 0x3140, 0x357c, 4096 0x35a8, 0x35cc, 4097 0x35ec, 0x35ec, 4098 0x3600, 0x5624, 4099 0x56cc, 0x575c, 4100 0x580c, 0x5814, 4101 0x5890, 0x58bc, 4102 0x5940, 0x59dc, 4103 0x59fc, 0x5a18, 4104 0x5a60, 0x5a9c, 4105 0x5b94, 0x5bfc, 4106 0x6000, 0x6040, 4107 0x6058, 0x614c, 4108 0x7700, 0x7798, 4109 0x77c0, 0x78fc, 4110 0x7b00, 0x7c54, 4111 0x7d00, 0x7efc, 4112 0x8dc0, 0x8de0, 4113 0x8df8, 0x8e84, 4114 0x8ea0, 0x8f84, 4115 0x8fc0, 0x90f8, 4116 0x9400, 0x9470, 4117 0x9600, 0x96f4, 4118 0x9800, 0x9808, 4119 0x9820, 0x983c, 4120 0x9850, 0x9864, 4121 0x9c00, 0x9c6c, 4122 0x9c80, 0x9cec, 4123 0x9d00, 0x9d6c, 4124 0x9d80, 0x9dec, 4125 0x9e00, 0x9e6c, 4126 0x9e80, 0x9eec, 4127 0x9f00, 0x9f6c, 4128 0x9f80, 0xa020, 4129 0xd004, 0xd03c, 4130 0xdfc0, 0xdfe0, 4131 0xe000, 0x11088, 4132 0x1109c, 0x11110, 4133 0x11118, 0x1117c, 4134 0x11190, 0x11204, 4135 0x19040, 0x1906c, 4136 0x19078, 0x19080, 4137 0x1908c, 0x19124, 4138 0x19150, 0x191b0, 4139 0x191d0, 0x191e8, 4140 0x19238, 0x19290, 4141 0x193f8, 0x19474, 4142 0x19490, 0x194cc, 4143 0x194f0, 0x194f8, 4144 0x19c00, 0x19c60, 4145 0x19c94, 0x19e10, 4146 0x19e50, 0x19f34, 4147 0x19f40, 0x19f50, 4148 0x19f90, 0x19fe4, 4149 0x1a000, 0x1a06c, 4150 0x1a0b0, 0x1a120, 4151 0x1a128, 0x1a138, 4152 0x1a190, 0x1a1c4, 4153 0x1a1fc, 0x1a1fc, 4154 0x1e008, 0x1e00c, 4155 0x1e040, 0x1e04c, 4156 0x1e284, 0x1e290, 4157 0x1e2c0, 0x1e2c0, 4158 0x1e2e0, 0x1e2e0, 4159 0x1e300, 0x1e384, 4160 0x1e3c0, 0x1e3c8, 4161 0x1e408, 0x1e40c, 4162 0x1e440, 0x1e44c, 4163 0x1e684, 0x1e690, 4164 0x1e6c0, 0x1e6c0, 4165 0x1e6e0, 0x1e6e0, 4166 0x1e700, 0x1e784, 4167 0x1e7c0, 0x1e7c8, 4168 0x1e808, 0x1e80c, 4169 0x1e840, 0x1e84c, 4170 0x1ea84, 0x1ea90, 4171 0x1eac0, 0x1eac0, 4172 0x1eae0, 0x1eae0, 4173 0x1eb00, 0x1eb84, 4174 0x1ebc0, 0x1ebc8, 4175 0x1ec08, 0x1ec0c, 4176 0x1ec40, 0x1ec4c, 4177 0x1ee84, 0x1ee90, 4178 0x1eec0, 0x1eec0, 4179 0x1eee0, 0x1eee0, 4180 0x1ef00, 0x1ef84, 4181 0x1efc0, 0x1efc8, 4182 0x1f008, 0x1f00c, 4183 0x1f040, 0x1f04c, 4184 0x1f284, 0x1f290, 4185 0x1f2c0, 0x1f2c0, 4186 0x1f2e0, 0x1f2e0, 4187 0x1f300, 0x1f384, 4188 0x1f3c0, 0x1f3c8, 4189 0x1f408, 0x1f40c, 4190 0x1f440, 0x1f44c, 4191 0x1f684, 0x1f690, 4192 0x1f6c0, 0x1f6c0, 4193 0x1f6e0, 0x1f6e0, 4194 0x1f700, 0x1f784, 4195 0x1f7c0, 0x1f7c8, 4196 0x1f808, 0x1f80c, 4197 0x1f840, 0x1f84c, 4198 0x1fa84, 0x1fa90, 4199 0x1fac0, 0x1fac0, 4200 0x1fae0, 0x1fae0, 4201 0x1fb00, 0x1fb84, 4202 0x1fbc0, 0x1fbc8, 4203 0x1fc08, 0x1fc0c, 4204 0x1fc40, 0x1fc4c, 4205 0x1fe84, 0x1fe90, 4206 0x1fec0, 0x1fec0, 4207 0x1fee0, 0x1fee0, 4208 0x1ff00, 0x1ff84, 4209 0x1ffc0, 0x1ffc8, 4210 0x30000, 0x30030, 4211 0x30100, 0x30144, 4212 0x30190, 0x301d0, 4213 0x30200, 0x30318, 4214 0x30400, 0x3052c, 4215 0x30540, 0x3061c, 4216 0x30800, 0x30834, 4217 0x308c0, 0x30908, 4218 0x30910, 0x309ac, 4219 0x30a00, 0x30a2c, 4220 0x30a44, 0x30a50, 4221 0x30a74, 0x30c24, 4222 0x30d00, 0x30d00, 4223 0x30d08, 0x30d14, 4224 0x30d1c, 0x30d20, 4225 0x30d3c, 0x30d50, 4226 0x31200, 0x3120c, 4227 0x31220, 0x31220, 4228 0x31240, 0x31240, 4229 0x31600, 0x3160c, 4230 0x31a00, 0x31a1c, 4231 0x31e00, 0x31e20, 4232 0x31e38, 0x31e3c, 4233 0x31e80, 0x31e80, 4234 0x31e88, 0x31ea8, 4235 0x31eb0, 0x31eb4, 4236 0x31ec8, 0x31ed4, 4237 0x31fb8, 0x32004, 4238 0x32200, 0x32200, 4239 0x32208, 0x32240, 4240 0x32248, 0x32280, 4241 0x32288, 0x322c0, 4242 0x322c8, 0x322fc, 4243 0x32600, 0x32630, 4244 0x32a00, 0x32abc, 4245 0x32b00, 0x32b70, 4246 0x33000, 0x33048, 4247 0x33060, 0x3309c, 4248 0x330f0, 0x33148, 4249 0x33160, 0x3319c, 4250 0x331f0, 0x332e4, 4251 0x332f8, 0x333e4, 4252 0x333f8, 0x33448, 4253 0x33460, 0x3349c, 4254 0x334f0, 0x33548, 4255 0x33560, 0x3359c, 4256 0x335f0, 0x336e4, 4257 0x336f8, 0x337e4, 4258 0x337f8, 0x337fc, 4259 0x33814, 0x33814, 4260 0x3382c, 0x3382c, 4261 0x33880, 0x3388c, 4262 0x338e8, 0x338ec, 4263 0x33900, 0x33948, 4264 0x33960, 0x3399c, 4265 0x339f0, 0x33ae4, 4266 0x33af8, 0x33b10, 4267 0x33b28, 0x33b28, 4268 0x33b3c, 0x33b50, 4269 0x33bf0, 0x33c10, 4270 0x33c28, 0x33c28, 4271 0x33c3c, 0x33c50, 4272 0x33cf0, 0x33cfc, 4273 0x34000, 0x34030, 4274 0x34100, 0x34144, 4275 0x34190, 0x341d0, 4276 0x34200, 0x34318, 4277 0x34400, 0x3452c, 4278 0x34540, 0x3461c, 4279 0x34800, 0x34834, 4280 0x348c0, 0x34908, 4281 0x34910, 0x349ac, 4282 0x34a00, 0x34a2c, 4283 0x34a44, 0x34a50, 4284 0x34a74, 0x34c24, 4285 0x34d00, 0x34d00, 4286 0x34d08, 0x34d14, 4287 0x34d1c, 0x34d20, 4288 0x34d3c, 0x34d50, 4289 0x35200, 0x3520c, 4290 0x35220, 0x35220, 4291 0x35240, 0x35240, 4292 0x35600, 0x3560c, 4293 0x35a00, 0x35a1c, 4294 0x35e00, 0x35e20, 4295 0x35e38, 0x35e3c, 4296 0x35e80, 0x35e80, 4297 0x35e88, 0x35ea8, 4298 0x35eb0, 0x35eb4, 4299 0x35ec8, 0x35ed4, 4300 0x35fb8, 0x36004, 4301 0x36200, 0x36200, 4302 0x36208, 0x36240, 4303 0x36248, 0x36280, 4304 0x36288, 0x362c0, 4305 0x362c8, 0x362fc, 4306 0x36600, 0x36630, 4307 0x36a00, 0x36abc, 4308 0x36b00, 0x36b70, 4309 0x37000, 0x37048, 4310 0x37060, 0x3709c, 4311 0x370f0, 0x37148, 4312 0x37160, 0x3719c, 4313 0x371f0, 0x372e4, 4314 0x372f8, 0x373e4, 4315 0x373f8, 0x37448, 4316 0x37460, 0x3749c, 4317 0x374f0, 0x37548, 4318 0x37560, 0x3759c, 4319 0x375f0, 0x376e4, 4320 0x376f8, 0x377e4, 4321 0x377f8, 0x377fc, 4322 0x37814, 0x37814, 4323 0x3782c, 0x3782c, 4324 0x37880, 0x3788c, 4325 0x378e8, 0x378ec, 4326 0x37900, 0x37948, 4327 0x37960, 0x3799c, 4328 0x379f0, 0x37ae4, 4329 0x37af8, 0x37b10, 4330 0x37b28, 0x37b28, 4331 0x37b3c, 0x37b50, 4332 0x37bf0, 0x37c10, 4333 0x37c28, 0x37c28, 4334 0x37c3c, 0x37c50, 4335 0x37cf0, 0x37cfc, 4336 0x38000, 0x38030, 4337 0x38100, 0x38144, 4338 0x38190, 0x381d0, 4339 0x38200, 0x38318, 4340 0x38400, 0x3852c, 4341 0x38540, 0x3861c, 4342 0x38800, 0x38834, 4343 0x388c0, 0x38908, 4344 0x38910, 0x389ac, 4345 0x38a00, 0x38a2c, 4346 0x38a44, 0x38a50, 4347 0x38a74, 0x38c24, 4348 0x38d00, 0x38d00, 4349 0x38d08, 0x38d14, 4350 0x38d1c, 0x38d20, 4351 0x38d3c, 0x38d50, 4352 0x39200, 0x3920c, 4353 0x39220, 0x39220, 4354 0x39240, 0x39240, 4355 0x39600, 0x3960c, 4356 0x39a00, 0x39a1c, 4357 0x39e00, 0x39e20, 4358 0x39e38, 0x39e3c, 4359 0x39e80, 0x39e80, 4360 0x39e88, 0x39ea8, 4361 0x39eb0, 0x39eb4, 4362 0x39ec8, 0x39ed4, 4363 0x39fb8, 0x3a004, 4364 0x3a200, 0x3a200, 4365 0x3a208, 0x3a240, 4366 0x3a248, 0x3a280, 4367 0x3a288, 0x3a2c0, 4368 0x3a2c8, 0x3a2fc, 4369 0x3a600, 0x3a630, 4370 0x3aa00, 0x3aabc, 4371 0x3ab00, 0x3ab70, 4372 0x3b000, 0x3b048, 4373 0x3b060, 0x3b09c, 4374 0x3b0f0, 0x3b148, 4375 0x3b160, 0x3b19c, 4376 0x3b1f0, 0x3b2e4, 4377 0x3b2f8, 0x3b3e4, 4378 0x3b3f8, 0x3b448, 4379 0x3b460, 0x3b49c, 4380 0x3b4f0, 0x3b548, 4381 0x3b560, 0x3b59c, 4382 0x3b5f0, 0x3b6e4, 4383 0x3b6f8, 0x3b7e4, 4384 0x3b7f8, 0x3b7fc, 4385 0x3b814, 0x3b814, 4386 0x3b82c, 0x3b82c, 4387 0x3b880, 0x3b88c, 4388 0x3b8e8, 0x3b8ec, 4389 0x3b900, 0x3b948, 4390 0x3b960, 0x3b99c, 4391 0x3b9f0, 0x3bae4, 4392 0x3baf8, 0x3bb10, 4393 0x3bb28, 0x3bb28, 4394 0x3bb3c, 0x3bb50, 4395 0x3bbf0, 0x3bc10, 4396 0x3bc28, 0x3bc28, 4397 0x3bc3c, 0x3bc50, 4398 0x3bcf0, 0x3bcfc, 4399 0x3c000, 0x3c030, 4400 0x3c100, 0x3c144, 4401 0x3c190, 0x3c1d0, 4402 0x3c200, 0x3c318, 4403 0x3c400, 0x3c52c, 4404 0x3c540, 0x3c61c, 4405 0x3c800, 0x3c834, 4406 0x3c8c0, 0x3c908, 4407 0x3c910, 0x3c9ac, 4408 0x3ca00, 0x3ca2c, 4409 0x3ca44, 0x3ca50, 4410 0x3ca74, 0x3cc24, 4411 0x3cd00, 0x3cd00, 4412 0x3cd08, 0x3cd14, 4413 0x3cd1c, 0x3cd20, 4414 0x3cd3c, 0x3cd50, 4415 0x3d200, 0x3d20c, 4416 0x3d220, 0x3d220, 4417 0x3d240, 0x3d240, 4418 0x3d600, 0x3d60c, 4419 0x3da00, 0x3da1c, 4420 0x3de00, 0x3de20, 4421 0x3de38, 0x3de3c, 4422 0x3de80, 0x3de80, 4423 0x3de88, 0x3dea8, 4424 0x3deb0, 0x3deb4, 4425 0x3dec8, 0x3ded4, 4426 0x3dfb8, 0x3e004, 4427 0x3e200, 0x3e200, 4428 0x3e208, 0x3e240, 4429 0x3e248, 0x3e280, 4430 0x3e288, 0x3e2c0, 4431 0x3e2c8, 0x3e2fc, 4432 0x3e600, 0x3e630, 4433 0x3ea00, 0x3eabc, 4434 0x3eb00, 0x3eb70, 4435 0x3f000, 0x3f048, 4436 0x3f060, 0x3f09c, 4437 0x3f0f0, 0x3f148, 4438 0x3f160, 0x3f19c, 4439 0x3f1f0, 0x3f2e4, 4440 0x3f2f8, 0x3f3e4, 4441 0x3f3f8, 0x3f448, 4442 0x3f460, 0x3f49c, 4443 0x3f4f0, 0x3f548, 4444 0x3f560, 0x3f59c, 4445 0x3f5f0, 0x3f6e4, 4446 0x3f6f8, 0x3f7e4, 4447 0x3f7f8, 0x3f7fc, 4448 0x3f814, 0x3f814, 4449 0x3f82c, 0x3f82c, 4450 0x3f880, 0x3f88c, 4451 0x3f8e8, 0x3f8ec, 4452 0x3f900, 0x3f948, 4453 0x3f960, 0x3f99c, 4454 0x3f9f0, 0x3fae4, 4455 0x3faf8, 0x3fb10, 4456 0x3fb28, 0x3fb28, 4457 0x3fb3c, 0x3fb50, 4458 0x3fbf0, 0x3fc10, 4459 0x3fc28, 0x3fc28, 4460 0x3fc3c, 0x3fc50, 4461 0x3fcf0, 0x3fcfc, 4462 0x40000, 0x4000c, 4463 0x40040, 0x40068, 4464 0x4007c, 0x40144, 4465 0x40180, 0x4018c, 4466 0x40200, 0x40298, 4467 0x402ac, 0x4033c, 4468 0x403f8, 0x403fc, 4469 0x41304, 0x413c4, 4470 0x41400, 0x4141c, 4471 0x41480, 0x414d0, 4472 0x44000, 0x44078, 4473 0x440c0, 0x44278, 4474 0x442c0, 0x44478, 4475 0x444c0, 0x44678, 4476 0x446c0, 0x44878, 4477 0x448c0, 0x449fc, 4478 0x45000, 0x45068, 4479 0x45080, 0x45084, 4480 0x450a0, 0x450b0, 4481 0x45200, 0x45268, 4482 0x45280, 0x45284, 4483 0x452a0, 0x452b0, 4484 0x460c0, 0x460e4, 4485 0x47000, 0x4708c, 4486 0x47200, 0x47250, 4487 0x47400, 0x47420, 4488 0x47600, 0x47618, 4489 0x47800, 0x47814, 4490 0x48000, 0x4800c, 4491 0x48040, 0x48068, 4492 0x4807c, 0x48144, 4493 0x48180, 0x4818c, 4494 0x48200, 0x48298, 4495 0x482ac, 0x4833c, 4496 0x483f8, 0x483fc, 4497 0x49304, 0x493c4, 4498 0x49400, 0x4941c, 4499 0x49480, 0x494d0, 4500 0x4c000, 0x4c078, 4501 0x4c0c0, 0x4c278, 4502 0x4c2c0, 0x4c478, 4503 0x4c4c0, 0x4c678, 4504 0x4c6c0, 0x4c878, 4505 0x4c8c0, 0x4c9fc, 4506 0x4d000, 0x4d068, 4507 0x4d080, 0x4d084, 4508 0x4d0a0, 0x4d0b0, 4509 0x4d200, 0x4d268, 4510 0x4d280, 0x4d284, 4511 0x4d2a0, 0x4d2b0, 4512 0x4e0c0, 0x4e0e4, 4513 0x4f000, 0x4f08c, 4514 0x4f200, 0x4f250, 4515 0x4f400, 0x4f420, 4516 0x4f600, 0x4f618, 4517 0x4f800, 0x4f814, 4518 0x50000, 0x500cc, 4519 0x50400, 0x50400, 4520 0x50800, 0x508cc, 4521 0x50c00, 0x50c00, 4522 0x51000, 0x5101c, 4523 0x51300, 0x51308, 4524 }; 4525 4526 if (is_t4(sc)) { 4527 reg_ranges = &t4_reg_ranges[0]; 4528 n = nitems(t4_reg_ranges); 4529 } else { 4530 reg_ranges = &t5_reg_ranges[0]; 4531 n = nitems(t5_reg_ranges); 4532 } 4533 4534 regs->version = chip_id(sc) | chip_rev(sc) << 10; 4535 for (i = 0; i < n; i += 2) 4536 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 4537 } 4538 4539 static void 4540 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi) 4541 { 4542 int i; 4543 u_int v, tnl_cong_drops; 4544 struct timeval tv; 4545 const struct timeval interval = {0, 250000}; /* 250ms */ 4546 4547 getmicrotime(&tv); 4548 timevalsub(&tv, &interval); 4549 if (timevalcmp(&tv, &pi->last_refreshed, <)) 4550 return; 4551 4552 tnl_cong_drops = 0; 4553 t4_get_port_stats(sc, pi->tx_chan, &pi->stats); 4554 for (i = 0; i < NCHAN; i++) { 4555 if (pi->rx_chan_map & (1 << i)) { 4556 mtx_lock(&sc->regwin_lock); 4557 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 4558 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 4559 mtx_unlock(&sc->regwin_lock); 4560 tnl_cong_drops += v; 4561 } 4562 } 4563 pi->tnl_cong_drops = tnl_cong_drops; 4564 getmicrotime(&pi->last_refreshed); 4565 } 4566 4567 static void 4568 cxgbe_tick(void *arg) 4569 { 4570 struct port_info *pi = arg; 4571 struct adapter *sc = pi->adapter; 4572 struct ifnet *ifp = pi->ifp; 4573 4574 PORT_LOCK(pi); 4575 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 4576 PORT_UNLOCK(pi); 4577 return; /* without scheduling another callout */ 4578 } 4579 4580 cxgbe_refresh_stats(sc, pi); 4581 4582 callout_schedule(&pi->tick, hz); 4583 PORT_UNLOCK(pi); 4584 } 4585 4586 static void 4587 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 4588 { 4589 struct ifnet *vlan; 4590 4591 if (arg != ifp || ifp->if_type != IFT_ETHER) 4592 return; 4593 4594 vlan = VLAN_DEVAT(ifp, vid); 4595 VLAN_SETCOOKIE(vlan, ifp); 4596 } 4597 4598 static int 4599 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 4600 { 4601 4602 #ifdef INVARIANTS 4603 panic("%s: opcode 0x%02x on iq %p with payload %p", 4604 __func__, rss->opcode, iq, m); 4605 #else 4606 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 4607 __func__, rss->opcode, iq, m); 4608 m_freem(m); 4609 #endif 4610 return (EDOOFUS); 4611 } 4612 4613 int 4614 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 4615 { 4616 uintptr_t *loc, new; 4617 4618 if (opcode >= nitems(sc->cpl_handler)) 4619 return (EINVAL); 4620 4621 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 4622 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 4623 atomic_store_rel_ptr(loc, new); 4624 4625 return (0); 4626 } 4627 4628 static int 4629 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 4630 { 4631 4632 #ifdef INVARIANTS 4633 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 4634 #else 4635 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 4636 __func__, iq, ctrl); 4637 #endif 4638 return (EDOOFUS); 4639 } 4640 4641 int 4642 t4_register_an_handler(struct adapter *sc, an_handler_t h) 4643 { 4644 uintptr_t *loc, new; 4645 4646 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 4647 loc = (uintptr_t *) &sc->an_handler; 4648 atomic_store_rel_ptr(loc, new); 4649 4650 return (0); 4651 } 4652 4653 static int 4654 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 4655 { 4656 const struct cpl_fw6_msg *cpl = 4657 __containerof(rpl, struct cpl_fw6_msg, data[0]); 4658 4659 #ifdef INVARIANTS 4660 panic("%s: fw_msg type %d", __func__, cpl->type); 4661 #else 4662 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 4663 #endif 4664 return (EDOOFUS); 4665 } 4666 4667 int 4668 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 4669 { 4670 uintptr_t *loc, new; 4671 4672 if (type >= nitems(sc->fw_msg_handler)) 4673 return (EINVAL); 4674 4675 /* 4676 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 4677 * handler dispatch table. Reject any attempt to install a handler for 4678 * this subtype. 4679 */ 4680 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 4681 return (EINVAL); 4682 4683 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 4684 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 4685 atomic_store_rel_ptr(loc, new); 4686 4687 return (0); 4688 } 4689 4690 static int 4691 t4_sysctls(struct adapter *sc) 4692 { 4693 struct sysctl_ctx_list *ctx; 4694 struct sysctl_oid *oid; 4695 struct sysctl_oid_list *children, *c0; 4696 static char *caps[] = { 4697 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 4698 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL" /* caps[1] niccaps */ 4699 "\6HASHFILTER\7ETHOFLD", 4700 "\20\1TOE", /* caps[2] toecaps */ 4701 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 4702 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 4703 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 4704 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 4705 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 4706 "\4PO_INITIAOR\5PO_TARGET" 4707 }; 4708 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 4709 4710 ctx = device_get_sysctl_ctx(sc->dev); 4711 4712 /* 4713 * dev.t4nex.X. 4714 */ 4715 oid = device_get_sysctl_tree(sc->dev); 4716 c0 = children = SYSCTL_CHILDREN(oid); 4717 4718 sc->sc_do_rxcopy = 1; 4719 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, 4720 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); 4721 4722 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 4723 sc->params.nports, "# of ports"); 4724 4725 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 4726 NULL, chip_rev(sc), "chip hardware revision"); 4727 4728 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 4729 CTLFLAG_RD, sc->fw_version, 0, "firmware version"); 4730 4731 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 4732 CTLFLAG_RD, sc->cfg_file, 0, "configuration file"); 4733 4734 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 4735 sc->cfcsum, "config file checksum"); 4736 4737 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 4738 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 4739 sysctl_bitfield, "A", "available doorbells"); 4740 4741 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 4742 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 4743 sysctl_bitfield, "A", "available link capabilities"); 4744 4745 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 4746 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 4747 sysctl_bitfield, "A", "available NIC capabilities"); 4748 4749 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 4750 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 4751 sysctl_bitfield, "A", "available TCP offload capabilities"); 4752 4753 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 4754 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 4755 sysctl_bitfield, "A", "available RDMA capabilities"); 4756 4757 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 4758 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 4759 sysctl_bitfield, "A", "available iSCSI capabilities"); 4760 4761 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 4762 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 4763 sysctl_bitfield, "A", "available FCoE capabilities"); 4764 4765 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 4766 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 4767 4768 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 4769 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 4770 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 4771 "interrupt holdoff timer values (us)"); 4772 4773 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 4774 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 4775 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 4776 "interrupt holdoff packet counter values"); 4777 4778 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, 4779 NULL, sc->tids.nftids, "number of filters"); 4780 4781 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | 4782 CTLFLAG_RD, sc, 0, sysctl_temperature, "I", 4783 "chip temperature (in Celsius)"); 4784 4785 t4_sge_sysctls(sc, ctx, children); 4786 4787 sc->lro_timeout = 100; 4788 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, 4789 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); 4790 4791 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "debug_flags", CTLFLAG_RW, 4792 &sc->debug_flags, 0, "flags to enable runtime debugging"); 4793 4794 #ifdef SBUF_DRAIN 4795 /* 4796 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 4797 */ 4798 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 4799 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 4800 "logs and miscellaneous information"); 4801 children = SYSCTL_CHILDREN(oid); 4802 4803 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 4804 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4805 sysctl_cctrl, "A", "congestion control"); 4806 4807 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 4808 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4809 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 4810 4811 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 4812 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 4813 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 4814 4815 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 4816 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 4817 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 4818 4819 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 4820 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 4821 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 4822 4823 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 4824 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 4825 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 4826 4827 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 4828 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 4829 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 4830 4831 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 4832 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4833 sysctl_cim_la, "A", "CIM logic analyzer"); 4834 4835 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", 4836 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4837 sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); 4838 4839 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 4840 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 4841 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 4842 4843 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 4844 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 4845 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 4846 4847 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 4848 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 4849 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 4850 4851 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 4852 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 4853 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 4854 4855 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 4856 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 4857 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 4858 4859 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 4860 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 4861 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 4862 4863 if (is_t5(sc)) { 4864 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 4865 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 4866 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 4867 4868 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 4869 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 4870 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 4871 } 4872 4873 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", 4874 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4875 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); 4876 4877 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 4878 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4879 sysctl_cim_qcfg, "A", "CIM queue configuration"); 4880 4881 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 4882 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4883 sysctl_cpl_stats, "A", "CPL statistics"); 4884 4885 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 4886 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4887 sysctl_ddp_stats, "A", "non-TCP DDP statistics"); 4888 4889 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 4890 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4891 sysctl_devlog, "A", "firmware's device log"); 4892 4893 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 4894 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4895 sysctl_fcoe_stats, "A", "FCoE statistics"); 4896 4897 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 4898 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4899 sysctl_hw_sched, "A", "hardware scheduler "); 4900 4901 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 4902 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4903 sysctl_l2t, "A", "hardware L2 table"); 4904 4905 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 4906 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4907 sysctl_lb_stats, "A", "loopback statistics"); 4908 4909 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 4910 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4911 sysctl_meminfo, "A", "memory regions"); 4912 4913 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", 4914 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4915 sysctl_mps_tcam, "A", "MPS TCAM entries"); 4916 4917 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 4918 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4919 sysctl_path_mtus, "A", "path MTUs"); 4920 4921 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 4922 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4923 sysctl_pm_stats, "A", "PM statistics"); 4924 4925 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 4926 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4927 sysctl_rdma_stats, "A", "RDMA statistics"); 4928 4929 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 4930 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4931 sysctl_tcp_stats, "A", "TCP statistics"); 4932 4933 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 4934 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4935 sysctl_tids, "A", "TID information"); 4936 4937 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 4938 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4939 sysctl_tp_err_stats, "A", "TP error statistics"); 4940 4941 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", 4942 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4943 sysctl_tp_la, "A", "TP logic analyzer"); 4944 4945 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 4946 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4947 sysctl_tx_rate, "A", "Tx rate"); 4948 4949 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", 4950 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4951 sysctl_ulprx_la, "A", "ULPRX logic analyzer"); 4952 4953 if (is_t5(sc)) { 4954 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 4955 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4956 sysctl_wcwr_stats, "A", "write combined work requests"); 4957 } 4958 #endif 4959 4960 #ifdef TCP_OFFLOAD 4961 if (is_offload(sc)) { 4962 /* 4963 * dev.t4nex.X.toe. 4964 */ 4965 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 4966 NULL, "TOE parameters"); 4967 children = SYSCTL_CHILDREN(oid); 4968 4969 sc->tt.sndbuf = 256 * 1024; 4970 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 4971 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 4972 4973 sc->tt.ddp = 0; 4974 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 4975 &sc->tt.ddp, 0, "DDP allowed"); 4976 4977 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 4978 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 4979 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 4980 4981 sc->tt.ddp_thres = 4982 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 4983 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 4984 &sc->tt.ddp_thres, 0, "DDP threshold"); 4985 4986 sc->tt.rx_coalesce = 1; 4987 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", 4988 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); 4989 4990 sc->tt.tx_align = 1; 4991 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align", 4992 CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload"); 4993 } 4994 #endif 4995 4996 4997 return (0); 4998 } 4999 5000 static int 5001 cxgbe_sysctls(struct port_info *pi) 5002 { 5003 struct sysctl_ctx_list *ctx; 5004 struct sysctl_oid *oid; 5005 struct sysctl_oid_list *children; 5006 struct adapter *sc = pi->adapter; 5007 5008 ctx = device_get_sysctl_ctx(pi->dev); 5009 5010 /* 5011 * dev.cxgbe.X. 5012 */ 5013 oid = device_get_sysctl_tree(pi->dev); 5014 children = SYSCTL_CHILDREN(oid); 5015 5016 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | 5017 CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); 5018 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { 5019 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 5020 CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I", 5021 "PHY temperature (in Celsius)"); 5022 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", 5023 CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I", 5024 "PHY firmware version"); 5025 } 5026 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 5027 &pi->nrxq, 0, "# of rx queues"); 5028 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 5029 &pi->ntxq, 0, "# of tx queues"); 5030 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 5031 &pi->first_rxq, 0, "index of first rx queue"); 5032 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 5033 &pi->first_txq, 0, "index of first tx queue"); 5034 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT | 5035 CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU", 5036 "Reserve queue 0 for non-flowid packets"); 5037 5038 #ifdef TCP_OFFLOAD 5039 if (is_offload(sc)) { 5040 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 5041 &pi->nofldrxq, 0, 5042 "# of rx queues for offloaded TCP connections"); 5043 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 5044 &pi->nofldtxq, 0, 5045 "# of tx queues for offloaded TCP connections"); 5046 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 5047 CTLFLAG_RD, &pi->first_ofld_rxq, 0, 5048 "index of first TOE rx queue"); 5049 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 5050 CTLFLAG_RD, &pi->first_ofld_txq, 0, 5051 "index of first TOE tx queue"); 5052 } 5053 #endif 5054 #ifdef DEV_NETMAP 5055 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD, 5056 &pi->nnmrxq, 0, "# of rx queues for netmap"); 5057 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD, 5058 &pi->nnmtxq, 0, "# of tx queues for netmap"); 5059 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq", 5060 CTLFLAG_RD, &pi->first_nm_rxq, 0, 5061 "index of first netmap rx queue"); 5062 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq", 5063 CTLFLAG_RD, &pi->first_nm_txq, 0, 5064 "index of first netmap tx queue"); 5065 #endif 5066 5067 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 5068 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 5069 "holdoff timer index"); 5070 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 5071 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 5072 "holdoff packet counter index"); 5073 5074 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 5075 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 5076 "rx queue size"); 5077 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 5078 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 5079 "tx queue size"); 5080 5081 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings", 5082 CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings, 5083 "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)"); 5084 5085 /* 5086 * dev.cxgbe.X.stats. 5087 */ 5088 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 5089 NULL, "port statistics"); 5090 children = SYSCTL_CHILDREN(oid); 5091 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD, 5092 &pi->tx_parse_error, 0, 5093 "# of tx packets with invalid length or # of segments"); 5094 5095 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 5096 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 5097 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ 5098 sysctl_handle_t4_reg64, "QU", desc) 5099 5100 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 5101 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 5102 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 5103 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 5104 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 5105 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 5106 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 5107 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 5108 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 5109 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 5110 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 5111 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 5112 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 5113 "# of tx frames in this range", 5114 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 5115 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 5116 "# of tx frames in this range", 5117 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 5118 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 5119 "# of tx frames in this range", 5120 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 5121 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 5122 "# of tx frames in this range", 5123 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 5124 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 5125 "# of tx frames in this range", 5126 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 5127 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 5128 "# of tx frames in this range", 5129 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 5130 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 5131 "# of tx frames in this range", 5132 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 5133 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 5134 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 5135 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 5136 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 5137 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 5138 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 5139 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 5140 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 5141 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 5142 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 5143 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 5144 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 5145 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 5146 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 5147 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 5148 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 5149 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 5150 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 5151 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 5152 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 5153 5154 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 5155 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 5156 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 5157 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 5158 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 5159 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 5160 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 5161 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 5162 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 5163 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 5164 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 5165 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 5166 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 5167 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 5168 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 5169 "# of frames received with bad FCS", 5170 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 5171 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 5172 "# of frames received with length error", 5173 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 5174 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 5175 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 5176 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 5177 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 5178 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 5179 "# of rx frames in this range", 5180 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 5181 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 5182 "# of rx frames in this range", 5183 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 5184 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 5185 "# of rx frames in this range", 5186 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 5187 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 5188 "# of rx frames in this range", 5189 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 5190 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 5191 "# of rx frames in this range", 5192 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 5193 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 5194 "# of rx frames in this range", 5195 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 5196 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 5197 "# of rx frames in this range", 5198 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 5199 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 5200 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 5201 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 5202 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 5203 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 5204 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 5205 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 5206 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 5207 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 5208 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 5209 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 5210 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 5211 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 5212 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 5213 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 5214 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 5215 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 5216 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 5217 5218 #undef SYSCTL_ADD_T4_REG64 5219 5220 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 5221 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 5222 &pi->stats.name, desc) 5223 5224 /* We get these from port_stats and they may be stale by upto 1s */ 5225 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 5226 "# drops due to buffer-group 0 overflows"); 5227 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 5228 "# drops due to buffer-group 1 overflows"); 5229 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 5230 "# drops due to buffer-group 2 overflows"); 5231 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 5232 "# drops due to buffer-group 3 overflows"); 5233 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 5234 "# of buffer-group 0 truncated packets"); 5235 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 5236 "# of buffer-group 1 truncated packets"); 5237 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 5238 "# of buffer-group 2 truncated packets"); 5239 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 5240 "# of buffer-group 3 truncated packets"); 5241 5242 #undef SYSCTL_ADD_T4_PORTSTAT 5243 5244 return (0); 5245 } 5246 5247 static int 5248 sysctl_int_array(SYSCTL_HANDLER_ARGS) 5249 { 5250 int rc, *i, space = 0; 5251 struct sbuf sb; 5252 5253 sbuf_new_for_sysctl(&sb, NULL, 64, req); 5254 for (i = arg1; arg2; arg2 -= sizeof(int), i++) { 5255 if (space) 5256 sbuf_printf(&sb, " "); 5257 sbuf_printf(&sb, "%d", *i); 5258 space = 1; 5259 } 5260 rc = sbuf_finish(&sb); 5261 sbuf_delete(&sb); 5262 return (rc); 5263 } 5264 5265 static int 5266 sysctl_bitfield(SYSCTL_HANDLER_ARGS) 5267 { 5268 int rc; 5269 struct sbuf *sb; 5270 5271 rc = sysctl_wire_old_buffer(req, 0); 5272 if (rc != 0) 5273 return(rc); 5274 5275 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5276 if (sb == NULL) 5277 return (ENOMEM); 5278 5279 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 5280 rc = sbuf_finish(sb); 5281 sbuf_delete(sb); 5282 5283 return (rc); 5284 } 5285 5286 static int 5287 sysctl_btphy(SYSCTL_HANDLER_ARGS) 5288 { 5289 struct port_info *pi = arg1; 5290 int op = arg2; 5291 struct adapter *sc = pi->adapter; 5292 u_int v; 5293 int rc; 5294 5295 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt"); 5296 if (rc) 5297 return (rc); 5298 /* XXX: magic numbers */ 5299 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, 5300 &v); 5301 end_synchronized_op(sc, 0); 5302 if (rc) 5303 return (rc); 5304 if (op == 0) 5305 v /= 256; 5306 5307 rc = sysctl_handle_int(oidp, &v, 0, req); 5308 return (rc); 5309 } 5310 5311 static int 5312 sysctl_noflowq(SYSCTL_HANDLER_ARGS) 5313 { 5314 struct port_info *pi = arg1; 5315 int rc, val; 5316 5317 val = pi->rsrv_noflowq; 5318 rc = sysctl_handle_int(oidp, &val, 0, req); 5319 if (rc != 0 || req->newptr == NULL) 5320 return (rc); 5321 5322 if ((val >= 1) && (pi->ntxq > 1)) 5323 pi->rsrv_noflowq = 1; 5324 else 5325 pi->rsrv_noflowq = 0; 5326 5327 return (rc); 5328 } 5329 5330 static int 5331 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 5332 { 5333 struct port_info *pi = arg1; 5334 struct adapter *sc = pi->adapter; 5335 int idx, rc, i; 5336 struct sge_rxq *rxq; 5337 #ifdef TCP_OFFLOAD 5338 struct sge_ofld_rxq *ofld_rxq; 5339 #endif 5340 uint8_t v; 5341 5342 idx = pi->tmr_idx; 5343 5344 rc = sysctl_handle_int(oidp, &idx, 0, req); 5345 if (rc != 0 || req->newptr == NULL) 5346 return (rc); 5347 5348 if (idx < 0 || idx >= SGE_NTIMERS) 5349 return (EINVAL); 5350 5351 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5352 "t4tmr"); 5353 if (rc) 5354 return (rc); 5355 5356 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 5357 for_each_rxq(pi, i, rxq) { 5358 #ifdef atomic_store_rel_8 5359 atomic_store_rel_8(&rxq->iq.intr_params, v); 5360 #else 5361 rxq->iq.intr_params = v; 5362 #endif 5363 } 5364 #ifdef TCP_OFFLOAD 5365 for_each_ofld_rxq(pi, i, ofld_rxq) { 5366 #ifdef atomic_store_rel_8 5367 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); 5368 #else 5369 ofld_rxq->iq.intr_params = v; 5370 #endif 5371 } 5372 #endif 5373 pi->tmr_idx = idx; 5374 5375 end_synchronized_op(sc, LOCK_HELD); 5376 return (0); 5377 } 5378 5379 static int 5380 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 5381 { 5382 struct port_info *pi = arg1; 5383 struct adapter *sc = pi->adapter; 5384 int idx, rc; 5385 5386 idx = pi->pktc_idx; 5387 5388 rc = sysctl_handle_int(oidp, &idx, 0, req); 5389 if (rc != 0 || req->newptr == NULL) 5390 return (rc); 5391 5392 if (idx < -1 || idx >= SGE_NCOUNTERS) 5393 return (EINVAL); 5394 5395 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5396 "t4pktc"); 5397 if (rc) 5398 return (rc); 5399 5400 if (pi->flags & PORT_INIT_DONE) 5401 rc = EBUSY; /* cannot be changed once the queues are created */ 5402 else 5403 pi->pktc_idx = idx; 5404 5405 end_synchronized_op(sc, LOCK_HELD); 5406 return (rc); 5407 } 5408 5409 static int 5410 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 5411 { 5412 struct port_info *pi = arg1; 5413 struct adapter *sc = pi->adapter; 5414 int qsize, rc; 5415 5416 qsize = pi->qsize_rxq; 5417 5418 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5419 if (rc != 0 || req->newptr == NULL) 5420 return (rc); 5421 5422 if (qsize < 128 || (qsize & 7)) 5423 return (EINVAL); 5424 5425 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5426 "t4rxqs"); 5427 if (rc) 5428 return (rc); 5429 5430 if (pi->flags & PORT_INIT_DONE) 5431 rc = EBUSY; /* cannot be changed once the queues are created */ 5432 else 5433 pi->qsize_rxq = qsize; 5434 5435 end_synchronized_op(sc, LOCK_HELD); 5436 return (rc); 5437 } 5438 5439 static int 5440 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 5441 { 5442 struct port_info *pi = arg1; 5443 struct adapter *sc = pi->adapter; 5444 int qsize, rc; 5445 5446 qsize = pi->qsize_txq; 5447 5448 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5449 if (rc != 0 || req->newptr == NULL) 5450 return (rc); 5451 5452 if (qsize < 128 || qsize > 65536) 5453 return (EINVAL); 5454 5455 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5456 "t4txqs"); 5457 if (rc) 5458 return (rc); 5459 5460 if (pi->flags & PORT_INIT_DONE) 5461 rc = EBUSY; /* cannot be changed once the queues are created */ 5462 else 5463 pi->qsize_txq = qsize; 5464 5465 end_synchronized_op(sc, LOCK_HELD); 5466 return (rc); 5467 } 5468 5469 static int 5470 sysctl_pause_settings(SYSCTL_HANDLER_ARGS) 5471 { 5472 struct port_info *pi = arg1; 5473 struct adapter *sc = pi->adapter; 5474 struct link_config *lc = &pi->link_cfg; 5475 int rc; 5476 5477 if (req->newptr == NULL) { 5478 struct sbuf *sb; 5479 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX"; 5480 5481 rc = sysctl_wire_old_buffer(req, 0); 5482 if (rc != 0) 5483 return(rc); 5484 5485 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5486 if (sb == NULL) 5487 return (ENOMEM); 5488 5489 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits); 5490 rc = sbuf_finish(sb); 5491 sbuf_delete(sb); 5492 } else { 5493 char s[2]; 5494 int n; 5495 5496 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX)); 5497 s[1] = 0; 5498 5499 rc = sysctl_handle_string(oidp, s, sizeof(s), req); 5500 if (rc != 0) 5501 return(rc); 5502 5503 if (s[1] != 0) 5504 return (EINVAL); 5505 if (s[0] < '0' || s[0] > '9') 5506 return (EINVAL); /* not a number */ 5507 n = s[0] - '0'; 5508 if (n & ~(PAUSE_TX | PAUSE_RX)) 5509 return (EINVAL); /* some other bit is set too */ 5510 5511 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE"); 5512 if (rc) 5513 return (rc); 5514 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) { 5515 int link_ok = lc->link_ok; 5516 5517 lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX); 5518 lc->requested_fc |= n; 5519 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc); 5520 lc->link_ok = link_ok; /* restore */ 5521 } 5522 end_synchronized_op(sc, 0); 5523 } 5524 5525 return (rc); 5526 } 5527 5528 static int 5529 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 5530 { 5531 struct adapter *sc = arg1; 5532 int reg = arg2; 5533 uint64_t val; 5534 5535 val = t4_read_reg64(sc, reg); 5536 5537 return (sysctl_handle_64(oidp, &val, 0, req)); 5538 } 5539 5540 static int 5541 sysctl_temperature(SYSCTL_HANDLER_ARGS) 5542 { 5543 struct adapter *sc = arg1; 5544 int rc, t; 5545 uint32_t param, val; 5546 5547 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); 5548 if (rc) 5549 return (rc); 5550 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 5551 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 5552 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); 5553 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 5554 end_synchronized_op(sc, 0); 5555 if (rc) 5556 return (rc); 5557 5558 /* unknown is returned as 0 but we display -1 in that case */ 5559 t = val == 0 ? -1 : val; 5560 5561 rc = sysctl_handle_int(oidp, &t, 0, req); 5562 return (rc); 5563 } 5564 5565 #ifdef SBUF_DRAIN 5566 static int 5567 sysctl_cctrl(SYSCTL_HANDLER_ARGS) 5568 { 5569 struct adapter *sc = arg1; 5570 struct sbuf *sb; 5571 int rc, i; 5572 uint16_t incr[NMTUS][NCCTRL_WIN]; 5573 static const char *dec_fac[] = { 5574 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 5575 "0.9375" 5576 }; 5577 5578 rc = sysctl_wire_old_buffer(req, 0); 5579 if (rc != 0) 5580 return (rc); 5581 5582 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5583 if (sb == NULL) 5584 return (ENOMEM); 5585 5586 t4_read_cong_tbl(sc, incr); 5587 5588 for (i = 0; i < NCCTRL_WIN; ++i) { 5589 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 5590 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 5591 incr[5][i], incr[6][i], incr[7][i]); 5592 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 5593 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 5594 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 5595 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 5596 } 5597 5598 rc = sbuf_finish(sb); 5599 sbuf_delete(sb); 5600 5601 return (rc); 5602 } 5603 5604 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 5605 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 5606 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 5607 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 5608 }; 5609 5610 static int 5611 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 5612 { 5613 struct adapter *sc = arg1; 5614 struct sbuf *sb; 5615 int rc, i, n, qid = arg2; 5616 uint32_t *buf, *p; 5617 char *qtype; 5618 u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5; 5619 5620 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 5621 ("%s: bad qid %d\n", __func__, qid)); 5622 5623 if (qid < CIM_NUM_IBQ) { 5624 /* inbound queue */ 5625 qtype = "IBQ"; 5626 n = 4 * CIM_IBQ_SIZE; 5627 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5628 rc = t4_read_cim_ibq(sc, qid, buf, n); 5629 } else { 5630 /* outbound queue */ 5631 qtype = "OBQ"; 5632 qid -= CIM_NUM_IBQ; 5633 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 5634 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5635 rc = t4_read_cim_obq(sc, qid, buf, n); 5636 } 5637 5638 if (rc < 0) { 5639 rc = -rc; 5640 goto done; 5641 } 5642 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 5643 5644 rc = sysctl_wire_old_buffer(req, 0); 5645 if (rc != 0) 5646 goto done; 5647 5648 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5649 if (sb == NULL) { 5650 rc = ENOMEM; 5651 goto done; 5652 } 5653 5654 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 5655 for (i = 0, p = buf; i < n; i += 16, p += 4) 5656 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 5657 p[2], p[3]); 5658 5659 rc = sbuf_finish(sb); 5660 sbuf_delete(sb); 5661 done: 5662 free(buf, M_CXGBE); 5663 return (rc); 5664 } 5665 5666 static int 5667 sysctl_cim_la(SYSCTL_HANDLER_ARGS) 5668 { 5669 struct adapter *sc = arg1; 5670 u_int cfg; 5671 struct sbuf *sb; 5672 uint32_t *buf, *p; 5673 int rc; 5674 5675 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 5676 if (rc != 0) 5677 return (rc); 5678 5679 rc = sysctl_wire_old_buffer(req, 0); 5680 if (rc != 0) 5681 return (rc); 5682 5683 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5684 if (sb == NULL) 5685 return (ENOMEM); 5686 5687 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 5688 M_ZERO | M_WAITOK); 5689 5690 rc = -t4_cim_read_la(sc, buf, NULL); 5691 if (rc != 0) 5692 goto done; 5693 5694 sbuf_printf(sb, "Status Data PC%s", 5695 cfg & F_UPDBGLACAPTPCONLY ? "" : 5696 " LS0Stat LS0Addr LS0Data"); 5697 5698 KASSERT((sc->params.cim_la_size & 7) == 0, 5699 ("%s: p will walk off the end of buf", __func__)); 5700 5701 for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) { 5702 if (cfg & F_UPDBGLACAPTPCONLY) { 5703 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 5704 p[6], p[7]); 5705 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 5706 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 5707 p[4] & 0xff, p[5] >> 8); 5708 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 5709 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5710 p[1] & 0xf, p[2] >> 4); 5711 } else { 5712 sbuf_printf(sb, 5713 "\n %02x %x%07x %x%07x %08x %08x " 5714 "%08x%08x%08x%08x", 5715 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5716 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 5717 p[6], p[7]); 5718 } 5719 } 5720 5721 rc = sbuf_finish(sb); 5722 sbuf_delete(sb); 5723 done: 5724 free(buf, M_CXGBE); 5725 return (rc); 5726 } 5727 5728 static int 5729 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) 5730 { 5731 struct adapter *sc = arg1; 5732 u_int i; 5733 struct sbuf *sb; 5734 uint32_t *buf, *p; 5735 int rc; 5736 5737 rc = sysctl_wire_old_buffer(req, 0); 5738 if (rc != 0) 5739 return (rc); 5740 5741 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5742 if (sb == NULL) 5743 return (ENOMEM); 5744 5745 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, 5746 M_ZERO | M_WAITOK); 5747 5748 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); 5749 p = buf; 5750 5751 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5752 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], 5753 p[1], p[0]); 5754 } 5755 5756 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); 5757 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5758 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", 5759 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, 5760 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, 5761 (p[1] >> 2) | ((p[2] & 3) << 30), 5762 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, 5763 p[0] & 1); 5764 } 5765 5766 rc = sbuf_finish(sb); 5767 sbuf_delete(sb); 5768 free(buf, M_CXGBE); 5769 return (rc); 5770 } 5771 5772 static int 5773 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) 5774 { 5775 struct adapter *sc = arg1; 5776 u_int i; 5777 struct sbuf *sb; 5778 uint32_t *buf, *p; 5779 int rc; 5780 5781 rc = sysctl_wire_old_buffer(req, 0); 5782 if (rc != 0) 5783 return (rc); 5784 5785 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5786 if (sb == NULL) 5787 return (ENOMEM); 5788 5789 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, 5790 M_ZERO | M_WAITOK); 5791 5792 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); 5793 p = buf; 5794 5795 sbuf_printf(sb, "Cntl ID DataBE Addr Data"); 5796 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5797 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", 5798 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, 5799 p[4], p[3], p[2], p[1], p[0]); 5800 } 5801 5802 sbuf_printf(sb, "\n\nCntl ID Data"); 5803 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5804 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", 5805 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); 5806 } 5807 5808 rc = sbuf_finish(sb); 5809 sbuf_delete(sb); 5810 free(buf, M_CXGBE); 5811 return (rc); 5812 } 5813 5814 static int 5815 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 5816 { 5817 struct adapter *sc = arg1; 5818 struct sbuf *sb; 5819 int rc, i; 5820 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5821 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5822 uint16_t thres[CIM_NUM_IBQ]; 5823 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 5824 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 5825 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 5826 5827 if (is_t4(sc)) { 5828 cim_num_obq = CIM_NUM_OBQ; 5829 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 5830 obq_rdaddr = A_UP_OBQ_0_REALADDR; 5831 } else { 5832 cim_num_obq = CIM_NUM_OBQ_T5; 5833 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 5834 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 5835 } 5836 nq = CIM_NUM_IBQ + cim_num_obq; 5837 5838 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 5839 if (rc == 0) 5840 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 5841 if (rc != 0) 5842 return (rc); 5843 5844 t4_read_cimq_cfg(sc, base, size, thres); 5845 5846 rc = sysctl_wire_old_buffer(req, 0); 5847 if (rc != 0) 5848 return (rc); 5849 5850 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5851 if (sb == NULL) 5852 return (ENOMEM); 5853 5854 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 5855 5856 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 5857 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 5858 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 5859 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5860 G_QUEREMFLITS(p[2]) * 16); 5861 for ( ; i < nq; i++, p += 4, wr += 2) 5862 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 5863 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 5864 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5865 G_QUEREMFLITS(p[2]) * 16); 5866 5867 rc = sbuf_finish(sb); 5868 sbuf_delete(sb); 5869 5870 return (rc); 5871 } 5872 5873 static int 5874 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 5875 { 5876 struct adapter *sc = arg1; 5877 struct sbuf *sb; 5878 int rc; 5879 struct tp_cpl_stats stats; 5880 5881 rc = sysctl_wire_old_buffer(req, 0); 5882 if (rc != 0) 5883 return (rc); 5884 5885 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5886 if (sb == NULL) 5887 return (ENOMEM); 5888 5889 t4_tp_get_cpl_stats(sc, &stats); 5890 5891 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 5892 "channel 3\n"); 5893 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 5894 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 5895 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 5896 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 5897 5898 rc = sbuf_finish(sb); 5899 sbuf_delete(sb); 5900 5901 return (rc); 5902 } 5903 5904 static int 5905 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 5906 { 5907 struct adapter *sc = arg1; 5908 struct sbuf *sb; 5909 int rc; 5910 struct tp_usm_stats stats; 5911 5912 rc = sysctl_wire_old_buffer(req, 0); 5913 if (rc != 0) 5914 return(rc); 5915 5916 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5917 if (sb == NULL) 5918 return (ENOMEM); 5919 5920 t4_get_usm_stats(sc, &stats); 5921 5922 sbuf_printf(sb, "Frames: %u\n", stats.frames); 5923 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 5924 sbuf_printf(sb, "Drops: %u", stats.drops); 5925 5926 rc = sbuf_finish(sb); 5927 sbuf_delete(sb); 5928 5929 return (rc); 5930 } 5931 5932 const char *devlog_level_strings[] = { 5933 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 5934 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 5935 [FW_DEVLOG_LEVEL_ERR] = "ERR", 5936 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 5937 [FW_DEVLOG_LEVEL_INFO] = "INFO", 5938 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 5939 }; 5940 5941 const char *devlog_facility_strings[] = { 5942 [FW_DEVLOG_FACILITY_CORE] = "CORE", 5943 [FW_DEVLOG_FACILITY_CF] = "CF", 5944 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 5945 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 5946 [FW_DEVLOG_FACILITY_RES] = "RES", 5947 [FW_DEVLOG_FACILITY_HW] = "HW", 5948 [FW_DEVLOG_FACILITY_FLR] = "FLR", 5949 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 5950 [FW_DEVLOG_FACILITY_PHY] = "PHY", 5951 [FW_DEVLOG_FACILITY_MAC] = "MAC", 5952 [FW_DEVLOG_FACILITY_PORT] = "PORT", 5953 [FW_DEVLOG_FACILITY_VI] = "VI", 5954 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 5955 [FW_DEVLOG_FACILITY_ACL] = "ACL", 5956 [FW_DEVLOG_FACILITY_TM] = "TM", 5957 [FW_DEVLOG_FACILITY_QFC] = "QFC", 5958 [FW_DEVLOG_FACILITY_DCB] = "DCB", 5959 [FW_DEVLOG_FACILITY_ETH] = "ETH", 5960 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 5961 [FW_DEVLOG_FACILITY_RI] = "RI", 5962 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 5963 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 5964 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 5965 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 5966 }; 5967 5968 static int 5969 sysctl_devlog(SYSCTL_HANDLER_ARGS) 5970 { 5971 struct adapter *sc = arg1; 5972 struct devlog_params *dparams = &sc->params.devlog; 5973 struct fw_devlog_e *buf, *e; 5974 int i, j, rc, nentries, first = 0, m; 5975 struct sbuf *sb; 5976 uint64_t ftstamp = UINT64_MAX; 5977 5978 if (dparams->start == 0) { 5979 dparams->memtype = FW_MEMTYPE_EDC0; 5980 dparams->start = 0x84000; 5981 dparams->size = 32768; 5982 } 5983 5984 nentries = dparams->size / sizeof(struct fw_devlog_e); 5985 5986 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 5987 if (buf == NULL) 5988 return (ENOMEM); 5989 5990 m = fwmtype_to_hwmtype(dparams->memtype); 5991 rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf); 5992 if (rc != 0) 5993 goto done; 5994 5995 for (i = 0; i < nentries; i++) { 5996 e = &buf[i]; 5997 5998 if (e->timestamp == 0) 5999 break; /* end */ 6000 6001 e->timestamp = be64toh(e->timestamp); 6002 e->seqno = be32toh(e->seqno); 6003 for (j = 0; j < 8; j++) 6004 e->params[j] = be32toh(e->params[j]); 6005 6006 if (e->timestamp < ftstamp) { 6007 ftstamp = e->timestamp; 6008 first = i; 6009 } 6010 } 6011 6012 if (buf[first].timestamp == 0) 6013 goto done; /* nothing in the log */ 6014 6015 rc = sysctl_wire_old_buffer(req, 0); 6016 if (rc != 0) 6017 goto done; 6018 6019 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6020 if (sb == NULL) { 6021 rc = ENOMEM; 6022 goto done; 6023 } 6024 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 6025 "Seq#", "Tstamp", "Level", "Facility", "Message"); 6026 6027 i = first; 6028 do { 6029 e = &buf[i]; 6030 if (e->timestamp == 0) 6031 break; /* end */ 6032 6033 sbuf_printf(sb, "%10d %15ju %8s %8s ", 6034 e->seqno, e->timestamp, 6035 (e->level < nitems(devlog_level_strings) ? 6036 devlog_level_strings[e->level] : "UNKNOWN"), 6037 (e->facility < nitems(devlog_facility_strings) ? 6038 devlog_facility_strings[e->facility] : "UNKNOWN")); 6039 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 6040 e->params[2], e->params[3], e->params[4], 6041 e->params[5], e->params[6], e->params[7]); 6042 6043 if (++i == nentries) 6044 i = 0; 6045 } while (i != first); 6046 6047 rc = sbuf_finish(sb); 6048 sbuf_delete(sb); 6049 done: 6050 free(buf, M_CXGBE); 6051 return (rc); 6052 } 6053 6054 static int 6055 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 6056 { 6057 struct adapter *sc = arg1; 6058 struct sbuf *sb; 6059 int rc; 6060 struct tp_fcoe_stats stats[4]; 6061 6062 rc = sysctl_wire_old_buffer(req, 0); 6063 if (rc != 0) 6064 return (rc); 6065 6066 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6067 if (sb == NULL) 6068 return (ENOMEM); 6069 6070 t4_get_fcoe_stats(sc, 0, &stats[0]); 6071 t4_get_fcoe_stats(sc, 1, &stats[1]); 6072 t4_get_fcoe_stats(sc, 2, &stats[2]); 6073 t4_get_fcoe_stats(sc, 3, &stats[3]); 6074 6075 sbuf_printf(sb, " channel 0 channel 1 " 6076 "channel 2 channel 3\n"); 6077 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 6078 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 6079 stats[3].octetsDDP); 6080 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 6081 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 6082 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 6083 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 6084 stats[3].framesDrop); 6085 6086 rc = sbuf_finish(sb); 6087 sbuf_delete(sb); 6088 6089 return (rc); 6090 } 6091 6092 static int 6093 sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 6094 { 6095 struct adapter *sc = arg1; 6096 struct sbuf *sb; 6097 int rc, i; 6098 unsigned int map, kbps, ipg, mode; 6099 unsigned int pace_tab[NTX_SCHED]; 6100 6101 rc = sysctl_wire_old_buffer(req, 0); 6102 if (rc != 0) 6103 return (rc); 6104 6105 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6106 if (sb == NULL) 6107 return (ENOMEM); 6108 6109 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 6110 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 6111 t4_read_pace_tbl(sc, pace_tab); 6112 6113 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 6114 "Class IPG (0.1 ns) Flow IPG (us)"); 6115 6116 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 6117 t4_get_tx_sched(sc, i, &kbps, &ipg); 6118 sbuf_printf(sb, "\n %u %-5s %u ", i, 6119 (mode & (1 << i)) ? "flow" : "class", map & 3); 6120 if (kbps) 6121 sbuf_printf(sb, "%9u ", kbps); 6122 else 6123 sbuf_printf(sb, " disabled "); 6124 6125 if (ipg) 6126 sbuf_printf(sb, "%13u ", ipg); 6127 else 6128 sbuf_printf(sb, " disabled "); 6129 6130 if (pace_tab[i]) 6131 sbuf_printf(sb, "%10u", pace_tab[i]); 6132 else 6133 sbuf_printf(sb, " disabled"); 6134 } 6135 6136 rc = sbuf_finish(sb); 6137 sbuf_delete(sb); 6138 6139 return (rc); 6140 } 6141 6142 static int 6143 sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 6144 { 6145 struct adapter *sc = arg1; 6146 struct sbuf *sb; 6147 int rc, i, j; 6148 uint64_t *p0, *p1; 6149 struct lb_port_stats s[2]; 6150 static const char *stat_name[] = { 6151 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 6152 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 6153 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 6154 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 6155 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 6156 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 6157 "BG2FramesTrunc:", "BG3FramesTrunc:" 6158 }; 6159 6160 rc = sysctl_wire_old_buffer(req, 0); 6161 if (rc != 0) 6162 return (rc); 6163 6164 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6165 if (sb == NULL) 6166 return (ENOMEM); 6167 6168 memset(s, 0, sizeof(s)); 6169 6170 for (i = 0; i < 4; i += 2) { 6171 t4_get_lb_stats(sc, i, &s[0]); 6172 t4_get_lb_stats(sc, i + 1, &s[1]); 6173 6174 p0 = &s[0].octets; 6175 p1 = &s[1].octets; 6176 sbuf_printf(sb, "%s Loopback %u" 6177 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 6178 6179 for (j = 0; j < nitems(stat_name); j++) 6180 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 6181 *p0++, *p1++); 6182 } 6183 6184 rc = sbuf_finish(sb); 6185 sbuf_delete(sb); 6186 6187 return (rc); 6188 } 6189 6190 static int 6191 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) 6192 { 6193 int rc = 0; 6194 struct port_info *pi = arg1; 6195 struct sbuf *sb; 6196 static const char *linkdnreasons[] = { 6197 "non-specific", "remote fault", "autoneg failed", "reserved3", 6198 "PHY overheated", "unknown", "rx los", "reserved7" 6199 }; 6200 6201 rc = sysctl_wire_old_buffer(req, 0); 6202 if (rc != 0) 6203 return(rc); 6204 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); 6205 if (sb == NULL) 6206 return (ENOMEM); 6207 6208 if (pi->linkdnrc < 0) 6209 sbuf_printf(sb, "n/a"); 6210 else if (pi->linkdnrc < nitems(linkdnreasons)) 6211 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]); 6212 else 6213 sbuf_printf(sb, "%d", pi->linkdnrc); 6214 6215 rc = sbuf_finish(sb); 6216 sbuf_delete(sb); 6217 6218 return (rc); 6219 } 6220 6221 struct mem_desc { 6222 unsigned int base; 6223 unsigned int limit; 6224 unsigned int idx; 6225 }; 6226 6227 static int 6228 mem_desc_cmp(const void *a, const void *b) 6229 { 6230 return ((const struct mem_desc *)a)->base - 6231 ((const struct mem_desc *)b)->base; 6232 } 6233 6234 static void 6235 mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 6236 unsigned int to) 6237 { 6238 unsigned int size; 6239 6240 size = to - from + 1; 6241 if (size == 0) 6242 return; 6243 6244 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 6245 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 6246 } 6247 6248 static int 6249 sysctl_meminfo(SYSCTL_HANDLER_ARGS) 6250 { 6251 struct adapter *sc = arg1; 6252 struct sbuf *sb; 6253 int rc, i, n; 6254 uint32_t lo, hi, used, alloc; 6255 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 6256 static const char *region[] = { 6257 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 6258 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 6259 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 6260 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 6261 "RQUDP region:", "PBL region:", "TXPBL region:", 6262 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 6263 "On-chip queues:" 6264 }; 6265 struct mem_desc avail[4]; 6266 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 6267 struct mem_desc *md = mem; 6268 6269 rc = sysctl_wire_old_buffer(req, 0); 6270 if (rc != 0) 6271 return (rc); 6272 6273 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6274 if (sb == NULL) 6275 return (ENOMEM); 6276 6277 for (i = 0; i < nitems(mem); i++) { 6278 mem[i].limit = 0; 6279 mem[i].idx = i; 6280 } 6281 6282 /* Find and sort the populated memory ranges */ 6283 i = 0; 6284 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 6285 if (lo & F_EDRAM0_ENABLE) { 6286 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 6287 avail[i].base = G_EDRAM0_BASE(hi) << 20; 6288 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 6289 avail[i].idx = 0; 6290 i++; 6291 } 6292 if (lo & F_EDRAM1_ENABLE) { 6293 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 6294 avail[i].base = G_EDRAM1_BASE(hi) << 20; 6295 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 6296 avail[i].idx = 1; 6297 i++; 6298 } 6299 if (lo & F_EXT_MEM_ENABLE) { 6300 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 6301 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 6302 avail[i].limit = avail[i].base + 6303 (G_EXT_MEM_SIZE(hi) << 20); 6304 avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */ 6305 i++; 6306 } 6307 if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) { 6308 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 6309 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 6310 avail[i].limit = avail[i].base + 6311 (G_EXT_MEM1_SIZE(hi) << 20); 6312 avail[i].idx = 4; 6313 i++; 6314 } 6315 if (!i) /* no memory available */ 6316 return 0; 6317 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 6318 6319 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 6320 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 6321 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 6322 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 6323 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 6324 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 6325 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 6326 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 6327 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 6328 6329 /* the next few have explicit upper bounds */ 6330 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 6331 md->limit = md->base - 1 + 6332 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 6333 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 6334 md++; 6335 6336 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 6337 md->limit = md->base - 1 + 6338 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 6339 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 6340 md++; 6341 6342 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6343 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 6344 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 6345 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 6346 } else { 6347 md->base = 0; 6348 md->idx = nitems(region); /* hide it */ 6349 } 6350 md++; 6351 6352 #define ulp_region(reg) \ 6353 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 6354 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 6355 6356 ulp_region(RX_ISCSI); 6357 ulp_region(RX_TDDP); 6358 ulp_region(TX_TPT); 6359 ulp_region(RX_STAG); 6360 ulp_region(RX_RQ); 6361 ulp_region(RX_RQUDP); 6362 ulp_region(RX_PBL); 6363 ulp_region(TX_PBL); 6364 #undef ulp_region 6365 6366 md->base = 0; 6367 md->idx = nitems(region); 6368 if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) { 6369 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); 6370 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc, 6371 A_SGE_DBVFIFO_SIZE))) << 2) - 1; 6372 } 6373 md++; 6374 6375 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 6376 md->limit = md->base + sc->tids.ntids - 1; 6377 md++; 6378 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 6379 md->limit = md->base + sc->tids.ntids - 1; 6380 md++; 6381 6382 md->base = sc->vres.ocq.start; 6383 if (sc->vres.ocq.size) 6384 md->limit = md->base + sc->vres.ocq.size - 1; 6385 else 6386 md->idx = nitems(region); /* hide it */ 6387 md++; 6388 6389 /* add any address-space holes, there can be up to 3 */ 6390 for (n = 0; n < i - 1; n++) 6391 if (avail[n].limit < avail[n + 1].base) 6392 (md++)->base = avail[n].limit; 6393 if (avail[n].limit) 6394 (md++)->base = avail[n].limit; 6395 6396 n = md - mem; 6397 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 6398 6399 for (lo = 0; lo < i; lo++) 6400 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 6401 avail[lo].limit - 1); 6402 6403 sbuf_printf(sb, "\n"); 6404 for (i = 0; i < n; i++) { 6405 if (mem[i].idx >= nitems(region)) 6406 continue; /* skip holes */ 6407 if (!mem[i].limit) 6408 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 6409 mem_region_show(sb, region[mem[i].idx], mem[i].base, 6410 mem[i].limit); 6411 } 6412 6413 sbuf_printf(sb, "\n"); 6414 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 6415 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 6416 mem_region_show(sb, "uP RAM:", lo, hi); 6417 6418 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 6419 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 6420 mem_region_show(sb, "uP Extmem2:", lo, hi); 6421 6422 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 6423 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 6424 G_PMRXMAXPAGE(lo), 6425 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 6426 (lo & F_PMRXNUMCHN) ? 2 : 1); 6427 6428 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 6429 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 6430 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 6431 G_PMTXMAXPAGE(lo), 6432 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 6433 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 6434 sbuf_printf(sb, "%u p-structs\n", 6435 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 6436 6437 for (i = 0; i < 4; i++) { 6438 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 6439 if (is_t4(sc)) { 6440 used = G_USED(lo); 6441 alloc = G_ALLOC(lo); 6442 } else { 6443 used = G_T5_USED(lo); 6444 alloc = G_T5_ALLOC(lo); 6445 } 6446 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 6447 i, used, alloc); 6448 } 6449 for (i = 0; i < 4; i++) { 6450 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 6451 if (is_t4(sc)) { 6452 used = G_USED(lo); 6453 alloc = G_ALLOC(lo); 6454 } else { 6455 used = G_T5_USED(lo); 6456 alloc = G_T5_ALLOC(lo); 6457 } 6458 sbuf_printf(sb, 6459 "\nLoopback %d using %u pages out of %u allocated", 6460 i, used, alloc); 6461 } 6462 6463 rc = sbuf_finish(sb); 6464 sbuf_delete(sb); 6465 6466 return (rc); 6467 } 6468 6469 static inline void 6470 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) 6471 { 6472 *mask = x | y; 6473 y = htobe64(y); 6474 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); 6475 } 6476 6477 static int 6478 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) 6479 { 6480 struct adapter *sc = arg1; 6481 struct sbuf *sb; 6482 int rc, i, n; 6483 6484 rc = sysctl_wire_old_buffer(req, 0); 6485 if (rc != 0) 6486 return (rc); 6487 6488 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6489 if (sb == NULL) 6490 return (ENOMEM); 6491 6492 sbuf_printf(sb, 6493 "Idx Ethernet address Mask Vld Ports PF" 6494 " VF Replication P0 P1 P2 P3 ML"); 6495 n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES : 6496 NUM_MPS_T5_CLS_SRAM_L_INSTANCES; 6497 for (i = 0; i < n; i++) { 6498 uint64_t tcamx, tcamy, mask; 6499 uint32_t cls_lo, cls_hi; 6500 uint8_t addr[ETHER_ADDR_LEN]; 6501 6502 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); 6503 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); 6504 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 6505 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 6506 6507 if (tcamx & tcamy) 6508 continue; 6509 6510 tcamxy2valmask(tcamx, tcamy, addr, &mask); 6511 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" 6512 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], 6513 addr[3], addr[4], addr[5], (uintmax_t)mask, 6514 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', 6515 G_PORTMAP(cls_hi), G_PF(cls_lo), 6516 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); 6517 6518 if (cls_lo & F_REPLICATE) { 6519 struct fw_ldst_cmd ldst_cmd; 6520 6521 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 6522 ldst_cmd.op_to_addrspace = 6523 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 6524 F_FW_CMD_REQUEST | F_FW_CMD_READ | 6525 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 6526 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 6527 ldst_cmd.u.mps.rplc.fid_idx = 6528 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 6529 V_FW_LDST_CMD_IDX(i)); 6530 6531 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 6532 "t4mps"); 6533 if (rc) 6534 break; 6535 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 6536 sizeof(ldst_cmd), &ldst_cmd); 6537 end_synchronized_op(sc, 0); 6538 6539 if (rc != 0) { 6540 sbuf_printf(sb, 6541 " ------------ error %3u ------------", rc); 6542 rc = 0; 6543 } else { 6544 sbuf_printf(sb, " %08x %08x %08x %08x", 6545 be32toh(ldst_cmd.u.mps.rplc.rplc127_96), 6546 be32toh(ldst_cmd.u.mps.rplc.rplc95_64), 6547 be32toh(ldst_cmd.u.mps.rplc.rplc63_32), 6548 be32toh(ldst_cmd.u.mps.rplc.rplc31_0)); 6549 } 6550 } else 6551 sbuf_printf(sb, "%36s", ""); 6552 6553 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), 6554 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), 6555 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); 6556 } 6557 6558 if (rc) 6559 (void) sbuf_finish(sb); 6560 else 6561 rc = sbuf_finish(sb); 6562 sbuf_delete(sb); 6563 6564 return (rc); 6565 } 6566 6567 static int 6568 sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 6569 { 6570 struct adapter *sc = arg1; 6571 struct sbuf *sb; 6572 int rc; 6573 uint16_t mtus[NMTUS]; 6574 6575 rc = sysctl_wire_old_buffer(req, 0); 6576 if (rc != 0) 6577 return (rc); 6578 6579 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6580 if (sb == NULL) 6581 return (ENOMEM); 6582 6583 t4_read_mtu_tbl(sc, mtus, NULL); 6584 6585 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 6586 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 6587 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 6588 mtus[14], mtus[15]); 6589 6590 rc = sbuf_finish(sb); 6591 sbuf_delete(sb); 6592 6593 return (rc); 6594 } 6595 6596 static int 6597 sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 6598 { 6599 struct adapter *sc = arg1; 6600 struct sbuf *sb; 6601 int rc, i; 6602 uint32_t cnt[PM_NSTATS]; 6603 uint64_t cyc[PM_NSTATS]; 6604 static const char *rx_stats[] = { 6605 "Read:", "Write bypass:", "Write mem:", "Flush:" 6606 }; 6607 static const char *tx_stats[] = { 6608 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:" 6609 }; 6610 6611 rc = sysctl_wire_old_buffer(req, 0); 6612 if (rc != 0) 6613 return (rc); 6614 6615 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6616 if (sb == NULL) 6617 return (ENOMEM); 6618 6619 t4_pmtx_get_stats(sc, cnt, cyc); 6620 sbuf_printf(sb, " Tx pcmds Tx bytes"); 6621 for (i = 0; i < ARRAY_SIZE(tx_stats); i++) 6622 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i], 6623 cyc[i]); 6624 6625 t4_pmrx_get_stats(sc, cnt, cyc); 6626 sbuf_printf(sb, "\n Rx pcmds Rx bytes"); 6627 for (i = 0; i < ARRAY_SIZE(rx_stats); i++) 6628 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i], 6629 cyc[i]); 6630 6631 rc = sbuf_finish(sb); 6632 sbuf_delete(sb); 6633 6634 return (rc); 6635 } 6636 6637 static int 6638 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 6639 { 6640 struct adapter *sc = arg1; 6641 struct sbuf *sb; 6642 int rc; 6643 struct tp_rdma_stats stats; 6644 6645 rc = sysctl_wire_old_buffer(req, 0); 6646 if (rc != 0) 6647 return (rc); 6648 6649 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6650 if (sb == NULL) 6651 return (ENOMEM); 6652 6653 t4_tp_get_rdma_stats(sc, &stats); 6654 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 6655 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 6656 6657 rc = sbuf_finish(sb); 6658 sbuf_delete(sb); 6659 6660 return (rc); 6661 } 6662 6663 static int 6664 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 6665 { 6666 struct adapter *sc = arg1; 6667 struct sbuf *sb; 6668 int rc; 6669 struct tp_tcp_stats v4, v6; 6670 6671 rc = sysctl_wire_old_buffer(req, 0); 6672 if (rc != 0) 6673 return (rc); 6674 6675 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6676 if (sb == NULL) 6677 return (ENOMEM); 6678 6679 t4_tp_get_tcp_stats(sc, &v4, &v6); 6680 sbuf_printf(sb, 6681 " IP IPv6\n"); 6682 sbuf_printf(sb, "OutRsts: %20u %20u\n", 6683 v4.tcpOutRsts, v6.tcpOutRsts); 6684 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 6685 v4.tcpInSegs, v6.tcpInSegs); 6686 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 6687 v4.tcpOutSegs, v6.tcpOutSegs); 6688 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 6689 v4.tcpRetransSegs, v6.tcpRetransSegs); 6690 6691 rc = sbuf_finish(sb); 6692 sbuf_delete(sb); 6693 6694 return (rc); 6695 } 6696 6697 static int 6698 sysctl_tids(SYSCTL_HANDLER_ARGS) 6699 { 6700 struct adapter *sc = arg1; 6701 struct sbuf *sb; 6702 int rc; 6703 struct tid_info *t = &sc->tids; 6704 6705 rc = sysctl_wire_old_buffer(req, 0); 6706 if (rc != 0) 6707 return (rc); 6708 6709 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6710 if (sb == NULL) 6711 return (ENOMEM); 6712 6713 if (t->natids) { 6714 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 6715 t->atids_in_use); 6716 } 6717 6718 if (t->ntids) { 6719 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6720 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 6721 6722 if (b) { 6723 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 6724 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6725 t->ntids - 1); 6726 } else { 6727 sbuf_printf(sb, "TID range: %u-%u", 6728 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6729 t->ntids - 1); 6730 } 6731 } else 6732 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 6733 sbuf_printf(sb, ", in use: %u\n", 6734 atomic_load_acq_int(&t->tids_in_use)); 6735 } 6736 6737 if (t->nstids) { 6738 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 6739 t->stid_base + t->nstids - 1, t->stids_in_use); 6740 } 6741 6742 if (t->nftids) { 6743 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 6744 t->ftid_base + t->nftids - 1); 6745 } 6746 6747 if (t->netids) { 6748 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base, 6749 t->etid_base + t->netids - 1); 6750 } 6751 6752 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 6753 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 6754 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 6755 6756 rc = sbuf_finish(sb); 6757 sbuf_delete(sb); 6758 6759 return (rc); 6760 } 6761 6762 static int 6763 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 6764 { 6765 struct adapter *sc = arg1; 6766 struct sbuf *sb; 6767 int rc; 6768 struct tp_err_stats stats; 6769 6770 rc = sysctl_wire_old_buffer(req, 0); 6771 if (rc != 0) 6772 return (rc); 6773 6774 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6775 if (sb == NULL) 6776 return (ENOMEM); 6777 6778 t4_tp_get_err_stats(sc, &stats); 6779 6780 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6781 "channel 3\n"); 6782 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 6783 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 6784 stats.macInErrs[3]); 6785 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 6786 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 6787 stats.hdrInErrs[3]); 6788 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 6789 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 6790 stats.tcpInErrs[3]); 6791 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 6792 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 6793 stats.tcp6InErrs[3]); 6794 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 6795 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 6796 stats.tnlCongDrops[3]); 6797 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 6798 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 6799 stats.tnlTxDrops[3]); 6800 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 6801 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 6802 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 6803 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 6804 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 6805 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 6806 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 6807 stats.ofldNoNeigh, stats.ofldCongDefer); 6808 6809 rc = sbuf_finish(sb); 6810 sbuf_delete(sb); 6811 6812 return (rc); 6813 } 6814 6815 struct field_desc { 6816 const char *name; 6817 u_int start; 6818 u_int width; 6819 }; 6820 6821 static void 6822 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) 6823 { 6824 char buf[32]; 6825 int line_size = 0; 6826 6827 while (f->name) { 6828 uint64_t mask = (1ULL << f->width) - 1; 6829 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, 6830 ((uintmax_t)v >> f->start) & mask); 6831 6832 if (line_size + len >= 79) { 6833 line_size = 8; 6834 sbuf_printf(sb, "\n "); 6835 } 6836 sbuf_printf(sb, "%s ", buf); 6837 line_size += len + 1; 6838 f++; 6839 } 6840 sbuf_printf(sb, "\n"); 6841 } 6842 6843 static struct field_desc tp_la0[] = { 6844 { "RcfOpCodeOut", 60, 4 }, 6845 { "State", 56, 4 }, 6846 { "WcfState", 52, 4 }, 6847 { "RcfOpcSrcOut", 50, 2 }, 6848 { "CRxError", 49, 1 }, 6849 { "ERxError", 48, 1 }, 6850 { "SanityFailed", 47, 1 }, 6851 { "SpuriousMsg", 46, 1 }, 6852 { "FlushInputMsg", 45, 1 }, 6853 { "FlushInputCpl", 44, 1 }, 6854 { "RssUpBit", 43, 1 }, 6855 { "RssFilterHit", 42, 1 }, 6856 { "Tid", 32, 10 }, 6857 { "InitTcb", 31, 1 }, 6858 { "LineNumber", 24, 7 }, 6859 { "Emsg", 23, 1 }, 6860 { "EdataOut", 22, 1 }, 6861 { "Cmsg", 21, 1 }, 6862 { "CdataOut", 20, 1 }, 6863 { "EreadPdu", 19, 1 }, 6864 { "CreadPdu", 18, 1 }, 6865 { "TunnelPkt", 17, 1 }, 6866 { "RcfPeerFin", 16, 1 }, 6867 { "RcfReasonOut", 12, 4 }, 6868 { "TxCchannel", 10, 2 }, 6869 { "RcfTxChannel", 8, 2 }, 6870 { "RxEchannel", 6, 2 }, 6871 { "RcfRxChannel", 5, 1 }, 6872 { "RcfDataOutSrdy", 4, 1 }, 6873 { "RxDvld", 3, 1 }, 6874 { "RxOoDvld", 2, 1 }, 6875 { "RxCongestion", 1, 1 }, 6876 { "TxCongestion", 0, 1 }, 6877 { NULL } 6878 }; 6879 6880 static struct field_desc tp_la1[] = { 6881 { "CplCmdIn", 56, 8 }, 6882 { "CplCmdOut", 48, 8 }, 6883 { "ESynOut", 47, 1 }, 6884 { "EAckOut", 46, 1 }, 6885 { "EFinOut", 45, 1 }, 6886 { "ERstOut", 44, 1 }, 6887 { "SynIn", 43, 1 }, 6888 { "AckIn", 42, 1 }, 6889 { "FinIn", 41, 1 }, 6890 { "RstIn", 40, 1 }, 6891 { "DataIn", 39, 1 }, 6892 { "DataInVld", 38, 1 }, 6893 { "PadIn", 37, 1 }, 6894 { "RxBufEmpty", 36, 1 }, 6895 { "RxDdp", 35, 1 }, 6896 { "RxFbCongestion", 34, 1 }, 6897 { "TxFbCongestion", 33, 1 }, 6898 { "TxPktSumSrdy", 32, 1 }, 6899 { "RcfUlpType", 28, 4 }, 6900 { "Eread", 27, 1 }, 6901 { "Ebypass", 26, 1 }, 6902 { "Esave", 25, 1 }, 6903 { "Static0", 24, 1 }, 6904 { "Cread", 23, 1 }, 6905 { "Cbypass", 22, 1 }, 6906 { "Csave", 21, 1 }, 6907 { "CPktOut", 20, 1 }, 6908 { "RxPagePoolFull", 18, 2 }, 6909 { "RxLpbkPkt", 17, 1 }, 6910 { "TxLpbkPkt", 16, 1 }, 6911 { "RxVfValid", 15, 1 }, 6912 { "SynLearned", 14, 1 }, 6913 { "SetDelEntry", 13, 1 }, 6914 { "SetInvEntry", 12, 1 }, 6915 { "CpcmdDvld", 11, 1 }, 6916 { "CpcmdSave", 10, 1 }, 6917 { "RxPstructsFull", 8, 2 }, 6918 { "EpcmdDvld", 7, 1 }, 6919 { "EpcmdFlush", 6, 1 }, 6920 { "EpcmdTrimPrefix", 5, 1 }, 6921 { "EpcmdTrimPostfix", 4, 1 }, 6922 { "ERssIp4Pkt", 3, 1 }, 6923 { "ERssIp6Pkt", 2, 1 }, 6924 { "ERssTcpUdpPkt", 1, 1 }, 6925 { "ERssFceFipPkt", 0, 1 }, 6926 { NULL } 6927 }; 6928 6929 static struct field_desc tp_la2[] = { 6930 { "CplCmdIn", 56, 8 }, 6931 { "MpsVfVld", 55, 1 }, 6932 { "MpsPf", 52, 3 }, 6933 { "MpsVf", 44, 8 }, 6934 { "SynIn", 43, 1 }, 6935 { "AckIn", 42, 1 }, 6936 { "FinIn", 41, 1 }, 6937 { "RstIn", 40, 1 }, 6938 { "DataIn", 39, 1 }, 6939 { "DataInVld", 38, 1 }, 6940 { "PadIn", 37, 1 }, 6941 { "RxBufEmpty", 36, 1 }, 6942 { "RxDdp", 35, 1 }, 6943 { "RxFbCongestion", 34, 1 }, 6944 { "TxFbCongestion", 33, 1 }, 6945 { "TxPktSumSrdy", 32, 1 }, 6946 { "RcfUlpType", 28, 4 }, 6947 { "Eread", 27, 1 }, 6948 { "Ebypass", 26, 1 }, 6949 { "Esave", 25, 1 }, 6950 { "Static0", 24, 1 }, 6951 { "Cread", 23, 1 }, 6952 { "Cbypass", 22, 1 }, 6953 { "Csave", 21, 1 }, 6954 { "CPktOut", 20, 1 }, 6955 { "RxPagePoolFull", 18, 2 }, 6956 { "RxLpbkPkt", 17, 1 }, 6957 { "TxLpbkPkt", 16, 1 }, 6958 { "RxVfValid", 15, 1 }, 6959 { "SynLearned", 14, 1 }, 6960 { "SetDelEntry", 13, 1 }, 6961 { "SetInvEntry", 12, 1 }, 6962 { "CpcmdDvld", 11, 1 }, 6963 { "CpcmdSave", 10, 1 }, 6964 { "RxPstructsFull", 8, 2 }, 6965 { "EpcmdDvld", 7, 1 }, 6966 { "EpcmdFlush", 6, 1 }, 6967 { "EpcmdTrimPrefix", 5, 1 }, 6968 { "EpcmdTrimPostfix", 4, 1 }, 6969 { "ERssIp4Pkt", 3, 1 }, 6970 { "ERssIp6Pkt", 2, 1 }, 6971 { "ERssTcpUdpPkt", 1, 1 }, 6972 { "ERssFceFipPkt", 0, 1 }, 6973 { NULL } 6974 }; 6975 6976 static void 6977 tp_la_show(struct sbuf *sb, uint64_t *p, int idx) 6978 { 6979 6980 field_desc_show(sb, *p, tp_la0); 6981 } 6982 6983 static void 6984 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) 6985 { 6986 6987 if (idx) 6988 sbuf_printf(sb, "\n"); 6989 field_desc_show(sb, p[0], tp_la0); 6990 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6991 field_desc_show(sb, p[1], tp_la0); 6992 } 6993 6994 static void 6995 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) 6996 { 6997 6998 if (idx) 6999 sbuf_printf(sb, "\n"); 7000 field_desc_show(sb, p[0], tp_la0); 7001 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 7002 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); 7003 } 7004 7005 static int 7006 sysctl_tp_la(SYSCTL_HANDLER_ARGS) 7007 { 7008 struct adapter *sc = arg1; 7009 struct sbuf *sb; 7010 uint64_t *buf, *p; 7011 int rc; 7012 u_int i, inc; 7013 void (*show_func)(struct sbuf *, uint64_t *, int); 7014 7015 rc = sysctl_wire_old_buffer(req, 0); 7016 if (rc != 0) 7017 return (rc); 7018 7019 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7020 if (sb == NULL) 7021 return (ENOMEM); 7022 7023 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); 7024 7025 t4_tp_read_la(sc, buf, NULL); 7026 p = buf; 7027 7028 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { 7029 case 2: 7030 inc = 2; 7031 show_func = tp_la_show2; 7032 break; 7033 case 3: 7034 inc = 2; 7035 show_func = tp_la_show3; 7036 break; 7037 default: 7038 inc = 1; 7039 show_func = tp_la_show; 7040 } 7041 7042 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) 7043 (*show_func)(sb, p, i); 7044 7045 rc = sbuf_finish(sb); 7046 sbuf_delete(sb); 7047 free(buf, M_CXGBE); 7048 return (rc); 7049 } 7050 7051 static int 7052 sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 7053 { 7054 struct adapter *sc = arg1; 7055 struct sbuf *sb; 7056 int rc; 7057 u64 nrate[NCHAN], orate[NCHAN]; 7058 7059 rc = sysctl_wire_old_buffer(req, 0); 7060 if (rc != 0) 7061 return (rc); 7062 7063 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7064 if (sb == NULL) 7065 return (ENOMEM); 7066 7067 t4_get_chan_txrate(sc, nrate, orate); 7068 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 7069 "channel 3\n"); 7070 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 7071 nrate[0], nrate[1], nrate[2], nrate[3]); 7072 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 7073 orate[0], orate[1], orate[2], orate[3]); 7074 7075 rc = sbuf_finish(sb); 7076 sbuf_delete(sb); 7077 7078 return (rc); 7079 } 7080 7081 static int 7082 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) 7083 { 7084 struct adapter *sc = arg1; 7085 struct sbuf *sb; 7086 uint32_t *buf, *p; 7087 int rc, i; 7088 7089 rc = sysctl_wire_old_buffer(req, 0); 7090 if (rc != 0) 7091 return (rc); 7092 7093 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7094 if (sb == NULL) 7095 return (ENOMEM); 7096 7097 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, 7098 M_ZERO | M_WAITOK); 7099 7100 t4_ulprx_read_la(sc, buf); 7101 p = buf; 7102 7103 sbuf_printf(sb, " Pcmd Type Message" 7104 " Data"); 7105 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { 7106 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", 7107 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); 7108 } 7109 7110 rc = sbuf_finish(sb); 7111 sbuf_delete(sb); 7112 free(buf, M_CXGBE); 7113 return (rc); 7114 } 7115 7116 static int 7117 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 7118 { 7119 struct adapter *sc = arg1; 7120 struct sbuf *sb; 7121 int rc, v; 7122 7123 rc = sysctl_wire_old_buffer(req, 0); 7124 if (rc != 0) 7125 return (rc); 7126 7127 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7128 if (sb == NULL) 7129 return (ENOMEM); 7130 7131 v = t4_read_reg(sc, A_SGE_STAT_CFG); 7132 if (G_STATSOURCE_T5(v) == 7) { 7133 if (G_STATMODE(v) == 0) { 7134 sbuf_printf(sb, "total %d, incomplete %d", 7135 t4_read_reg(sc, A_SGE_STAT_TOTAL), 7136 t4_read_reg(sc, A_SGE_STAT_MATCH)); 7137 } else if (G_STATMODE(v) == 1) { 7138 sbuf_printf(sb, "total %d, data overflow %d", 7139 t4_read_reg(sc, A_SGE_STAT_TOTAL), 7140 t4_read_reg(sc, A_SGE_STAT_MATCH)); 7141 } 7142 } 7143 rc = sbuf_finish(sb); 7144 sbuf_delete(sb); 7145 7146 return (rc); 7147 } 7148 #endif 7149 7150 static uint32_t 7151 fconf_to_mode(uint32_t fconf) 7152 { 7153 uint32_t mode; 7154 7155 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 7156 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 7157 7158 if (fconf & F_FRAGMENTATION) 7159 mode |= T4_FILTER_IP_FRAGMENT; 7160 7161 if (fconf & F_MPSHITTYPE) 7162 mode |= T4_FILTER_MPS_HIT_TYPE; 7163 7164 if (fconf & F_MACMATCH) 7165 mode |= T4_FILTER_MAC_IDX; 7166 7167 if (fconf & F_ETHERTYPE) 7168 mode |= T4_FILTER_ETH_TYPE; 7169 7170 if (fconf & F_PROTOCOL) 7171 mode |= T4_FILTER_IP_PROTO; 7172 7173 if (fconf & F_TOS) 7174 mode |= T4_FILTER_IP_TOS; 7175 7176 if (fconf & F_VLAN) 7177 mode |= T4_FILTER_VLAN; 7178 7179 if (fconf & F_VNIC_ID) 7180 mode |= T4_FILTER_VNIC; 7181 7182 if (fconf & F_PORT) 7183 mode |= T4_FILTER_PORT; 7184 7185 if (fconf & F_FCOE) 7186 mode |= T4_FILTER_FCoE; 7187 7188 return (mode); 7189 } 7190 7191 static uint32_t 7192 mode_to_fconf(uint32_t mode) 7193 { 7194 uint32_t fconf = 0; 7195 7196 if (mode & T4_FILTER_IP_FRAGMENT) 7197 fconf |= F_FRAGMENTATION; 7198 7199 if (mode & T4_FILTER_MPS_HIT_TYPE) 7200 fconf |= F_MPSHITTYPE; 7201 7202 if (mode & T4_FILTER_MAC_IDX) 7203 fconf |= F_MACMATCH; 7204 7205 if (mode & T4_FILTER_ETH_TYPE) 7206 fconf |= F_ETHERTYPE; 7207 7208 if (mode & T4_FILTER_IP_PROTO) 7209 fconf |= F_PROTOCOL; 7210 7211 if (mode & T4_FILTER_IP_TOS) 7212 fconf |= F_TOS; 7213 7214 if (mode & T4_FILTER_VLAN) 7215 fconf |= F_VLAN; 7216 7217 if (mode & T4_FILTER_VNIC) 7218 fconf |= F_VNIC_ID; 7219 7220 if (mode & T4_FILTER_PORT) 7221 fconf |= F_PORT; 7222 7223 if (mode & T4_FILTER_FCoE) 7224 fconf |= F_FCOE; 7225 7226 return (fconf); 7227 } 7228 7229 static uint32_t 7230 fspec_to_fconf(struct t4_filter_specification *fs) 7231 { 7232 uint32_t fconf = 0; 7233 7234 if (fs->val.frag || fs->mask.frag) 7235 fconf |= F_FRAGMENTATION; 7236 7237 if (fs->val.matchtype || fs->mask.matchtype) 7238 fconf |= F_MPSHITTYPE; 7239 7240 if (fs->val.macidx || fs->mask.macidx) 7241 fconf |= F_MACMATCH; 7242 7243 if (fs->val.ethtype || fs->mask.ethtype) 7244 fconf |= F_ETHERTYPE; 7245 7246 if (fs->val.proto || fs->mask.proto) 7247 fconf |= F_PROTOCOL; 7248 7249 if (fs->val.tos || fs->mask.tos) 7250 fconf |= F_TOS; 7251 7252 if (fs->val.vlan_vld || fs->mask.vlan_vld) 7253 fconf |= F_VLAN; 7254 7255 if (fs->val.vnic_vld || fs->mask.vnic_vld) 7256 fconf |= F_VNIC_ID; 7257 7258 if (fs->val.iport || fs->mask.iport) 7259 fconf |= F_PORT; 7260 7261 if (fs->val.fcoe || fs->mask.fcoe) 7262 fconf |= F_FCOE; 7263 7264 return (fconf); 7265 } 7266 7267 static int 7268 get_filter_mode(struct adapter *sc, uint32_t *mode) 7269 { 7270 int rc; 7271 uint32_t fconf; 7272 7273 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7274 "t4getfm"); 7275 if (rc) 7276 return (rc); 7277 7278 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 7279 A_TP_VLAN_PRI_MAP); 7280 7281 if (sc->params.tp.vlan_pri_map != fconf) { 7282 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 7283 device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map, 7284 fconf); 7285 } 7286 7287 *mode = fconf_to_mode(fconf); 7288 7289 end_synchronized_op(sc, LOCK_HELD); 7290 return (0); 7291 } 7292 7293 static int 7294 set_filter_mode(struct adapter *sc, uint32_t mode) 7295 { 7296 uint32_t fconf; 7297 int rc; 7298 7299 fconf = mode_to_fconf(mode); 7300 7301 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7302 "t4setfm"); 7303 if (rc) 7304 return (rc); 7305 7306 if (sc->tids.ftids_in_use > 0) { 7307 rc = EBUSY; 7308 goto done; 7309 } 7310 7311 #ifdef TCP_OFFLOAD 7312 if (uld_active(sc, ULD_TOM)) { 7313 rc = EBUSY; 7314 goto done; 7315 } 7316 #endif 7317 7318 rc = -t4_set_filter_mode(sc, fconf); 7319 done: 7320 end_synchronized_op(sc, LOCK_HELD); 7321 return (rc); 7322 } 7323 7324 static inline uint64_t 7325 get_filter_hits(struct adapter *sc, uint32_t fid) 7326 { 7327 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 7328 uint64_t hits; 7329 7330 memwin_info(sc, 0, &mw_base, NULL); 7331 off = position_memwin(sc, 0, 7332 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 7333 if (is_t4(sc)) { 7334 hits = t4_read_reg64(sc, mw_base + off + 16); 7335 hits = be64toh(hits); 7336 } else { 7337 hits = t4_read_reg(sc, mw_base + off + 24); 7338 hits = be32toh(hits); 7339 } 7340 7341 return (hits); 7342 } 7343 7344 static int 7345 get_filter(struct adapter *sc, struct t4_filter *t) 7346 { 7347 int i, rc, nfilters = sc->tids.nftids; 7348 struct filter_entry *f; 7349 7350 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7351 "t4getf"); 7352 if (rc) 7353 return (rc); 7354 7355 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 7356 t->idx >= nfilters) { 7357 t->idx = 0xffffffff; 7358 goto done; 7359 } 7360 7361 f = &sc->tids.ftid_tab[t->idx]; 7362 for (i = t->idx; i < nfilters; i++, f++) { 7363 if (f->valid) { 7364 t->idx = i; 7365 t->l2tidx = f->l2t ? f->l2t->idx : 0; 7366 t->smtidx = f->smtidx; 7367 if (f->fs.hitcnts) 7368 t->hits = get_filter_hits(sc, t->idx); 7369 else 7370 t->hits = UINT64_MAX; 7371 t->fs = f->fs; 7372 7373 goto done; 7374 } 7375 } 7376 7377 t->idx = 0xffffffff; 7378 done: 7379 end_synchronized_op(sc, LOCK_HELD); 7380 return (0); 7381 } 7382 7383 static int 7384 set_filter(struct adapter *sc, struct t4_filter *t) 7385 { 7386 unsigned int nfilters, nports; 7387 struct filter_entry *f; 7388 int i, rc; 7389 7390 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 7391 if (rc) 7392 return (rc); 7393 7394 nfilters = sc->tids.nftids; 7395 nports = sc->params.nports; 7396 7397 if (nfilters == 0) { 7398 rc = ENOTSUP; 7399 goto done; 7400 } 7401 7402 if (!(sc->flags & FULL_INIT_DONE)) { 7403 rc = EAGAIN; 7404 goto done; 7405 } 7406 7407 if (t->idx >= nfilters) { 7408 rc = EINVAL; 7409 goto done; 7410 } 7411 7412 /* Validate against the global filter mode */ 7413 if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) != 7414 sc->params.tp.vlan_pri_map) { 7415 rc = E2BIG; 7416 goto done; 7417 } 7418 7419 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 7420 rc = EINVAL; 7421 goto done; 7422 } 7423 7424 if (t->fs.val.iport >= nports) { 7425 rc = EINVAL; 7426 goto done; 7427 } 7428 7429 /* Can't specify an iq if not steering to it */ 7430 if (!t->fs.dirsteer && t->fs.iq) { 7431 rc = EINVAL; 7432 goto done; 7433 } 7434 7435 /* IPv6 filter idx must be 4 aligned */ 7436 if (t->fs.type == 1 && 7437 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 7438 rc = EINVAL; 7439 goto done; 7440 } 7441 7442 if (sc->tids.ftid_tab == NULL) { 7443 KASSERT(sc->tids.ftids_in_use == 0, 7444 ("%s: no memory allocated but filters_in_use > 0", 7445 __func__)); 7446 7447 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 7448 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 7449 if (sc->tids.ftid_tab == NULL) { 7450 rc = ENOMEM; 7451 goto done; 7452 } 7453 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 7454 } 7455 7456 for (i = 0; i < 4; i++) { 7457 f = &sc->tids.ftid_tab[t->idx + i]; 7458 7459 if (f->pending || f->valid) { 7460 rc = EBUSY; 7461 goto done; 7462 } 7463 if (f->locked) { 7464 rc = EPERM; 7465 goto done; 7466 } 7467 7468 if (t->fs.type == 0) 7469 break; 7470 } 7471 7472 f = &sc->tids.ftid_tab[t->idx]; 7473 f->fs = t->fs; 7474 7475 rc = set_filter_wr(sc, t->idx); 7476 done: 7477 end_synchronized_op(sc, 0); 7478 7479 if (rc == 0) { 7480 mtx_lock(&sc->tids.ftid_lock); 7481 for (;;) { 7482 if (f->pending == 0) { 7483 rc = f->valid ? 0 : EIO; 7484 break; 7485 } 7486 7487 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7488 PCATCH, "t4setfw", 0)) { 7489 rc = EINPROGRESS; 7490 break; 7491 } 7492 } 7493 mtx_unlock(&sc->tids.ftid_lock); 7494 } 7495 return (rc); 7496 } 7497 7498 static int 7499 del_filter(struct adapter *sc, struct t4_filter *t) 7500 { 7501 unsigned int nfilters; 7502 struct filter_entry *f; 7503 int rc; 7504 7505 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 7506 if (rc) 7507 return (rc); 7508 7509 nfilters = sc->tids.nftids; 7510 7511 if (nfilters == 0) { 7512 rc = ENOTSUP; 7513 goto done; 7514 } 7515 7516 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 7517 t->idx >= nfilters) { 7518 rc = EINVAL; 7519 goto done; 7520 } 7521 7522 if (!(sc->flags & FULL_INIT_DONE)) { 7523 rc = EAGAIN; 7524 goto done; 7525 } 7526 7527 f = &sc->tids.ftid_tab[t->idx]; 7528 7529 if (f->pending) { 7530 rc = EBUSY; 7531 goto done; 7532 } 7533 if (f->locked) { 7534 rc = EPERM; 7535 goto done; 7536 } 7537 7538 if (f->valid) { 7539 t->fs = f->fs; /* extra info for the caller */ 7540 rc = del_filter_wr(sc, t->idx); 7541 } 7542 7543 done: 7544 end_synchronized_op(sc, 0); 7545 7546 if (rc == 0) { 7547 mtx_lock(&sc->tids.ftid_lock); 7548 for (;;) { 7549 if (f->pending == 0) { 7550 rc = f->valid ? EIO : 0; 7551 break; 7552 } 7553 7554 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7555 PCATCH, "t4delfw", 0)) { 7556 rc = EINPROGRESS; 7557 break; 7558 } 7559 } 7560 mtx_unlock(&sc->tids.ftid_lock); 7561 } 7562 7563 return (rc); 7564 } 7565 7566 static void 7567 clear_filter(struct filter_entry *f) 7568 { 7569 if (f->l2t) 7570 t4_l2t_release(f->l2t); 7571 7572 bzero(f, sizeof (*f)); 7573 } 7574 7575 static int 7576 set_filter_wr(struct adapter *sc, int fidx) 7577 { 7578 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7579 struct fw_filter_wr *fwr; 7580 unsigned int ftid; 7581 struct wrq_cookie cookie; 7582 7583 ASSERT_SYNCHRONIZED_OP(sc); 7584 7585 if (f->fs.newdmac || f->fs.newvlan) { 7586 /* This filter needs an L2T entry; allocate one. */ 7587 f->l2t = t4_l2t_alloc_switching(sc->l2t); 7588 if (f->l2t == NULL) 7589 return (EAGAIN); 7590 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 7591 f->fs.dmac)) { 7592 t4_l2t_release(f->l2t); 7593 f->l2t = NULL; 7594 return (ENOMEM); 7595 } 7596 } 7597 7598 ftid = sc->tids.ftid_base + fidx; 7599 7600 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 7601 if (fwr == NULL) 7602 return (ENOMEM); 7603 bzero(fwr, sizeof(*fwr)); 7604 7605 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 7606 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 7607 fwr->tid_to_iq = 7608 htobe32(V_FW_FILTER_WR_TID(ftid) | 7609 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 7610 V_FW_FILTER_WR_NOREPLY(0) | 7611 V_FW_FILTER_WR_IQ(f->fs.iq)); 7612 fwr->del_filter_to_l2tix = 7613 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 7614 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 7615 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 7616 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 7617 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 7618 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 7619 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 7620 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 7621 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 7622 f->fs.newvlan == VLAN_REWRITE) | 7623 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 7624 f->fs.newvlan == VLAN_REWRITE) | 7625 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 7626 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 7627 V_FW_FILTER_WR_PRIO(f->fs.prio) | 7628 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 7629 fwr->ethtype = htobe16(f->fs.val.ethtype); 7630 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 7631 fwr->frag_to_ovlan_vldm = 7632 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 7633 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 7634 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 7635 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 7636 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 7637 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 7638 fwr->smac_sel = 0; 7639 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 7640 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 7641 fwr->maci_to_matchtypem = 7642 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 7643 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 7644 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 7645 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 7646 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 7647 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 7648 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 7649 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 7650 fwr->ptcl = f->fs.val.proto; 7651 fwr->ptclm = f->fs.mask.proto; 7652 fwr->ttyp = f->fs.val.tos; 7653 fwr->ttypm = f->fs.mask.tos; 7654 fwr->ivlan = htobe16(f->fs.val.vlan); 7655 fwr->ivlanm = htobe16(f->fs.mask.vlan); 7656 fwr->ovlan = htobe16(f->fs.val.vnic); 7657 fwr->ovlanm = htobe16(f->fs.mask.vnic); 7658 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 7659 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 7660 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 7661 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 7662 fwr->lp = htobe16(f->fs.val.dport); 7663 fwr->lpm = htobe16(f->fs.mask.dport); 7664 fwr->fp = htobe16(f->fs.val.sport); 7665 fwr->fpm = htobe16(f->fs.mask.sport); 7666 if (f->fs.newsmac) 7667 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 7668 7669 f->pending = 1; 7670 sc->tids.ftids_in_use++; 7671 7672 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 7673 return (0); 7674 } 7675 7676 static int 7677 del_filter_wr(struct adapter *sc, int fidx) 7678 { 7679 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7680 struct fw_filter_wr *fwr; 7681 unsigned int ftid; 7682 struct wrq_cookie cookie; 7683 7684 ftid = sc->tids.ftid_base + fidx; 7685 7686 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 7687 if (fwr == NULL) 7688 return (ENOMEM); 7689 bzero(fwr, sizeof (*fwr)); 7690 7691 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 7692 7693 f->pending = 1; 7694 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 7695 return (0); 7696 } 7697 7698 int 7699 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 7700 { 7701 struct adapter *sc = iq->adapter; 7702 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 7703 unsigned int idx = GET_TID(rpl); 7704 unsigned int rc; 7705 struct filter_entry *f; 7706 7707 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 7708 rss->opcode)); 7709 7710 if (is_ftid(sc, idx)) { 7711 7712 idx -= sc->tids.ftid_base; 7713 f = &sc->tids.ftid_tab[idx]; 7714 rc = G_COOKIE(rpl->cookie); 7715 7716 mtx_lock(&sc->tids.ftid_lock); 7717 if (rc == FW_FILTER_WR_FLT_ADDED) { 7718 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 7719 __func__, idx)); 7720 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 7721 f->pending = 0; /* asynchronous setup completed */ 7722 f->valid = 1; 7723 } else { 7724 if (rc != FW_FILTER_WR_FLT_DELETED) { 7725 /* Add or delete failed, display an error */ 7726 log(LOG_ERR, 7727 "filter %u setup failed with error %u\n", 7728 idx, rc); 7729 } 7730 7731 clear_filter(f); 7732 sc->tids.ftids_in_use--; 7733 } 7734 wakeup(&sc->tids.ftid_tab); 7735 mtx_unlock(&sc->tids.ftid_lock); 7736 } 7737 7738 return (0); 7739 } 7740 7741 static int 7742 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 7743 { 7744 int rc; 7745 7746 if (cntxt->cid > M_CTXTQID) 7747 return (EINVAL); 7748 7749 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 7750 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 7751 return (EINVAL); 7752 7753 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 7754 if (rc) 7755 return (rc); 7756 7757 if (sc->flags & FW_OK) { 7758 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 7759 &cntxt->data[0]); 7760 if (rc == 0) 7761 goto done; 7762 } 7763 7764 /* 7765 * Read via firmware failed or wasn't even attempted. Read directly via 7766 * the backdoor. 7767 */ 7768 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 7769 done: 7770 end_synchronized_op(sc, 0); 7771 return (rc); 7772 } 7773 7774 static int 7775 load_fw(struct adapter *sc, struct t4_data *fw) 7776 { 7777 int rc; 7778 uint8_t *fw_data; 7779 7780 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 7781 if (rc) 7782 return (rc); 7783 7784 if (sc->flags & FULL_INIT_DONE) { 7785 rc = EBUSY; 7786 goto done; 7787 } 7788 7789 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 7790 if (fw_data == NULL) { 7791 rc = ENOMEM; 7792 goto done; 7793 } 7794 7795 rc = copyin(fw->data, fw_data, fw->len); 7796 if (rc == 0) 7797 rc = -t4_load_fw(sc, fw_data, fw->len); 7798 7799 free(fw_data, M_CXGBE); 7800 done: 7801 end_synchronized_op(sc, 0); 7802 return (rc); 7803 } 7804 7805 static int 7806 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 7807 { 7808 uint32_t addr, off, remaining, i, n; 7809 uint32_t *buf, *b; 7810 uint32_t mw_base, mw_aperture; 7811 int rc; 7812 uint8_t *dst; 7813 7814 rc = validate_mem_range(sc, mr->addr, mr->len); 7815 if (rc != 0) 7816 return (rc); 7817 7818 memwin_info(sc, win, &mw_base, &mw_aperture); 7819 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 7820 addr = mr->addr; 7821 remaining = mr->len; 7822 dst = (void *)mr->data; 7823 7824 while (remaining) { 7825 off = position_memwin(sc, win, addr); 7826 7827 /* number of bytes that we'll copy in the inner loop */ 7828 n = min(remaining, mw_aperture - off); 7829 for (i = 0; i < n; i += 4) 7830 *b++ = t4_read_reg(sc, mw_base + off + i); 7831 7832 rc = copyout(buf, dst, n); 7833 if (rc != 0) 7834 break; 7835 7836 b = buf; 7837 dst += n; 7838 remaining -= n; 7839 addr += n; 7840 } 7841 7842 free(buf, M_CXGBE); 7843 return (rc); 7844 } 7845 7846 static int 7847 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 7848 { 7849 int rc; 7850 7851 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 7852 return (EINVAL); 7853 7854 if (i2cd->len > sizeof(i2cd->data)) 7855 return (EFBIG); 7856 7857 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 7858 if (rc) 7859 return (rc); 7860 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 7861 i2cd->offset, i2cd->len, &i2cd->data[0]); 7862 end_synchronized_op(sc, 0); 7863 7864 return (rc); 7865 } 7866 7867 static int 7868 in_range(int val, int lo, int hi) 7869 { 7870 7871 return (val < 0 || (val <= hi && val >= lo)); 7872 } 7873 7874 static int 7875 set_sched_class(struct adapter *sc, struct t4_sched_params *p) 7876 { 7877 int fw_subcmd, fw_type, rc; 7878 7879 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc"); 7880 if (rc) 7881 return (rc); 7882 7883 if (!(sc->flags & FULL_INIT_DONE)) { 7884 rc = EAGAIN; 7885 goto done; 7886 } 7887 7888 /* 7889 * Translate the cxgbetool parameters into T4 firmware parameters. (The 7890 * sub-command and type are in common locations.) 7891 */ 7892 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG) 7893 fw_subcmd = FW_SCHED_SC_CONFIG; 7894 else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS) 7895 fw_subcmd = FW_SCHED_SC_PARAMS; 7896 else { 7897 rc = EINVAL; 7898 goto done; 7899 } 7900 if (p->type == SCHED_CLASS_TYPE_PACKET) 7901 fw_type = FW_SCHED_TYPE_PKTSCHED; 7902 else { 7903 rc = EINVAL; 7904 goto done; 7905 } 7906 7907 if (fw_subcmd == FW_SCHED_SC_CONFIG) { 7908 /* Vet our parameters ..*/ 7909 if (p->u.config.minmax < 0) { 7910 rc = EINVAL; 7911 goto done; 7912 } 7913 7914 /* And pass the request to the firmware ...*/ 7915 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1); 7916 goto done; 7917 } 7918 7919 if (fw_subcmd == FW_SCHED_SC_PARAMS) { 7920 int fw_level; 7921 int fw_mode; 7922 int fw_rateunit; 7923 int fw_ratemode; 7924 7925 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL) 7926 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL; 7927 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) 7928 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR; 7929 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) 7930 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL; 7931 else { 7932 rc = EINVAL; 7933 goto done; 7934 } 7935 7936 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS) 7937 fw_mode = FW_SCHED_PARAMS_MODE_CLASS; 7938 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW) 7939 fw_mode = FW_SCHED_PARAMS_MODE_FLOW; 7940 else { 7941 rc = EINVAL; 7942 goto done; 7943 } 7944 7945 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS) 7946 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 7947 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS) 7948 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE; 7949 else { 7950 rc = EINVAL; 7951 goto done; 7952 } 7953 7954 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL) 7955 fw_ratemode = FW_SCHED_PARAMS_RATE_REL; 7956 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS) 7957 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS; 7958 else { 7959 rc = EINVAL; 7960 goto done; 7961 } 7962 7963 /* Vet our parameters ... */ 7964 if (!in_range(p->u.params.channel, 0, 3) || 7965 !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) || 7966 !in_range(p->u.params.minrate, 0, 10000000) || 7967 !in_range(p->u.params.maxrate, 0, 10000000) || 7968 !in_range(p->u.params.weight, 0, 100)) { 7969 rc = ERANGE; 7970 goto done; 7971 } 7972 7973 /* 7974 * Translate any unset parameters into the firmware's 7975 * nomenclature and/or fail the call if the parameters 7976 * are required ... 7977 */ 7978 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 || 7979 p->u.params.channel < 0 || p->u.params.cl < 0) { 7980 rc = EINVAL; 7981 goto done; 7982 } 7983 if (p->u.params.minrate < 0) 7984 p->u.params.minrate = 0; 7985 if (p->u.params.maxrate < 0) { 7986 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7987 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7988 rc = EINVAL; 7989 goto done; 7990 } else 7991 p->u.params.maxrate = 0; 7992 } 7993 if (p->u.params.weight < 0) { 7994 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) { 7995 rc = EINVAL; 7996 goto done; 7997 } else 7998 p->u.params.weight = 0; 7999 } 8000 if (p->u.params.pktsize < 0) { 8001 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 8002 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 8003 rc = EINVAL; 8004 goto done; 8005 } else 8006 p->u.params.pktsize = 0; 8007 } 8008 8009 /* See what the firmware thinks of the request ... */ 8010 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode, 8011 fw_rateunit, fw_ratemode, p->u.params.channel, 8012 p->u.params.cl, p->u.params.minrate, p->u.params.maxrate, 8013 p->u.params.weight, p->u.params.pktsize, 1); 8014 goto done; 8015 } 8016 8017 rc = EINVAL; 8018 done: 8019 end_synchronized_op(sc, 0); 8020 return (rc); 8021 } 8022 8023 static int 8024 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p) 8025 { 8026 struct port_info *pi = NULL; 8027 struct sge_txq *txq; 8028 uint32_t fw_mnem, fw_queue, fw_class; 8029 int i, rc; 8030 8031 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq"); 8032 if (rc) 8033 return (rc); 8034 8035 if (!(sc->flags & FULL_INIT_DONE)) { 8036 rc = EAGAIN; 8037 goto done; 8038 } 8039 8040 if (p->port >= sc->params.nports) { 8041 rc = EINVAL; 8042 goto done; 8043 } 8044 8045 pi = sc->port[p->port]; 8046 if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) { 8047 rc = EINVAL; 8048 goto done; 8049 } 8050 8051 /* 8052 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX 8053 * Scheduling Class in this case). 8054 */ 8055 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 8056 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH)); 8057 fw_class = p->cl < 0 ? 0xffffffff : p->cl; 8058 8059 /* 8060 * If op.queue is non-negative, then we're only changing the scheduling 8061 * on a single specified TX queue. 8062 */ 8063 if (p->queue >= 0) { 8064 txq = &sc->sge.txq[pi->first_txq + p->queue]; 8065 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 8066 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 8067 &fw_class); 8068 goto done; 8069 } 8070 8071 /* 8072 * Change the scheduling on all the TX queues for the 8073 * interface. 8074 */ 8075 for_each_txq(pi, i, txq) { 8076 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 8077 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 8078 &fw_class); 8079 if (rc) 8080 goto done; 8081 } 8082 8083 rc = 0; 8084 done: 8085 end_synchronized_op(sc, 0); 8086 return (rc); 8087 } 8088 8089 int 8090 t4_os_find_pci_capability(struct adapter *sc, int cap) 8091 { 8092 int i; 8093 8094 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 8095 } 8096 8097 int 8098 t4_os_pci_save_state(struct adapter *sc) 8099 { 8100 device_t dev; 8101 struct pci_devinfo *dinfo; 8102 8103 dev = sc->dev; 8104 dinfo = device_get_ivars(dev); 8105 8106 pci_cfg_save(dev, dinfo, 0); 8107 return (0); 8108 } 8109 8110 int 8111 t4_os_pci_restore_state(struct adapter *sc) 8112 { 8113 device_t dev; 8114 struct pci_devinfo *dinfo; 8115 8116 dev = sc->dev; 8117 dinfo = device_get_ivars(dev); 8118 8119 pci_cfg_restore(dev, dinfo); 8120 return (0); 8121 } 8122 8123 void 8124 t4_os_portmod_changed(const struct adapter *sc, int idx) 8125 { 8126 struct port_info *pi = sc->port[idx]; 8127 static const char *mod_str[] = { 8128 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 8129 }; 8130 8131 build_medialist(pi, &pi->media); 8132 #ifdef DEV_NETMAP 8133 build_medialist(pi, &pi->nm_media); 8134 #endif 8135 8136 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 8137 if_printf(pi->ifp, "transceiver unplugged.\n"); 8138 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 8139 if_printf(pi->ifp, "unknown transceiver inserted.\n"); 8140 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 8141 if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 8142 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 8143 if_printf(pi->ifp, "%s transceiver inserted.\n", 8144 mod_str[pi->mod_type]); 8145 } else { 8146 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 8147 pi->mod_type); 8148 } 8149 } 8150 8151 void 8152 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason) 8153 { 8154 struct port_info *pi = sc->port[idx]; 8155 struct ifnet *ifp = pi->ifp; 8156 8157 if (link_stat) { 8158 pi->linkdnrc = -1; 8159 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 8160 if_link_state_change(ifp, LINK_STATE_UP); 8161 } else { 8162 if (reason >= 0) 8163 pi->linkdnrc = reason; 8164 if_link_state_change(ifp, LINK_STATE_DOWN); 8165 } 8166 } 8167 8168 void 8169 t4_iterate(void (*func)(struct adapter *, void *), void *arg) 8170 { 8171 struct adapter *sc; 8172 8173 sx_slock(&t4_list_lock); 8174 SLIST_FOREACH(sc, &t4_list, link) { 8175 /* 8176 * func should not make any assumptions about what state sc is 8177 * in - the only guarantee is that sc->sc_lock is a valid lock. 8178 */ 8179 func(sc, arg); 8180 } 8181 sx_sunlock(&t4_list_lock); 8182 } 8183 8184 static int 8185 t4_open(struct cdev *dev, int flags, int type, struct thread *td) 8186 { 8187 return (0); 8188 } 8189 8190 static int 8191 t4_close(struct cdev *dev, int flags, int type, struct thread *td) 8192 { 8193 return (0); 8194 } 8195 8196 static int 8197 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 8198 struct thread *td) 8199 { 8200 int rc; 8201 struct adapter *sc = dev->si_drv1; 8202 8203 rc = priv_check(td, PRIV_DRIVER); 8204 if (rc != 0) 8205 return (rc); 8206 8207 switch (cmd) { 8208 case CHELSIO_T4_GETREG: { 8209 struct t4_reg *edata = (struct t4_reg *)data; 8210 8211 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8212 return (EFAULT); 8213 8214 if (edata->size == 4) 8215 edata->val = t4_read_reg(sc, edata->addr); 8216 else if (edata->size == 8) 8217 edata->val = t4_read_reg64(sc, edata->addr); 8218 else 8219 return (EINVAL); 8220 8221 break; 8222 } 8223 case CHELSIO_T4_SETREG: { 8224 struct t4_reg *edata = (struct t4_reg *)data; 8225 8226 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8227 return (EFAULT); 8228 8229 if (edata->size == 4) { 8230 if (edata->val & 0xffffffff00000000) 8231 return (EINVAL); 8232 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 8233 } else if (edata->size == 8) 8234 t4_write_reg64(sc, edata->addr, edata->val); 8235 else 8236 return (EINVAL); 8237 break; 8238 } 8239 case CHELSIO_T4_REGDUMP: { 8240 struct t4_regdump *regs = (struct t4_regdump *)data; 8241 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 8242 uint8_t *buf; 8243 8244 if (regs->len < reglen) { 8245 regs->len = reglen; /* hint to the caller */ 8246 return (ENOBUFS); 8247 } 8248 8249 regs->len = reglen; 8250 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 8251 t4_get_regs(sc, regs, buf); 8252 rc = copyout(buf, regs->data, reglen); 8253 free(buf, M_CXGBE); 8254 break; 8255 } 8256 case CHELSIO_T4_GET_FILTER_MODE: 8257 rc = get_filter_mode(sc, (uint32_t *)data); 8258 break; 8259 case CHELSIO_T4_SET_FILTER_MODE: 8260 rc = set_filter_mode(sc, *(uint32_t *)data); 8261 break; 8262 case CHELSIO_T4_GET_FILTER: 8263 rc = get_filter(sc, (struct t4_filter *)data); 8264 break; 8265 case CHELSIO_T4_SET_FILTER: 8266 rc = set_filter(sc, (struct t4_filter *)data); 8267 break; 8268 case CHELSIO_T4_DEL_FILTER: 8269 rc = del_filter(sc, (struct t4_filter *)data); 8270 break; 8271 case CHELSIO_T4_GET_SGE_CONTEXT: 8272 rc = get_sge_context(sc, (struct t4_sge_context *)data); 8273 break; 8274 case CHELSIO_T4_LOAD_FW: 8275 rc = load_fw(sc, (struct t4_data *)data); 8276 break; 8277 case CHELSIO_T4_GET_MEM: 8278 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 8279 break; 8280 case CHELSIO_T4_GET_I2C: 8281 rc = read_i2c(sc, (struct t4_i2c_data *)data); 8282 break; 8283 case CHELSIO_T4_CLEAR_STATS: { 8284 int i; 8285 u_int port_id = *(uint32_t *)data; 8286 struct port_info *pi; 8287 8288 if (port_id >= sc->params.nports) 8289 return (EINVAL); 8290 pi = sc->port[port_id]; 8291 8292 /* MAC stats */ 8293 t4_clr_port_stats(sc, pi->tx_chan); 8294 pi->tx_parse_error = 0; 8295 8296 if (pi->flags & PORT_INIT_DONE) { 8297 struct sge_rxq *rxq; 8298 struct sge_txq *txq; 8299 struct sge_wrq *wrq; 8300 8301 for_each_rxq(pi, i, rxq) { 8302 #if defined(INET) || defined(INET6) 8303 rxq->lro.lro_queued = 0; 8304 rxq->lro.lro_flushed = 0; 8305 #endif 8306 rxq->rxcsum = 0; 8307 rxq->vlan_extraction = 0; 8308 } 8309 8310 for_each_txq(pi, i, txq) { 8311 txq->txcsum = 0; 8312 txq->tso_wrs = 0; 8313 txq->vlan_insertion = 0; 8314 txq->imm_wrs = 0; 8315 txq->sgl_wrs = 0; 8316 txq->txpkt_wrs = 0; 8317 txq->txpkts0_wrs = 0; 8318 txq->txpkts1_wrs = 0; 8319 txq->txpkts0_pkts = 0; 8320 txq->txpkts1_pkts = 0; 8321 mp_ring_reset_stats(txq->r); 8322 } 8323 8324 #ifdef TCP_OFFLOAD 8325 /* nothing to clear for each ofld_rxq */ 8326 8327 for_each_ofld_txq(pi, i, wrq) { 8328 wrq->tx_wrs_direct = 0; 8329 wrq->tx_wrs_copied = 0; 8330 } 8331 #endif 8332 wrq = &sc->sge.ctrlq[pi->port_id]; 8333 wrq->tx_wrs_direct = 0; 8334 wrq->tx_wrs_copied = 0; 8335 } 8336 break; 8337 } 8338 case CHELSIO_T4_SCHED_CLASS: 8339 rc = set_sched_class(sc, (struct t4_sched_params *)data); 8340 break; 8341 case CHELSIO_T4_SCHED_QUEUE: 8342 rc = set_sched_queue(sc, (struct t4_sched_queue *)data); 8343 break; 8344 case CHELSIO_T4_GET_TRACER: 8345 rc = t4_get_tracer(sc, (struct t4_tracer *)data); 8346 break; 8347 case CHELSIO_T4_SET_TRACER: 8348 rc = t4_set_tracer(sc, (struct t4_tracer *)data); 8349 break; 8350 default: 8351 rc = EINVAL; 8352 } 8353 8354 return (rc); 8355 } 8356 8357 #ifdef TCP_OFFLOAD 8358 void 8359 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask, 8360 const unsigned int *pgsz_order) 8361 { 8362 struct port_info *pi = ifp->if_softc; 8363 struct adapter *sc = pi->adapter; 8364 8365 t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask); 8366 t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) | 8367 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) | 8368 V_HPZ3(pgsz_order[3])); 8369 } 8370 8371 static int 8372 toe_capability(struct port_info *pi, int enable) 8373 { 8374 int rc; 8375 struct adapter *sc = pi->adapter; 8376 8377 ASSERT_SYNCHRONIZED_OP(sc); 8378 8379 if (!is_offload(sc)) 8380 return (ENODEV); 8381 8382 if (enable) { 8383 /* 8384 * We need the port's queues around so that we're able to send 8385 * and receive CPLs to/from the TOE even if the ifnet for this 8386 * port has never been UP'd administratively. 8387 */ 8388 if (!(pi->flags & PORT_INIT_DONE)) { 8389 rc = cxgbe_init_synchronized(pi); 8390 if (rc) 8391 return (rc); 8392 } 8393 8394 if (isset(&sc->offload_map, pi->port_id)) 8395 return (0); 8396 8397 if (!uld_active(sc, ULD_TOM)) { 8398 rc = t4_activate_uld(sc, ULD_TOM); 8399 if (rc == EAGAIN) { 8400 log(LOG_WARNING, 8401 "You must kldload t4_tom.ko before trying " 8402 "to enable TOE on a cxgbe interface.\n"); 8403 } 8404 if (rc != 0) 8405 return (rc); 8406 KASSERT(sc->tom_softc != NULL, 8407 ("%s: TOM activated but softc NULL", __func__)); 8408 KASSERT(uld_active(sc, ULD_TOM), 8409 ("%s: TOM activated but flag not set", __func__)); 8410 } 8411 8412 /* Activate iWARP and iSCSI too, if the modules are loaded. */ 8413 if (!uld_active(sc, ULD_IWARP)) 8414 (void) t4_activate_uld(sc, ULD_IWARP); 8415 if (!uld_active(sc, ULD_ISCSI)) 8416 (void) t4_activate_uld(sc, ULD_ISCSI); 8417 8418 setbit(&sc->offload_map, pi->port_id); 8419 } else { 8420 if (!isset(&sc->offload_map, pi->port_id)) 8421 return (0); 8422 8423 KASSERT(uld_active(sc, ULD_TOM), 8424 ("%s: TOM never initialized?", __func__)); 8425 clrbit(&sc->offload_map, pi->port_id); 8426 } 8427 8428 return (0); 8429 } 8430 8431 /* 8432 * Add an upper layer driver to the global list. 8433 */ 8434 int 8435 t4_register_uld(struct uld_info *ui) 8436 { 8437 int rc = 0; 8438 struct uld_info *u; 8439 8440 sx_xlock(&t4_uld_list_lock); 8441 SLIST_FOREACH(u, &t4_uld_list, link) { 8442 if (u->uld_id == ui->uld_id) { 8443 rc = EEXIST; 8444 goto done; 8445 } 8446 } 8447 8448 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 8449 ui->refcount = 0; 8450 done: 8451 sx_xunlock(&t4_uld_list_lock); 8452 return (rc); 8453 } 8454 8455 int 8456 t4_unregister_uld(struct uld_info *ui) 8457 { 8458 int rc = EINVAL; 8459 struct uld_info *u; 8460 8461 sx_xlock(&t4_uld_list_lock); 8462 8463 SLIST_FOREACH(u, &t4_uld_list, link) { 8464 if (u == ui) { 8465 if (ui->refcount > 0) { 8466 rc = EBUSY; 8467 goto done; 8468 } 8469 8470 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 8471 rc = 0; 8472 goto done; 8473 } 8474 } 8475 done: 8476 sx_xunlock(&t4_uld_list_lock); 8477 return (rc); 8478 } 8479 8480 int 8481 t4_activate_uld(struct adapter *sc, int id) 8482 { 8483 int rc; 8484 struct uld_info *ui; 8485 8486 ASSERT_SYNCHRONIZED_OP(sc); 8487 8488 if (id < 0 || id > ULD_MAX) 8489 return (EINVAL); 8490 rc = EAGAIN; /* kldoad the module with this ULD and try again. */ 8491 8492 sx_slock(&t4_uld_list_lock); 8493 8494 SLIST_FOREACH(ui, &t4_uld_list, link) { 8495 if (ui->uld_id == id) { 8496 if (!(sc->flags & FULL_INIT_DONE)) { 8497 rc = adapter_full_init(sc); 8498 if (rc != 0) 8499 break; 8500 } 8501 8502 rc = ui->activate(sc); 8503 if (rc == 0) { 8504 setbit(&sc->active_ulds, id); 8505 ui->refcount++; 8506 } 8507 break; 8508 } 8509 } 8510 8511 sx_sunlock(&t4_uld_list_lock); 8512 8513 return (rc); 8514 } 8515 8516 int 8517 t4_deactivate_uld(struct adapter *sc, int id) 8518 { 8519 int rc; 8520 struct uld_info *ui; 8521 8522 ASSERT_SYNCHRONIZED_OP(sc); 8523 8524 if (id < 0 || id > ULD_MAX) 8525 return (EINVAL); 8526 rc = ENXIO; 8527 8528 sx_slock(&t4_uld_list_lock); 8529 8530 SLIST_FOREACH(ui, &t4_uld_list, link) { 8531 if (ui->uld_id == id) { 8532 rc = ui->deactivate(sc); 8533 if (rc == 0) { 8534 clrbit(&sc->active_ulds, id); 8535 ui->refcount--; 8536 } 8537 break; 8538 } 8539 } 8540 8541 sx_sunlock(&t4_uld_list_lock); 8542 8543 return (rc); 8544 } 8545 8546 int 8547 uld_active(struct adapter *sc, int uld_id) 8548 { 8549 8550 MPASS(uld_id >= 0 && uld_id <= ULD_MAX); 8551 8552 return (isset(&sc->active_ulds, uld_id)); 8553 } 8554 #endif 8555 8556 /* 8557 * Come up with reasonable defaults for some of the tunables, provided they're 8558 * not set by the user (in which case we'll use the values as is). 8559 */ 8560 static void 8561 tweak_tunables(void) 8562 { 8563 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 8564 8565 if (t4_ntxq10g < 1) { 8566 #ifdef RSS 8567 t4_ntxq10g = rss_getnumbuckets(); 8568 #else 8569 t4_ntxq10g = min(nc, NTXQ_10G); 8570 #endif 8571 } 8572 8573 if (t4_ntxq1g < 1) { 8574 #ifdef RSS 8575 /* XXX: way too many for 1GbE? */ 8576 t4_ntxq1g = rss_getnumbuckets(); 8577 #else 8578 t4_ntxq1g = min(nc, NTXQ_1G); 8579 #endif 8580 } 8581 8582 if (t4_nrxq10g < 1) { 8583 #ifdef RSS 8584 t4_nrxq10g = rss_getnumbuckets(); 8585 #else 8586 t4_nrxq10g = min(nc, NRXQ_10G); 8587 #endif 8588 } 8589 8590 if (t4_nrxq1g < 1) { 8591 #ifdef RSS 8592 /* XXX: way too many for 1GbE? */ 8593 t4_nrxq1g = rss_getnumbuckets(); 8594 #else 8595 t4_nrxq1g = min(nc, NRXQ_1G); 8596 #endif 8597 } 8598 8599 #ifdef TCP_OFFLOAD 8600 if (t4_nofldtxq10g < 1) 8601 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 8602 8603 if (t4_nofldtxq1g < 1) 8604 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 8605 8606 if (t4_nofldrxq10g < 1) 8607 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 8608 8609 if (t4_nofldrxq1g < 1) 8610 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 8611 8612 if (t4_toecaps_allowed == -1) 8613 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 8614 #else 8615 if (t4_toecaps_allowed == -1) 8616 t4_toecaps_allowed = 0; 8617 #endif 8618 8619 #ifdef DEV_NETMAP 8620 if (t4_nnmtxq10g < 1) 8621 t4_nnmtxq10g = min(nc, NNMTXQ_10G); 8622 8623 if (t4_nnmtxq1g < 1) 8624 t4_nnmtxq1g = min(nc, NNMTXQ_1G); 8625 8626 if (t4_nnmrxq10g < 1) 8627 t4_nnmrxq10g = min(nc, NNMRXQ_10G); 8628 8629 if (t4_nnmrxq1g < 1) 8630 t4_nnmrxq1g = min(nc, NNMRXQ_1G); 8631 #endif 8632 8633 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 8634 t4_tmr_idx_10g = TMR_IDX_10G; 8635 8636 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 8637 t4_pktc_idx_10g = PKTC_IDX_10G; 8638 8639 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 8640 t4_tmr_idx_1g = TMR_IDX_1G; 8641 8642 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 8643 t4_pktc_idx_1g = PKTC_IDX_1G; 8644 8645 if (t4_qsize_txq < 128) 8646 t4_qsize_txq = 128; 8647 8648 if (t4_qsize_rxq < 128) 8649 t4_qsize_rxq = 128; 8650 while (t4_qsize_rxq & 7) 8651 t4_qsize_rxq++; 8652 8653 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 8654 } 8655 8656 static struct sx mlu; /* mod load unload */ 8657 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload"); 8658 8659 static int 8660 mod_event(module_t mod, int cmd, void *arg) 8661 { 8662 int rc = 0; 8663 static int loaded = 0; 8664 8665 switch (cmd) { 8666 case MOD_LOAD: 8667 sx_xlock(&mlu); 8668 if (loaded++ == 0) { 8669 t4_sge_modload(); 8670 sx_init(&t4_list_lock, "T4/T5 adapters"); 8671 SLIST_INIT(&t4_list); 8672 #ifdef TCP_OFFLOAD 8673 sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); 8674 SLIST_INIT(&t4_uld_list); 8675 #endif 8676 t4_tracer_modload(); 8677 tweak_tunables(); 8678 } 8679 sx_xunlock(&mlu); 8680 break; 8681 8682 case MOD_UNLOAD: 8683 sx_xlock(&mlu); 8684 if (--loaded == 0) { 8685 int tries; 8686 8687 sx_slock(&t4_list_lock); 8688 if (!SLIST_EMPTY(&t4_list)) { 8689 rc = EBUSY; 8690 sx_sunlock(&t4_list_lock); 8691 goto done_unload; 8692 } 8693 #ifdef TCP_OFFLOAD 8694 sx_slock(&t4_uld_list_lock); 8695 if (!SLIST_EMPTY(&t4_uld_list)) { 8696 rc = EBUSY; 8697 sx_sunlock(&t4_uld_list_lock); 8698 sx_sunlock(&t4_list_lock); 8699 goto done_unload; 8700 } 8701 #endif 8702 tries = 0; 8703 while (tries++ < 5 && t4_sge_extfree_refs() != 0) { 8704 uprintf("%ju clusters with custom free routine " 8705 "still is use.\n", t4_sge_extfree_refs()); 8706 pause("t4unload", 2 * hz); 8707 } 8708 #ifdef TCP_OFFLOAD 8709 sx_sunlock(&t4_uld_list_lock); 8710 #endif 8711 sx_sunlock(&t4_list_lock); 8712 8713 if (t4_sge_extfree_refs() == 0) { 8714 t4_tracer_modunload(); 8715 #ifdef TCP_OFFLOAD 8716 sx_destroy(&t4_uld_list_lock); 8717 #endif 8718 sx_destroy(&t4_list_lock); 8719 t4_sge_modunload(); 8720 loaded = 0; 8721 } else { 8722 rc = EBUSY; 8723 loaded++; /* undo earlier decrement */ 8724 } 8725 } 8726 done_unload: 8727 sx_xunlock(&mlu); 8728 break; 8729 } 8730 8731 return (rc); 8732 } 8733 8734 static devclass_t t4_devclass, t5_devclass; 8735 static devclass_t cxgbe_devclass, cxl_devclass; 8736 8737 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 8738 MODULE_VERSION(t4nex, 1); 8739 MODULE_DEPEND(t4nex, firmware, 1, 1, 1); 8740 #ifdef DEV_NETMAP 8741 MODULE_DEPEND(t4nex, netmap, 1, 1, 1); 8742 #endif /* DEV_NETMAP */ 8743 8744 8745 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 8746 MODULE_VERSION(t5nex, 1); 8747 MODULE_DEPEND(t5nex, firmware, 1, 1, 1); 8748 #ifdef DEV_NETMAP 8749 MODULE_DEPEND(t5nex, netmap, 1, 1, 1); 8750 #endif /* DEV_NETMAP */ 8751 8752 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 8753 MODULE_VERSION(cxgbe, 1); 8754 8755 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 8756 MODULE_VERSION(cxl, 1); 8757