1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2011 Chelsio Communications, Inc. 5 * All rights reserved. 6 * Written by: Navdeep Parhar <np@FreeBSD.org> 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include "opt_ddb.h" 34 #include "opt_inet.h" 35 #include "opt_inet6.h" 36 #include "opt_ratelimit.h" 37 #include "opt_rss.h" 38 39 #include <sys/param.h> 40 #include <sys/conf.h> 41 #include <sys/priv.h> 42 #include <sys/kernel.h> 43 #include <sys/bus.h> 44 #include <sys/module.h> 45 #include <sys/malloc.h> 46 #include <sys/queue.h> 47 #include <sys/taskqueue.h> 48 #include <sys/pciio.h> 49 #include <dev/pci/pcireg.h> 50 #include <dev/pci/pcivar.h> 51 #include <dev/pci/pci_private.h> 52 #include <sys/firmware.h> 53 #include <sys/sbuf.h> 54 #include <sys/smp.h> 55 #include <sys/socket.h> 56 #include <sys/sockio.h> 57 #include <sys/sysctl.h> 58 #include <net/ethernet.h> 59 #include <net/if.h> 60 #include <net/if_types.h> 61 #include <net/if_dl.h> 62 #include <net/if_vlan_var.h> 63 #ifdef RSS 64 #include <net/rss_config.h> 65 #endif 66 #include <netinet/in.h> 67 #include <netinet/ip.h> 68 #if defined(__i386__) || defined(__amd64__) 69 #include <machine/md_var.h> 70 #include <machine/cputypes.h> 71 #include <vm/vm.h> 72 #include <vm/pmap.h> 73 #endif 74 #include <crypto/rijndael/rijndael.h> 75 #ifdef DDB 76 #include <ddb/ddb.h> 77 #include <ddb/db_lex.h> 78 #endif 79 80 #include "common/common.h" 81 #include "common/t4_msg.h" 82 #include "common/t4_regs.h" 83 #include "common/t4_regs_values.h" 84 #include "cudbg/cudbg.h" 85 #include "t4_ioctl.h" 86 #include "t4_l2t.h" 87 #include "t4_mp_ring.h" 88 #include "t4_if.h" 89 #include "t4_smt.h" 90 91 /* T4 bus driver interface */ 92 static int t4_probe(device_t); 93 static int t4_attach(device_t); 94 static int t4_detach(device_t); 95 static int t4_ready(device_t); 96 static int t4_read_port_device(device_t, int, device_t *); 97 static device_method_t t4_methods[] = { 98 DEVMETHOD(device_probe, t4_probe), 99 DEVMETHOD(device_attach, t4_attach), 100 DEVMETHOD(device_detach, t4_detach), 101 102 DEVMETHOD(t4_is_main_ready, t4_ready), 103 DEVMETHOD(t4_read_port_device, t4_read_port_device), 104 105 DEVMETHOD_END 106 }; 107 static driver_t t4_driver = { 108 "t4nex", 109 t4_methods, 110 sizeof(struct adapter) 111 }; 112 113 114 /* T4 port (cxgbe) interface */ 115 static int cxgbe_probe(device_t); 116 static int cxgbe_attach(device_t); 117 static int cxgbe_detach(device_t); 118 device_method_t cxgbe_methods[] = { 119 DEVMETHOD(device_probe, cxgbe_probe), 120 DEVMETHOD(device_attach, cxgbe_attach), 121 DEVMETHOD(device_detach, cxgbe_detach), 122 { 0, 0 } 123 }; 124 static driver_t cxgbe_driver = { 125 "cxgbe", 126 cxgbe_methods, 127 sizeof(struct port_info) 128 }; 129 130 /* T4 VI (vcxgbe) interface */ 131 static int vcxgbe_probe(device_t); 132 static int vcxgbe_attach(device_t); 133 static int vcxgbe_detach(device_t); 134 static device_method_t vcxgbe_methods[] = { 135 DEVMETHOD(device_probe, vcxgbe_probe), 136 DEVMETHOD(device_attach, vcxgbe_attach), 137 DEVMETHOD(device_detach, vcxgbe_detach), 138 { 0, 0 } 139 }; 140 static driver_t vcxgbe_driver = { 141 "vcxgbe", 142 vcxgbe_methods, 143 sizeof(struct vi_info) 144 }; 145 146 static d_ioctl_t t4_ioctl; 147 148 static struct cdevsw t4_cdevsw = { 149 .d_version = D_VERSION, 150 .d_ioctl = t4_ioctl, 151 .d_name = "t4nex", 152 }; 153 154 /* T5 bus driver interface */ 155 static int t5_probe(device_t); 156 static device_method_t t5_methods[] = { 157 DEVMETHOD(device_probe, t5_probe), 158 DEVMETHOD(device_attach, t4_attach), 159 DEVMETHOD(device_detach, t4_detach), 160 161 DEVMETHOD(t4_is_main_ready, t4_ready), 162 DEVMETHOD(t4_read_port_device, t4_read_port_device), 163 164 DEVMETHOD_END 165 }; 166 static driver_t t5_driver = { 167 "t5nex", 168 t5_methods, 169 sizeof(struct adapter) 170 }; 171 172 173 /* T5 port (cxl) interface */ 174 static driver_t cxl_driver = { 175 "cxl", 176 cxgbe_methods, 177 sizeof(struct port_info) 178 }; 179 180 /* T5 VI (vcxl) interface */ 181 static driver_t vcxl_driver = { 182 "vcxl", 183 vcxgbe_methods, 184 sizeof(struct vi_info) 185 }; 186 187 /* T6 bus driver interface */ 188 static int t6_probe(device_t); 189 static device_method_t t6_methods[] = { 190 DEVMETHOD(device_probe, t6_probe), 191 DEVMETHOD(device_attach, t4_attach), 192 DEVMETHOD(device_detach, t4_detach), 193 194 DEVMETHOD(t4_is_main_ready, t4_ready), 195 DEVMETHOD(t4_read_port_device, t4_read_port_device), 196 197 DEVMETHOD_END 198 }; 199 static driver_t t6_driver = { 200 "t6nex", 201 t6_methods, 202 sizeof(struct adapter) 203 }; 204 205 206 /* T6 port (cc) interface */ 207 static driver_t cc_driver = { 208 "cc", 209 cxgbe_methods, 210 sizeof(struct port_info) 211 }; 212 213 /* T6 VI (vcc) interface */ 214 static driver_t vcc_driver = { 215 "vcc", 216 vcxgbe_methods, 217 sizeof(struct vi_info) 218 }; 219 220 /* ifnet interface */ 221 static void cxgbe_init(void *); 222 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 223 static int cxgbe_transmit(struct ifnet *, struct mbuf *); 224 static void cxgbe_qflush(struct ifnet *); 225 226 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 227 228 /* 229 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 230 * then ADAPTER_LOCK, then t4_uld_list_lock. 231 */ 232 static struct sx t4_list_lock; 233 SLIST_HEAD(, adapter) t4_list; 234 #ifdef TCP_OFFLOAD 235 static struct sx t4_uld_list_lock; 236 SLIST_HEAD(, uld_info) t4_uld_list; 237 #endif 238 239 /* 240 * Tunables. See tweak_tunables() too. 241 * 242 * Each tunable is set to a default value here if it's known at compile-time. 243 * Otherwise it is set to -n as an indication to tweak_tunables() that it should 244 * provide a reasonable default (upto n) when the driver is loaded. 245 * 246 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 247 * T5 are under hw.cxl. 248 */ 249 250 /* 251 * Number of queues for tx and rx, NIC and offload. 252 */ 253 #define NTXQ 16 254 int t4_ntxq = -NTXQ; 255 TUNABLE_INT("hw.cxgbe.ntxq", &t4_ntxq); 256 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq); /* Old name, undocumented */ 257 258 #define NRXQ 8 259 int t4_nrxq = -NRXQ; 260 TUNABLE_INT("hw.cxgbe.nrxq", &t4_nrxq); 261 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq); /* Old name, undocumented */ 262 263 #define NTXQ_VI 1 264 static int t4_ntxq_vi = -NTXQ_VI; 265 TUNABLE_INT("hw.cxgbe.ntxq_vi", &t4_ntxq_vi); 266 267 #define NRXQ_VI 1 268 static int t4_nrxq_vi = -NRXQ_VI; 269 TUNABLE_INT("hw.cxgbe.nrxq_vi", &t4_nrxq_vi); 270 271 static int t4_rsrv_noflowq = 0; 272 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq); 273 274 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 275 #define NOFLDTXQ 8 276 static int t4_nofldtxq = -NOFLDTXQ; 277 TUNABLE_INT("hw.cxgbe.nofldtxq", &t4_nofldtxq); 278 279 #define NOFLDRXQ 2 280 static int t4_nofldrxq = -NOFLDRXQ; 281 TUNABLE_INT("hw.cxgbe.nofldrxq", &t4_nofldrxq); 282 283 #define NOFLDTXQ_VI 1 284 static int t4_nofldtxq_vi = -NOFLDTXQ_VI; 285 TUNABLE_INT("hw.cxgbe.nofldtxq_vi", &t4_nofldtxq_vi); 286 287 #define NOFLDRXQ_VI 1 288 static int t4_nofldrxq_vi = -NOFLDRXQ_VI; 289 TUNABLE_INT("hw.cxgbe.nofldrxq_vi", &t4_nofldrxq_vi); 290 291 #define TMR_IDX_OFLD 1 292 int t4_tmr_idx_ofld = TMR_IDX_OFLD; 293 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_ofld", &t4_tmr_idx_ofld); 294 295 #define PKTC_IDX_OFLD (-1) 296 int t4_pktc_idx_ofld = PKTC_IDX_OFLD; 297 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_ofld", &t4_pktc_idx_ofld); 298 299 /* 0 means chip/fw default, non-zero number is value in microseconds */ 300 static u_long t4_toe_keepalive_idle = 0; 301 TUNABLE_ULONG("hw.cxgbe.toe.keepalive_idle", &t4_toe_keepalive_idle); 302 303 /* 0 means chip/fw default, non-zero number is value in microseconds */ 304 static u_long t4_toe_keepalive_interval = 0; 305 TUNABLE_ULONG("hw.cxgbe.toe.keepalive_interval", &t4_toe_keepalive_interval); 306 307 /* 0 means chip/fw default, non-zero number is # of keepalives before abort */ 308 static int t4_toe_keepalive_count = 0; 309 TUNABLE_INT("hw.cxgbe.toe.keepalive_count", &t4_toe_keepalive_count); 310 311 /* 0 means chip/fw default, non-zero number is value in microseconds */ 312 static u_long t4_toe_rexmt_min = 0; 313 TUNABLE_ULONG("hw.cxgbe.toe.rexmt_min", &t4_toe_rexmt_min); 314 315 /* 0 means chip/fw default, non-zero number is value in microseconds */ 316 static u_long t4_toe_rexmt_max = 0; 317 TUNABLE_ULONG("hw.cxgbe.toe.rexmt_max", &t4_toe_rexmt_max); 318 319 /* 0 means chip/fw default, non-zero number is # of rexmt before abort */ 320 static int t4_toe_rexmt_count = 0; 321 TUNABLE_INT("hw.cxgbe.toe.rexmt_count", &t4_toe_rexmt_count); 322 323 /* -1 means chip/fw default, other values are raw backoff values to use */ 324 static int t4_toe_rexmt_backoff[16] = { 325 -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 326 }; 327 TUNABLE_INT("hw.cxgbe.toe.rexmt_backoff.0", &t4_toe_rexmt_backoff[0]); 328 TUNABLE_INT("hw.cxgbe.toe.rexmt_backoff.1", &t4_toe_rexmt_backoff[1]); 329 TUNABLE_INT("hw.cxgbe.toe.rexmt_backoff.2", &t4_toe_rexmt_backoff[2]); 330 TUNABLE_INT("hw.cxgbe.toe.rexmt_backoff.3", &t4_toe_rexmt_backoff[3]); 331 TUNABLE_INT("hw.cxgbe.toe.rexmt_backoff.4", &t4_toe_rexmt_backoff[4]); 332 TUNABLE_INT("hw.cxgbe.toe.rexmt_backoff.5", &t4_toe_rexmt_backoff[5]); 333 TUNABLE_INT("hw.cxgbe.toe.rexmt_backoff.6", &t4_toe_rexmt_backoff[6]); 334 TUNABLE_INT("hw.cxgbe.toe.rexmt_backoff.7", &t4_toe_rexmt_backoff[7]); 335 TUNABLE_INT("hw.cxgbe.toe.rexmt_backoff.8", &t4_toe_rexmt_backoff[8]); 336 TUNABLE_INT("hw.cxgbe.toe.rexmt_backoff.9", &t4_toe_rexmt_backoff[9]); 337 TUNABLE_INT("hw.cxgbe.toe.rexmt_backoff.10", &t4_toe_rexmt_backoff[10]); 338 TUNABLE_INT("hw.cxgbe.toe.rexmt_backoff.11", &t4_toe_rexmt_backoff[11]); 339 TUNABLE_INT("hw.cxgbe.toe.rexmt_backoff.12", &t4_toe_rexmt_backoff[12]); 340 TUNABLE_INT("hw.cxgbe.toe.rexmt_backoff.13", &t4_toe_rexmt_backoff[13]); 341 TUNABLE_INT("hw.cxgbe.toe.rexmt_backoff.14", &t4_toe_rexmt_backoff[14]); 342 TUNABLE_INT("hw.cxgbe.toe.rexmt_backoff.15", &t4_toe_rexmt_backoff[15]); 343 #endif 344 345 #ifdef DEV_NETMAP 346 #define NNMTXQ_VI 2 347 static int t4_nnmtxq_vi = -NNMTXQ_VI; 348 TUNABLE_INT("hw.cxgbe.nnmtxq_vi", &t4_nnmtxq_vi); 349 350 #define NNMRXQ_VI 2 351 static int t4_nnmrxq_vi = -NNMRXQ_VI; 352 TUNABLE_INT("hw.cxgbe.nnmrxq_vi", &t4_nnmrxq_vi); 353 #endif 354 355 /* 356 * Holdoff parameters for ports. 357 */ 358 #define TMR_IDX 1 359 int t4_tmr_idx = TMR_IDX; 360 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx", &t4_tmr_idx); 361 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx); /* Old name */ 362 363 #define PKTC_IDX (-1) 364 int t4_pktc_idx = PKTC_IDX; 365 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx", &t4_pktc_idx); 366 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx); /* Old name */ 367 368 /* 369 * Size (# of entries) of each tx and rx queue. 370 */ 371 unsigned int t4_qsize_txq = TX_EQ_QSIZE; 372 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 373 374 unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 375 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 376 377 /* 378 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 379 */ 380 int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 381 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 382 383 /* 384 * Configuration file. All the _CF names here are special. 385 */ 386 #define DEFAULT_CF "default" 387 #define BUILTIN_CF "built-in" 388 #define FLASH_CF "flash" 389 #define UWIRE_CF "uwire" 390 #define FPGA_CF "fpga" 391 static char t4_cfg_file[32] = DEFAULT_CF; 392 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 393 394 /* 395 * PAUSE settings (bit 0, 1, 2 = rx_pause, tx_pause, pause_autoneg respectively). 396 * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them. 397 * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water 398 * mark or when signalled to do so, 0 to never emit PAUSE. 399 * pause_autoneg = 1 means PAUSE will be negotiated if possible and the 400 * negotiated settings will override rx_pause/tx_pause. 401 * Otherwise rx_pause/tx_pause are applied forcibly. 402 */ 403 static int t4_pause_settings = PAUSE_RX | PAUSE_TX | PAUSE_AUTONEG; 404 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings); 405 406 /* 407 * Forward Error Correction settings (bit 0, 1 = RS, BASER respectively). 408 * -1 to run with the firmware default. Same as FEC_AUTO (bit 5) 409 * 0 to disable FEC. 410 */ 411 static int t4_fec = -1; 412 TUNABLE_INT("hw.cxgbe.fec", &t4_fec); 413 414 /* 415 * Link autonegotiation. 416 * -1 to run with the firmware default. 417 * 0 to disable. 418 * 1 to enable. 419 */ 420 static int t4_autoneg = -1; 421 TUNABLE_INT("hw.cxgbe.autoneg", &t4_autoneg); 422 423 /* 424 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 425 * encouraged respectively). 426 */ 427 static unsigned int t4_fw_install = 1; 428 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 429 430 /* 431 * ASIC features that will be used. Disable the ones you don't want so that the 432 * chip resources aren't wasted on features that will not be used. 433 */ 434 static int t4_nbmcaps_allowed = 0; 435 TUNABLE_INT("hw.cxgbe.nbmcaps_allowed", &t4_nbmcaps_allowed); 436 437 static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 438 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 439 440 static int t4_switchcaps_allowed = FW_CAPS_CONFIG_SWITCH_INGRESS | 441 FW_CAPS_CONFIG_SWITCH_EGRESS; 442 TUNABLE_INT("hw.cxgbe.switchcaps_allowed", &t4_switchcaps_allowed); 443 444 #ifdef RATELIMIT 445 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC | 446 FW_CAPS_CONFIG_NIC_HASHFILTER | FW_CAPS_CONFIG_NIC_ETHOFLD; 447 #else 448 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC | 449 FW_CAPS_CONFIG_NIC_HASHFILTER; 450 #endif 451 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 452 453 static int t4_toecaps_allowed = -1; 454 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 455 456 static int t4_rdmacaps_allowed = -1; 457 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 458 459 static int t4_cryptocaps_allowed = -1; 460 TUNABLE_INT("hw.cxgbe.cryptocaps_allowed", &t4_cryptocaps_allowed); 461 462 static int t4_iscsicaps_allowed = -1; 463 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 464 465 static int t4_fcoecaps_allowed = 0; 466 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 467 468 static int t5_write_combine = 0; 469 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 470 471 static int t4_num_vis = 1; 472 TUNABLE_INT("hw.cxgbe.num_vis", &t4_num_vis); 473 /* 474 * PCIe Relaxed Ordering. 475 * -1: driver should figure out a good value. 476 * 0: disable RO. 477 * 1: enable RO. 478 * 2: leave RO alone. 479 */ 480 static int pcie_relaxed_ordering = -1; 481 TUNABLE_INT("hw.cxgbe.pcie_relaxed_ordering", &pcie_relaxed_ordering); 482 483 static int t4_panic_on_fatal_err = 0; 484 TUNABLE_INT("hw.cxgbe.panic_on_fatal_err", &t4_panic_on_fatal_err); 485 486 #ifdef TCP_OFFLOAD 487 /* 488 * TOE tunables. 489 */ 490 static int t4_cop_managed_offloading = 0; 491 TUNABLE_INT("hw.cxgbe.cop_managed_offloading", &t4_cop_managed_offloading); 492 #endif 493 494 /* Functions used by VIs to obtain unique MAC addresses for each VI. */ 495 static int vi_mac_funcs[] = { 496 FW_VI_FUNC_ETH, 497 FW_VI_FUNC_OFLD, 498 FW_VI_FUNC_IWARP, 499 FW_VI_FUNC_OPENISCSI, 500 FW_VI_FUNC_OPENFCOE, 501 FW_VI_FUNC_FOISCSI, 502 FW_VI_FUNC_FOFCOE, 503 }; 504 505 struct intrs_and_queues { 506 uint16_t intr_type; /* INTx, MSI, or MSI-X */ 507 uint16_t num_vis; /* number of VIs for each port */ 508 uint16_t nirq; /* Total # of vectors */ 509 uint16_t ntxq; /* # of NIC txq's for each port */ 510 uint16_t nrxq; /* # of NIC rxq's for each port */ 511 uint16_t nofldtxq; /* # of TOE/ETHOFLD txq's for each port */ 512 uint16_t nofldrxq; /* # of TOE rxq's for each port */ 513 514 /* The vcxgbe/vcxl interfaces use these and not the ones above. */ 515 uint16_t ntxq_vi; /* # of NIC txq's */ 516 uint16_t nrxq_vi; /* # of NIC rxq's */ 517 uint16_t nofldtxq_vi; /* # of TOE txq's */ 518 uint16_t nofldrxq_vi; /* # of TOE rxq's */ 519 uint16_t nnmtxq_vi; /* # of netmap txq's */ 520 uint16_t nnmrxq_vi; /* # of netmap rxq's */ 521 }; 522 523 static void setup_memwin(struct adapter *); 524 static void position_memwin(struct adapter *, int, uint32_t); 525 static int validate_mem_range(struct adapter *, uint32_t, int); 526 static int fwmtype_to_hwmtype(int); 527 static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 528 uint32_t *); 529 static int fixup_devlog_params(struct adapter *); 530 static int cfg_itype_and_nqueues(struct adapter *, struct intrs_and_queues *); 531 static int prep_firmware(struct adapter *); 532 static int partition_resources(struct adapter *, const struct firmware *, 533 const char *); 534 static int get_params__pre_init(struct adapter *); 535 static int get_params__post_init(struct adapter *); 536 static int set_params__post_init(struct adapter *); 537 static void t4_set_desc(struct adapter *); 538 static bool fixed_ifmedia(struct port_info *); 539 static void build_medialist(struct port_info *); 540 static void init_link_config(struct port_info *); 541 static int fixup_link_config(struct port_info *); 542 static int apply_link_config(struct port_info *); 543 static int cxgbe_init_synchronized(struct vi_info *); 544 static int cxgbe_uninit_synchronized(struct vi_info *); 545 static void quiesce_txq(struct adapter *, struct sge_txq *); 546 static void quiesce_wrq(struct adapter *, struct sge_wrq *); 547 static void quiesce_iq(struct adapter *, struct sge_iq *); 548 static void quiesce_fl(struct adapter *, struct sge_fl *); 549 static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 550 driver_intr_t *, void *, char *); 551 static int t4_free_irq(struct adapter *, struct irq *); 552 static void get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 553 static void vi_refresh_stats(struct adapter *, struct vi_info *); 554 static void cxgbe_refresh_stats(struct adapter *, struct port_info *); 555 static void cxgbe_tick(void *); 556 static void cxgbe_sysctls(struct port_info *); 557 static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 558 static int sysctl_bitfield_8b(SYSCTL_HANDLER_ARGS); 559 static int sysctl_bitfield_16b(SYSCTL_HANDLER_ARGS); 560 static int sysctl_btphy(SYSCTL_HANDLER_ARGS); 561 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); 562 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 563 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 564 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 565 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 566 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS); 567 static int sysctl_fec(SYSCTL_HANDLER_ARGS); 568 static int sysctl_autoneg(SYSCTL_HANDLER_ARGS); 569 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 570 static int sysctl_temperature(SYSCTL_HANDLER_ARGS); 571 static int sysctl_loadavg(SYSCTL_HANDLER_ARGS); 572 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 573 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 574 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 575 static int sysctl_cim_la_t6(SYSCTL_HANDLER_ARGS); 576 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); 577 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); 578 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 579 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 580 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 581 static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 582 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 583 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 584 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 585 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); 586 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 587 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); 588 static int sysctl_mps_tcam_t6(SYSCTL_HANDLER_ARGS); 589 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 590 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 591 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 592 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 593 static int sysctl_tids(SYSCTL_HANDLER_ARGS); 594 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 595 static int sysctl_tp_la_mask(SYSCTL_HANDLER_ARGS); 596 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 597 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 598 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 599 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 600 static int sysctl_cpus(SYSCTL_HANDLER_ARGS); 601 #ifdef TCP_OFFLOAD 602 static int sysctl_tls_rx_ports(SYSCTL_HANDLER_ARGS); 603 static int sysctl_tp_tick(SYSCTL_HANDLER_ARGS); 604 static int sysctl_tp_dack_timer(SYSCTL_HANDLER_ARGS); 605 static int sysctl_tp_timer(SYSCTL_HANDLER_ARGS); 606 static int sysctl_tp_shift_cnt(SYSCTL_HANDLER_ARGS); 607 static int sysctl_tp_backoff(SYSCTL_HANDLER_ARGS); 608 static int sysctl_holdoff_tmr_idx_ofld(SYSCTL_HANDLER_ARGS); 609 static int sysctl_holdoff_pktc_idx_ofld(SYSCTL_HANDLER_ARGS); 610 #endif 611 static int get_sge_context(struct adapter *, struct t4_sge_context *); 612 static int load_fw(struct adapter *, struct t4_data *); 613 static int load_cfg(struct adapter *, struct t4_data *); 614 static int load_boot(struct adapter *, struct t4_bootrom *); 615 static int load_bootcfg(struct adapter *, struct t4_data *); 616 static int cudbg_dump(struct adapter *, struct t4_cudbg_dump *); 617 static void free_offload_policy(struct t4_offload_policy *); 618 static int set_offload_policy(struct adapter *, struct t4_offload_policy *); 619 static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 620 static int read_i2c(struct adapter *, struct t4_i2c_data *); 621 #ifdef TCP_OFFLOAD 622 static int toe_capability(struct vi_info *, int); 623 #endif 624 static int mod_event(module_t, int, void *); 625 static int notify_siblings(device_t, int); 626 627 struct { 628 uint16_t device; 629 char *desc; 630 } t4_pciids[] = { 631 {0xa000, "Chelsio Terminator 4 FPGA"}, 632 {0x4400, "Chelsio T440-dbg"}, 633 {0x4401, "Chelsio T420-CR"}, 634 {0x4402, "Chelsio T422-CR"}, 635 {0x4403, "Chelsio T440-CR"}, 636 {0x4404, "Chelsio T420-BCH"}, 637 {0x4405, "Chelsio T440-BCH"}, 638 {0x4406, "Chelsio T440-CH"}, 639 {0x4407, "Chelsio T420-SO"}, 640 {0x4408, "Chelsio T420-CX"}, 641 {0x4409, "Chelsio T420-BT"}, 642 {0x440a, "Chelsio T404-BT"}, 643 {0x440e, "Chelsio T440-LP-CR"}, 644 }, t5_pciids[] = { 645 {0xb000, "Chelsio Terminator 5 FPGA"}, 646 {0x5400, "Chelsio T580-dbg"}, 647 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ 648 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ 649 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ 650 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ 651 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ 652 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ 653 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ 654 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ 655 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 656 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ 657 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ 658 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ 659 {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */ 660 {0x5418, "Chelsio T540-BT"}, /* 4 x 10GBaseT */ 661 {0x5419, "Chelsio T540-LP-BT"}, /* 4 x 10GBaseT */ 662 {0x541a, "Chelsio T540-SO-BT"}, /* 4 x 10GBaseT, nomem */ 663 {0x541b, "Chelsio T540-SO-CR"}, /* 4 x 10G, nomem */ 664 }, t6_pciids[] = { 665 {0xc006, "Chelsio Terminator 6 FPGA"}, /* T6 PE10K6 FPGA (PF0) */ 666 {0x6400, "Chelsio T6-DBG-25"}, /* 2 x 10/25G, debug */ 667 {0x6401, "Chelsio T6225-CR"}, /* 2 x 10/25G */ 668 {0x6402, "Chelsio T6225-SO-CR"}, /* 2 x 10/25G, nomem */ 669 {0x6403, "Chelsio T6425-CR"}, /* 4 x 10/25G */ 670 {0x6404, "Chelsio T6425-SO-CR"}, /* 4 x 10/25G, nomem */ 671 {0x6405, "Chelsio T6225-OCP-SO"}, /* 2 x 10/25G, nomem */ 672 {0x6406, "Chelsio T62100-OCP-SO"}, /* 2 x 40/50/100G, nomem */ 673 {0x6407, "Chelsio T62100-LP-CR"}, /* 2 x 40/50/100G */ 674 {0x6408, "Chelsio T62100-SO-CR"}, /* 2 x 40/50/100G, nomem */ 675 {0x6409, "Chelsio T6210-BT"}, /* 2 x 10GBASE-T */ 676 {0x640d, "Chelsio T62100-CR"}, /* 2 x 40/50/100G */ 677 {0x6410, "Chelsio T6-DBG-100"}, /* 2 x 40/50/100G, debug */ 678 {0x6411, "Chelsio T6225-LL-CR"}, /* 2 x 10/25G */ 679 {0x6414, "Chelsio T61100-OCP-SO"}, /* 1 x 40/50/100G, nomem */ 680 {0x6415, "Chelsio T6201-BT"}, /* 2 x 1000BASE-T */ 681 682 /* Custom */ 683 {0x6480, "Custom T6225-CR"}, 684 {0x6481, "Custom T62100-CR"}, 685 {0x6482, "Custom T6225-CR"}, 686 {0x6483, "Custom T62100-CR"}, 687 {0x6484, "Custom T64100-CR"}, 688 {0x6485, "Custom T6240-SO"}, 689 {0x6486, "Custom T6225-SO-CR"}, 690 {0x6487, "Custom T6225-CR"}, 691 }; 692 693 #ifdef TCP_OFFLOAD 694 /* 695 * service_iq_fl() has an iq and needs the fl. Offset of fl from the iq should 696 * be exactly the same for both rxq and ofld_rxq. 697 */ 698 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 699 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 700 #endif 701 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); 702 703 static int 704 t4_probe(device_t dev) 705 { 706 int i; 707 uint16_t v = pci_get_vendor(dev); 708 uint16_t d = pci_get_device(dev); 709 uint8_t f = pci_get_function(dev); 710 711 if (v != PCI_VENDOR_ID_CHELSIO) 712 return (ENXIO); 713 714 /* Attach only to PF0 of the FPGA */ 715 if (d == 0xa000 && f != 0) 716 return (ENXIO); 717 718 for (i = 0; i < nitems(t4_pciids); i++) { 719 if (d == t4_pciids[i].device) { 720 device_set_desc(dev, t4_pciids[i].desc); 721 return (BUS_PROBE_DEFAULT); 722 } 723 } 724 725 return (ENXIO); 726 } 727 728 static int 729 t5_probe(device_t dev) 730 { 731 int i; 732 uint16_t v = pci_get_vendor(dev); 733 uint16_t d = pci_get_device(dev); 734 uint8_t f = pci_get_function(dev); 735 736 if (v != PCI_VENDOR_ID_CHELSIO) 737 return (ENXIO); 738 739 /* Attach only to PF0 of the FPGA */ 740 if (d == 0xb000 && f != 0) 741 return (ENXIO); 742 743 for (i = 0; i < nitems(t5_pciids); i++) { 744 if (d == t5_pciids[i].device) { 745 device_set_desc(dev, t5_pciids[i].desc); 746 return (BUS_PROBE_DEFAULT); 747 } 748 } 749 750 return (ENXIO); 751 } 752 753 static int 754 t6_probe(device_t dev) 755 { 756 int i; 757 uint16_t v = pci_get_vendor(dev); 758 uint16_t d = pci_get_device(dev); 759 760 if (v != PCI_VENDOR_ID_CHELSIO) 761 return (ENXIO); 762 763 for (i = 0; i < nitems(t6_pciids); i++) { 764 if (d == t6_pciids[i].device) { 765 device_set_desc(dev, t6_pciids[i].desc); 766 return (BUS_PROBE_DEFAULT); 767 } 768 } 769 770 return (ENXIO); 771 } 772 773 static void 774 t5_attribute_workaround(device_t dev) 775 { 776 device_t root_port; 777 uint32_t v; 778 779 /* 780 * The T5 chips do not properly echo the No Snoop and Relaxed 781 * Ordering attributes when replying to a TLP from a Root 782 * Port. As a workaround, find the parent Root Port and 783 * disable No Snoop and Relaxed Ordering. Note that this 784 * affects all devices under this root port. 785 */ 786 root_port = pci_find_pcie_root_port(dev); 787 if (root_port == NULL) { 788 device_printf(dev, "Unable to find parent root port\n"); 789 return; 790 } 791 792 v = pcie_adjust_config(root_port, PCIER_DEVICE_CTL, 793 PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE, 0, 2); 794 if ((v & (PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE)) != 795 0) 796 device_printf(dev, "Disabled No Snoop/Relaxed Ordering on %s\n", 797 device_get_nameunit(root_port)); 798 } 799 800 static const struct devnames devnames[] = { 801 { 802 .nexus_name = "t4nex", 803 .ifnet_name = "cxgbe", 804 .vi_ifnet_name = "vcxgbe", 805 .pf03_drv_name = "t4iov", 806 .vf_nexus_name = "t4vf", 807 .vf_ifnet_name = "cxgbev" 808 }, { 809 .nexus_name = "t5nex", 810 .ifnet_name = "cxl", 811 .vi_ifnet_name = "vcxl", 812 .pf03_drv_name = "t5iov", 813 .vf_nexus_name = "t5vf", 814 .vf_ifnet_name = "cxlv" 815 }, { 816 .nexus_name = "t6nex", 817 .ifnet_name = "cc", 818 .vi_ifnet_name = "vcc", 819 .pf03_drv_name = "t6iov", 820 .vf_nexus_name = "t6vf", 821 .vf_ifnet_name = "ccv" 822 } 823 }; 824 825 void 826 t4_init_devnames(struct adapter *sc) 827 { 828 int id; 829 830 id = chip_id(sc); 831 if (id >= CHELSIO_T4 && id - CHELSIO_T4 < nitems(devnames)) 832 sc->names = &devnames[id - CHELSIO_T4]; 833 else { 834 device_printf(sc->dev, "chip id %d is not supported.\n", id); 835 sc->names = NULL; 836 } 837 } 838 839 static int 840 t4_attach(device_t dev) 841 { 842 struct adapter *sc; 843 int rc = 0, i, j, rqidx, tqidx, nports; 844 struct make_dev_args mda; 845 struct intrs_and_queues iaq; 846 struct sge *s; 847 uint32_t *buf; 848 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 849 int ofld_tqidx; 850 #endif 851 #ifdef TCP_OFFLOAD 852 int ofld_rqidx; 853 #endif 854 #ifdef DEV_NETMAP 855 int nm_rqidx, nm_tqidx; 856 #endif 857 int num_vis; 858 859 sc = device_get_softc(dev); 860 sc->dev = dev; 861 TUNABLE_INT_FETCH("hw.cxgbe.dflags", &sc->debug_flags); 862 863 if ((pci_get_device(dev) & 0xff00) == 0x5400) 864 t5_attribute_workaround(dev); 865 pci_enable_busmaster(dev); 866 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 867 uint32_t v; 868 869 pci_set_max_read_req(dev, 4096); 870 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 871 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5); 872 if (pcie_relaxed_ordering == 0 && 873 (v & PCIEM_CTL_RELAXED_ORD_ENABLE) != 0) { 874 v &= ~PCIEM_CTL_RELAXED_ORD_ENABLE; 875 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 876 } else if (pcie_relaxed_ordering == 1 && 877 (v & PCIEM_CTL_RELAXED_ORD_ENABLE) == 0) { 878 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 879 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 880 } 881 } 882 883 sc->sge_gts_reg = MYPF_REG(A_SGE_PF_GTS); 884 sc->sge_kdoorbell_reg = MYPF_REG(A_SGE_PF_KDOORBELL); 885 sc->traceq = -1; 886 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); 887 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", 888 device_get_nameunit(dev)); 889 890 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 891 device_get_nameunit(dev)); 892 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 893 t4_add_adapter(sc); 894 895 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 896 TAILQ_INIT(&sc->sfl); 897 callout_init_mtx(&sc->sfl_callout, &sc->sfl_lock, 0); 898 899 mtx_init(&sc->reg_lock, "indirect register access", 0, MTX_DEF); 900 901 sc->policy = NULL; 902 rw_init(&sc->policy_lock, "connection offload policy"); 903 904 rc = t4_map_bars_0_and_4(sc); 905 if (rc != 0) 906 goto done; /* error message displayed already */ 907 908 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 909 910 /* Prepare the adapter for operation. */ 911 buf = malloc(PAGE_SIZE, M_CXGBE, M_ZERO | M_WAITOK); 912 rc = -t4_prep_adapter(sc, buf); 913 free(buf, M_CXGBE); 914 if (rc != 0) { 915 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 916 goto done; 917 } 918 919 /* 920 * This is the real PF# to which we're attaching. Works from within PCI 921 * passthrough environments too, where pci_get_function() could return a 922 * different PF# depending on the passthrough configuration. We need to 923 * use the real PF# in all our communication with the firmware. 924 */ 925 j = t4_read_reg(sc, A_PL_WHOAMI); 926 sc->pf = chip_id(sc) <= CHELSIO_T5 ? G_SOURCEPF(j) : G_T6_SOURCEPF(j); 927 sc->mbox = sc->pf; 928 929 t4_init_devnames(sc); 930 if (sc->names == NULL) { 931 rc = ENOTSUP; 932 goto done; /* error message displayed already */ 933 } 934 935 /* 936 * Do this really early, with the memory windows set up even before the 937 * character device. The userland tool's register i/o and mem read 938 * will work even in "recovery mode". 939 */ 940 setup_memwin(sc); 941 if (t4_init_devlog_params(sc, 0) == 0) 942 fixup_devlog_params(sc); 943 make_dev_args_init(&mda); 944 mda.mda_devsw = &t4_cdevsw; 945 mda.mda_uid = UID_ROOT; 946 mda.mda_gid = GID_WHEEL; 947 mda.mda_mode = 0600; 948 mda.mda_si_drv1 = sc; 949 rc = make_dev_s(&mda, &sc->cdev, "%s", device_get_nameunit(dev)); 950 if (rc != 0) 951 device_printf(dev, "failed to create nexus char device: %d.\n", 952 rc); 953 954 /* Go no further if recovery mode has been requested. */ 955 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 956 device_printf(dev, "recovery mode.\n"); 957 goto done; 958 } 959 960 #if defined(__i386__) 961 if ((cpu_feature & CPUID_CX8) == 0) { 962 device_printf(dev, "64 bit atomics not available.\n"); 963 rc = ENOTSUP; 964 goto done; 965 } 966 #endif 967 968 /* Prepare the firmware for operation */ 969 rc = prep_firmware(sc); 970 if (rc != 0) 971 goto done; /* error message displayed already */ 972 973 rc = get_params__post_init(sc); 974 if (rc != 0) 975 goto done; /* error message displayed already */ 976 977 rc = set_params__post_init(sc); 978 if (rc != 0) 979 goto done; /* error message displayed already */ 980 981 rc = t4_map_bar_2(sc); 982 if (rc != 0) 983 goto done; /* error message displayed already */ 984 985 rc = t4_create_dma_tag(sc); 986 if (rc != 0) 987 goto done; /* error message displayed already */ 988 989 /* 990 * First pass over all the ports - allocate VIs and initialize some 991 * basic parameters like mac address, port type, etc. 992 */ 993 for_each_port(sc, i) { 994 struct port_info *pi; 995 996 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 997 sc->port[i] = pi; 998 999 /* These must be set before t4_port_init */ 1000 pi->adapter = sc; 1001 pi->port_id = i; 1002 /* 1003 * XXX: vi[0] is special so we can't delay this allocation until 1004 * pi->nvi's final value is known. 1005 */ 1006 pi->vi = malloc(sizeof(struct vi_info) * t4_num_vis, M_CXGBE, 1007 M_ZERO | M_WAITOK); 1008 1009 /* 1010 * Allocate the "main" VI and initialize parameters 1011 * like mac addr. 1012 */ 1013 rc = -t4_port_init(sc, sc->mbox, sc->pf, 0, i); 1014 if (rc != 0) { 1015 device_printf(dev, "unable to initialize port %d: %d\n", 1016 i, rc); 1017 free(pi->vi, M_CXGBE); 1018 free(pi, M_CXGBE); 1019 sc->port[i] = NULL; 1020 goto done; 1021 } 1022 1023 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 1024 device_get_nameunit(dev), i); 1025 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 1026 sc->chan_map[pi->tx_chan] = i; 1027 1028 /* All VIs on this port share this media. */ 1029 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 1030 cxgbe_media_status); 1031 1032 PORT_LOCK(pi); 1033 init_link_config(pi); 1034 fixup_link_config(pi); 1035 build_medialist(pi); 1036 if (fixed_ifmedia(pi)) 1037 pi->flags |= FIXED_IFMEDIA; 1038 PORT_UNLOCK(pi); 1039 1040 pi->dev = device_add_child(dev, sc->names->ifnet_name, -1); 1041 if (pi->dev == NULL) { 1042 device_printf(dev, 1043 "failed to add device for port %d.\n", i); 1044 rc = ENXIO; 1045 goto done; 1046 } 1047 pi->vi[0].dev = pi->dev; 1048 device_set_softc(pi->dev, pi); 1049 } 1050 1051 /* 1052 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 1053 */ 1054 nports = sc->params.nports; 1055 rc = cfg_itype_and_nqueues(sc, &iaq); 1056 if (rc != 0) 1057 goto done; /* error message displayed already */ 1058 1059 num_vis = iaq.num_vis; 1060 sc->intr_type = iaq.intr_type; 1061 sc->intr_count = iaq.nirq; 1062 1063 s = &sc->sge; 1064 s->nrxq = nports * iaq.nrxq; 1065 s->ntxq = nports * iaq.ntxq; 1066 if (num_vis > 1) { 1067 s->nrxq += nports * (num_vis - 1) * iaq.nrxq_vi; 1068 s->ntxq += nports * (num_vis - 1) * iaq.ntxq_vi; 1069 } 1070 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 1071 s->neq += nports; /* ctrl queues: 1 per port */ 1072 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 1073 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 1074 if (is_offload(sc) || is_ethoffload(sc)) { 1075 s->nofldtxq = nports * iaq.nofldtxq; 1076 if (num_vis > 1) 1077 s->nofldtxq += nports * (num_vis - 1) * iaq.nofldtxq_vi; 1078 s->neq += s->nofldtxq; 1079 1080 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 1081 M_CXGBE, M_ZERO | M_WAITOK); 1082 } 1083 #endif 1084 #ifdef TCP_OFFLOAD 1085 if (is_offload(sc)) { 1086 s->nofldrxq = nports * iaq.nofldrxq; 1087 if (num_vis > 1) 1088 s->nofldrxq += nports * (num_vis - 1) * iaq.nofldrxq_vi; 1089 s->neq += s->nofldrxq; /* free list */ 1090 s->niq += s->nofldrxq; 1091 1092 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 1093 M_CXGBE, M_ZERO | M_WAITOK); 1094 } 1095 #endif 1096 #ifdef DEV_NETMAP 1097 if (num_vis > 1) { 1098 s->nnmrxq = nports * (num_vis - 1) * iaq.nnmrxq_vi; 1099 s->nnmtxq = nports * (num_vis - 1) * iaq.nnmtxq_vi; 1100 } 1101 s->neq += s->nnmtxq + s->nnmrxq; 1102 s->niq += s->nnmrxq; 1103 1104 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq), 1105 M_CXGBE, M_ZERO | M_WAITOK); 1106 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq), 1107 M_CXGBE, M_ZERO | M_WAITOK); 1108 #endif 1109 1110 s->ctrlq = malloc(nports * sizeof(struct sge_wrq), M_CXGBE, 1111 M_ZERO | M_WAITOK); 1112 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 1113 M_ZERO | M_WAITOK); 1114 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 1115 M_ZERO | M_WAITOK); 1116 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 1117 M_ZERO | M_WAITOK); 1118 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 1119 M_ZERO | M_WAITOK); 1120 1121 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 1122 M_ZERO | M_WAITOK); 1123 1124 t4_init_l2t(sc, M_WAITOK); 1125 t4_init_smt(sc, M_WAITOK); 1126 t4_init_tx_sched(sc); 1127 #ifdef RATELIMIT 1128 t4_init_etid_table(sc); 1129 #endif 1130 1131 /* 1132 * Second pass over the ports. This time we know the number of rx and 1133 * tx queues that each port should get. 1134 */ 1135 rqidx = tqidx = 0; 1136 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 1137 ofld_tqidx = 0; 1138 #endif 1139 #ifdef TCP_OFFLOAD 1140 ofld_rqidx = 0; 1141 #endif 1142 #ifdef DEV_NETMAP 1143 nm_rqidx = nm_tqidx = 0; 1144 #endif 1145 for_each_port(sc, i) { 1146 struct port_info *pi = sc->port[i]; 1147 struct vi_info *vi; 1148 1149 if (pi == NULL) 1150 continue; 1151 1152 pi->nvi = num_vis; 1153 for_each_vi(pi, j, vi) { 1154 vi->pi = pi; 1155 vi->qsize_rxq = t4_qsize_rxq; 1156 vi->qsize_txq = t4_qsize_txq; 1157 1158 vi->first_rxq = rqidx; 1159 vi->first_txq = tqidx; 1160 vi->tmr_idx = t4_tmr_idx; 1161 vi->pktc_idx = t4_pktc_idx; 1162 vi->nrxq = j == 0 ? iaq.nrxq : iaq.nrxq_vi; 1163 vi->ntxq = j == 0 ? iaq.ntxq : iaq.ntxq_vi; 1164 1165 rqidx += vi->nrxq; 1166 tqidx += vi->ntxq; 1167 1168 if (j == 0 && vi->ntxq > 1) 1169 vi->rsrv_noflowq = t4_rsrv_noflowq ? 1 : 0; 1170 else 1171 vi->rsrv_noflowq = 0; 1172 1173 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 1174 vi->first_ofld_txq = ofld_tqidx; 1175 vi->nofldtxq = j == 0 ? iaq.nofldtxq : iaq.nofldtxq_vi; 1176 ofld_tqidx += vi->nofldtxq; 1177 #endif 1178 #ifdef TCP_OFFLOAD 1179 vi->ofld_tmr_idx = t4_tmr_idx_ofld; 1180 vi->ofld_pktc_idx = t4_pktc_idx_ofld; 1181 vi->first_ofld_rxq = ofld_rqidx; 1182 vi->nofldrxq = j == 0 ? iaq.nofldrxq : iaq.nofldrxq_vi; 1183 1184 ofld_rqidx += vi->nofldrxq; 1185 #endif 1186 #ifdef DEV_NETMAP 1187 if (j > 0) { 1188 vi->first_nm_rxq = nm_rqidx; 1189 vi->first_nm_txq = nm_tqidx; 1190 vi->nnmrxq = iaq.nnmrxq_vi; 1191 vi->nnmtxq = iaq.nnmtxq_vi; 1192 nm_rqidx += vi->nnmrxq; 1193 nm_tqidx += vi->nnmtxq; 1194 } 1195 #endif 1196 } 1197 } 1198 1199 rc = t4_setup_intr_handlers(sc); 1200 if (rc != 0) { 1201 device_printf(dev, 1202 "failed to setup interrupt handlers: %d\n", rc); 1203 goto done; 1204 } 1205 1206 rc = bus_generic_probe(dev); 1207 if (rc != 0) { 1208 device_printf(dev, "failed to probe child drivers: %d\n", rc); 1209 goto done; 1210 } 1211 1212 /* 1213 * Ensure thread-safe mailbox access (in debug builds). 1214 * 1215 * So far this was the only thread accessing the mailbox but various 1216 * ifnets and sysctls are about to be created and their handlers/ioctls 1217 * will access the mailbox from different threads. 1218 */ 1219 sc->flags |= CHK_MBOX_ACCESS; 1220 1221 rc = bus_generic_attach(dev); 1222 if (rc != 0) { 1223 device_printf(dev, 1224 "failed to attach all child ports: %d\n", rc); 1225 goto done; 1226 } 1227 1228 device_printf(dev, 1229 "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 1230 sc->params.pci.speed, sc->params.pci.width, sc->params.nports, 1231 sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" : 1232 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 1233 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 1234 1235 t4_set_desc(sc); 1236 1237 notify_siblings(dev, 0); 1238 1239 done: 1240 if (rc != 0 && sc->cdev) { 1241 /* cdev was created and so cxgbetool works; recover that way. */ 1242 device_printf(dev, 1243 "error during attach, adapter is now in recovery mode.\n"); 1244 rc = 0; 1245 } 1246 1247 if (rc != 0) 1248 t4_detach_common(dev); 1249 else 1250 t4_sysctls(sc); 1251 1252 return (rc); 1253 } 1254 1255 static int 1256 t4_ready(device_t dev) 1257 { 1258 struct adapter *sc; 1259 1260 sc = device_get_softc(dev); 1261 if (sc->flags & FW_OK) 1262 return (0); 1263 return (ENXIO); 1264 } 1265 1266 static int 1267 t4_read_port_device(device_t dev, int port, device_t *child) 1268 { 1269 struct adapter *sc; 1270 struct port_info *pi; 1271 1272 sc = device_get_softc(dev); 1273 if (port < 0 || port >= MAX_NPORTS) 1274 return (EINVAL); 1275 pi = sc->port[port]; 1276 if (pi == NULL || pi->dev == NULL) 1277 return (ENXIO); 1278 *child = pi->dev; 1279 return (0); 1280 } 1281 1282 static int 1283 notify_siblings(device_t dev, int detaching) 1284 { 1285 device_t sibling; 1286 int error, i; 1287 1288 error = 0; 1289 for (i = 0; i < PCI_FUNCMAX; i++) { 1290 if (i == pci_get_function(dev)) 1291 continue; 1292 sibling = pci_find_dbsf(pci_get_domain(dev), pci_get_bus(dev), 1293 pci_get_slot(dev), i); 1294 if (sibling == NULL || !device_is_attached(sibling)) 1295 continue; 1296 if (detaching) 1297 error = T4_DETACH_CHILD(sibling); 1298 else 1299 (void)T4_ATTACH_CHILD(sibling); 1300 if (error) 1301 break; 1302 } 1303 return (error); 1304 } 1305 1306 /* 1307 * Idempotent 1308 */ 1309 static int 1310 t4_detach(device_t dev) 1311 { 1312 struct adapter *sc; 1313 int rc; 1314 1315 sc = device_get_softc(dev); 1316 1317 rc = notify_siblings(dev, 1); 1318 if (rc) { 1319 device_printf(dev, 1320 "failed to detach sibling devices: %d\n", rc); 1321 return (rc); 1322 } 1323 1324 return (t4_detach_common(dev)); 1325 } 1326 1327 int 1328 t4_detach_common(device_t dev) 1329 { 1330 struct adapter *sc; 1331 struct port_info *pi; 1332 int i, rc; 1333 1334 sc = device_get_softc(dev); 1335 1336 if (sc->cdev) { 1337 destroy_dev(sc->cdev); 1338 sc->cdev = NULL; 1339 } 1340 1341 sc->flags &= ~CHK_MBOX_ACCESS; 1342 if (sc->flags & FULL_INIT_DONE) { 1343 if (!(sc->flags & IS_VF)) 1344 t4_intr_disable(sc); 1345 } 1346 1347 if (device_is_attached(dev)) { 1348 rc = bus_generic_detach(dev); 1349 if (rc) { 1350 device_printf(dev, 1351 "failed to detach child devices: %d\n", rc); 1352 return (rc); 1353 } 1354 } 1355 1356 for (i = 0; i < sc->intr_count; i++) 1357 t4_free_irq(sc, &sc->irq[i]); 1358 1359 if ((sc->flags & (IS_VF | FW_OK)) == FW_OK) 1360 t4_free_tx_sched(sc); 1361 1362 for (i = 0; i < MAX_NPORTS; i++) { 1363 pi = sc->port[i]; 1364 if (pi) { 1365 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->vi[0].viid); 1366 if (pi->dev) 1367 device_delete_child(dev, pi->dev); 1368 1369 mtx_destroy(&pi->pi_lock); 1370 free(pi->vi, M_CXGBE); 1371 free(pi, M_CXGBE); 1372 } 1373 } 1374 1375 device_delete_children(dev); 1376 1377 if (sc->flags & FULL_INIT_DONE) 1378 adapter_full_uninit(sc); 1379 1380 if ((sc->flags & (IS_VF | FW_OK)) == FW_OK) 1381 t4_fw_bye(sc, sc->mbox); 1382 1383 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 1384 pci_release_msi(dev); 1385 1386 if (sc->regs_res) 1387 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 1388 sc->regs_res); 1389 1390 if (sc->udbs_res) 1391 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 1392 sc->udbs_res); 1393 1394 if (sc->msix_res) 1395 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 1396 sc->msix_res); 1397 1398 if (sc->l2t) 1399 t4_free_l2t(sc->l2t); 1400 if (sc->smt) 1401 t4_free_smt(sc->smt); 1402 #ifdef RATELIMIT 1403 t4_free_etid_table(sc); 1404 #endif 1405 1406 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 1407 free(sc->sge.ofld_txq, M_CXGBE); 1408 #endif 1409 #ifdef TCP_OFFLOAD 1410 free(sc->sge.ofld_rxq, M_CXGBE); 1411 #endif 1412 #ifdef DEV_NETMAP 1413 free(sc->sge.nm_rxq, M_CXGBE); 1414 free(sc->sge.nm_txq, M_CXGBE); 1415 #endif 1416 free(sc->irq, M_CXGBE); 1417 free(sc->sge.rxq, M_CXGBE); 1418 free(sc->sge.txq, M_CXGBE); 1419 free(sc->sge.ctrlq, M_CXGBE); 1420 free(sc->sge.iqmap, M_CXGBE); 1421 free(sc->sge.eqmap, M_CXGBE); 1422 free(sc->tids.ftid_tab, M_CXGBE); 1423 free(sc->tids.hpftid_tab, M_CXGBE); 1424 free_hftid_hash(&sc->tids); 1425 free(sc->tids.atid_tab, M_CXGBE); 1426 free(sc->tids.tid_tab, M_CXGBE); 1427 free(sc->tt.tls_rx_ports, M_CXGBE); 1428 t4_destroy_dma_tag(sc); 1429 if (mtx_initialized(&sc->sc_lock)) { 1430 sx_xlock(&t4_list_lock); 1431 SLIST_REMOVE(&t4_list, sc, adapter, link); 1432 sx_xunlock(&t4_list_lock); 1433 mtx_destroy(&sc->sc_lock); 1434 } 1435 1436 callout_drain(&sc->sfl_callout); 1437 if (mtx_initialized(&sc->tids.ftid_lock)) { 1438 mtx_destroy(&sc->tids.ftid_lock); 1439 cv_destroy(&sc->tids.ftid_cv); 1440 } 1441 if (mtx_initialized(&sc->tids.atid_lock)) 1442 mtx_destroy(&sc->tids.atid_lock); 1443 if (mtx_initialized(&sc->sfl_lock)) 1444 mtx_destroy(&sc->sfl_lock); 1445 if (mtx_initialized(&sc->ifp_lock)) 1446 mtx_destroy(&sc->ifp_lock); 1447 if (mtx_initialized(&sc->reg_lock)) 1448 mtx_destroy(&sc->reg_lock); 1449 1450 if (rw_initialized(&sc->policy_lock)) { 1451 rw_destroy(&sc->policy_lock); 1452 #ifdef TCP_OFFLOAD 1453 if (sc->policy != NULL) 1454 free_offload_policy(sc->policy); 1455 #endif 1456 } 1457 1458 for (i = 0; i < NUM_MEMWIN; i++) { 1459 struct memwin *mw = &sc->memwin[i]; 1460 1461 if (rw_initialized(&mw->mw_lock)) 1462 rw_destroy(&mw->mw_lock); 1463 } 1464 1465 bzero(sc, sizeof(*sc)); 1466 1467 return (0); 1468 } 1469 1470 static int 1471 cxgbe_probe(device_t dev) 1472 { 1473 char buf[128]; 1474 struct port_info *pi = device_get_softc(dev); 1475 1476 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 1477 device_set_desc_copy(dev, buf); 1478 1479 return (BUS_PROBE_DEFAULT); 1480 } 1481 1482 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 1483 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 1484 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS | \ 1485 IFCAP_HWRXTSTMP) 1486 #define T4_CAP_ENABLE (T4_CAP) 1487 1488 static int 1489 cxgbe_vi_attach(device_t dev, struct vi_info *vi) 1490 { 1491 struct ifnet *ifp; 1492 struct sbuf *sb; 1493 1494 vi->xact_addr_filt = -1; 1495 callout_init(&vi->tick, 1); 1496 1497 /* Allocate an ifnet and set it up */ 1498 ifp = if_alloc(IFT_ETHER); 1499 if (ifp == NULL) { 1500 device_printf(dev, "Cannot allocate ifnet\n"); 1501 return (ENOMEM); 1502 } 1503 vi->ifp = ifp; 1504 ifp->if_softc = vi; 1505 1506 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1507 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1508 1509 ifp->if_init = cxgbe_init; 1510 ifp->if_ioctl = cxgbe_ioctl; 1511 ifp->if_transmit = cxgbe_transmit; 1512 ifp->if_qflush = cxgbe_qflush; 1513 ifp->if_get_counter = cxgbe_get_counter; 1514 #ifdef RATELIMIT 1515 ifp->if_snd_tag_alloc = cxgbe_snd_tag_alloc; 1516 ifp->if_snd_tag_modify = cxgbe_snd_tag_modify; 1517 ifp->if_snd_tag_query = cxgbe_snd_tag_query; 1518 ifp->if_snd_tag_free = cxgbe_snd_tag_free; 1519 #endif 1520 1521 ifp->if_capabilities = T4_CAP; 1522 ifp->if_capenable = T4_CAP_ENABLE; 1523 #ifdef TCP_OFFLOAD 1524 if (vi->nofldrxq != 0) 1525 ifp->if_capabilities |= IFCAP_TOE; 1526 #endif 1527 #ifdef DEV_NETMAP 1528 if (vi->nnmrxq != 0) 1529 ifp->if_capabilities |= IFCAP_NETMAP; 1530 #endif 1531 #ifdef RATELIMIT 1532 if (is_ethoffload(vi->pi->adapter) && vi->nofldtxq != 0) { 1533 ifp->if_capabilities |= IFCAP_TXRTLMT; 1534 ifp->if_capenable |= IFCAP_TXRTLMT; 1535 } 1536 #endif 1537 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1538 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1539 1540 ifp->if_hw_tsomax = IP_MAXPACKET; 1541 ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS_TSO; 1542 #ifdef RATELIMIT 1543 if (is_ethoffload(vi->pi->adapter) && vi->nofldtxq != 0) 1544 ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS_EO_TSO; 1545 #endif 1546 ifp->if_hw_tsomaxsegsize = 0; 1547 1548 ether_ifattach(ifp, vi->hw_addr); 1549 #ifdef DEV_NETMAP 1550 if (ifp->if_capabilities & IFCAP_NETMAP) 1551 cxgbe_nm_attach(vi); 1552 #endif 1553 sb = sbuf_new_auto(); 1554 sbuf_printf(sb, "%d txq, %d rxq (NIC)", vi->ntxq, vi->nrxq); 1555 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 1556 switch (ifp->if_capabilities & (IFCAP_TOE | IFCAP_TXRTLMT)) { 1557 case IFCAP_TOE: 1558 sbuf_printf(sb, "; %d txq (TOE)", vi->nofldtxq); 1559 break; 1560 case IFCAP_TOE | IFCAP_TXRTLMT: 1561 sbuf_printf(sb, "; %d txq (TOE/ETHOFLD)", vi->nofldtxq); 1562 break; 1563 case IFCAP_TXRTLMT: 1564 sbuf_printf(sb, "; %d txq (ETHOFLD)", vi->nofldtxq); 1565 break; 1566 } 1567 #endif 1568 #ifdef TCP_OFFLOAD 1569 if (ifp->if_capabilities & IFCAP_TOE) 1570 sbuf_printf(sb, ", %d rxq (TOE)", vi->nofldrxq); 1571 #endif 1572 #ifdef DEV_NETMAP 1573 if (ifp->if_capabilities & IFCAP_NETMAP) 1574 sbuf_printf(sb, "; %d txq, %d rxq (netmap)", 1575 vi->nnmtxq, vi->nnmrxq); 1576 #endif 1577 sbuf_finish(sb); 1578 device_printf(dev, "%s\n", sbuf_data(sb)); 1579 sbuf_delete(sb); 1580 1581 vi_sysctls(vi); 1582 1583 return (0); 1584 } 1585 1586 static int 1587 cxgbe_attach(device_t dev) 1588 { 1589 struct port_info *pi = device_get_softc(dev); 1590 struct adapter *sc = pi->adapter; 1591 struct vi_info *vi; 1592 int i, rc; 1593 1594 callout_init_mtx(&pi->tick, &pi->pi_lock, 0); 1595 1596 rc = cxgbe_vi_attach(dev, &pi->vi[0]); 1597 if (rc) 1598 return (rc); 1599 1600 for_each_vi(pi, i, vi) { 1601 if (i == 0) 1602 continue; 1603 vi->dev = device_add_child(dev, sc->names->vi_ifnet_name, -1); 1604 if (vi->dev == NULL) { 1605 device_printf(dev, "failed to add VI %d\n", i); 1606 continue; 1607 } 1608 device_set_softc(vi->dev, vi); 1609 } 1610 1611 cxgbe_sysctls(pi); 1612 1613 bus_generic_attach(dev); 1614 1615 return (0); 1616 } 1617 1618 static void 1619 cxgbe_vi_detach(struct vi_info *vi) 1620 { 1621 struct ifnet *ifp = vi->ifp; 1622 1623 ether_ifdetach(ifp); 1624 1625 /* Let detach proceed even if these fail. */ 1626 #ifdef DEV_NETMAP 1627 if (ifp->if_capabilities & IFCAP_NETMAP) 1628 cxgbe_nm_detach(vi); 1629 #endif 1630 cxgbe_uninit_synchronized(vi); 1631 callout_drain(&vi->tick); 1632 vi_full_uninit(vi); 1633 1634 if_free(vi->ifp); 1635 vi->ifp = NULL; 1636 } 1637 1638 static int 1639 cxgbe_detach(device_t dev) 1640 { 1641 struct port_info *pi = device_get_softc(dev); 1642 struct adapter *sc = pi->adapter; 1643 int rc; 1644 1645 /* Detach the extra VIs first. */ 1646 rc = bus_generic_detach(dev); 1647 if (rc) 1648 return (rc); 1649 device_delete_children(dev); 1650 1651 doom_vi(sc, &pi->vi[0]); 1652 1653 if (pi->flags & HAS_TRACEQ) { 1654 sc->traceq = -1; /* cloner should not create ifnet */ 1655 t4_tracer_port_detach(sc); 1656 } 1657 1658 cxgbe_vi_detach(&pi->vi[0]); 1659 callout_drain(&pi->tick); 1660 ifmedia_removeall(&pi->media); 1661 1662 end_synchronized_op(sc, 0); 1663 1664 return (0); 1665 } 1666 1667 static void 1668 cxgbe_init(void *arg) 1669 { 1670 struct vi_info *vi = arg; 1671 struct adapter *sc = vi->pi->adapter; 1672 1673 if (begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4init") != 0) 1674 return; 1675 cxgbe_init_synchronized(vi); 1676 end_synchronized_op(sc, 0); 1677 } 1678 1679 static int 1680 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1681 { 1682 int rc = 0, mtu, flags; 1683 struct vi_info *vi = ifp->if_softc; 1684 struct port_info *pi = vi->pi; 1685 struct adapter *sc = pi->adapter; 1686 struct ifreq *ifr = (struct ifreq *)data; 1687 uint32_t mask; 1688 1689 switch (cmd) { 1690 case SIOCSIFMTU: 1691 mtu = ifr->ifr_mtu; 1692 if (mtu < ETHERMIN || mtu > MAX_MTU) 1693 return (EINVAL); 1694 1695 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4mtu"); 1696 if (rc) 1697 return (rc); 1698 ifp->if_mtu = mtu; 1699 if (vi->flags & VI_INIT_DONE) { 1700 t4_update_fl_bufsize(ifp); 1701 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1702 rc = update_mac_settings(ifp, XGMAC_MTU); 1703 } 1704 end_synchronized_op(sc, 0); 1705 break; 1706 1707 case SIOCSIFFLAGS: 1708 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4flg"); 1709 if (rc) 1710 return (rc); 1711 1712 if (ifp->if_flags & IFF_UP) { 1713 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1714 flags = vi->if_flags; 1715 if ((ifp->if_flags ^ flags) & 1716 (IFF_PROMISC | IFF_ALLMULTI)) { 1717 rc = update_mac_settings(ifp, 1718 XGMAC_PROMISC | XGMAC_ALLMULTI); 1719 } 1720 } else { 1721 rc = cxgbe_init_synchronized(vi); 1722 } 1723 vi->if_flags = ifp->if_flags; 1724 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1725 rc = cxgbe_uninit_synchronized(vi); 1726 } 1727 end_synchronized_op(sc, 0); 1728 break; 1729 1730 case SIOCADDMULTI: 1731 case SIOCDELMULTI: 1732 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4multi"); 1733 if (rc) 1734 return (rc); 1735 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1736 rc = update_mac_settings(ifp, XGMAC_MCADDRS); 1737 end_synchronized_op(sc, 0); 1738 break; 1739 1740 case SIOCSIFCAP: 1741 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4cap"); 1742 if (rc) 1743 return (rc); 1744 1745 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1746 if (mask & IFCAP_TXCSUM) { 1747 ifp->if_capenable ^= IFCAP_TXCSUM; 1748 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1749 1750 if (IFCAP_TSO4 & ifp->if_capenable && 1751 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1752 ifp->if_capenable &= ~IFCAP_TSO4; 1753 if_printf(ifp, 1754 "tso4 disabled due to -txcsum.\n"); 1755 } 1756 } 1757 if (mask & IFCAP_TXCSUM_IPV6) { 1758 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1759 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1760 1761 if (IFCAP_TSO6 & ifp->if_capenable && 1762 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1763 ifp->if_capenable &= ~IFCAP_TSO6; 1764 if_printf(ifp, 1765 "tso6 disabled due to -txcsum6.\n"); 1766 } 1767 } 1768 if (mask & IFCAP_RXCSUM) 1769 ifp->if_capenable ^= IFCAP_RXCSUM; 1770 if (mask & IFCAP_RXCSUM_IPV6) 1771 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1772 1773 /* 1774 * Note that we leave CSUM_TSO alone (it is always set). The 1775 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1776 * sending a TSO request our way, so it's sufficient to toggle 1777 * IFCAP_TSOx only. 1778 */ 1779 if (mask & IFCAP_TSO4) { 1780 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1781 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1782 if_printf(ifp, "enable txcsum first.\n"); 1783 rc = EAGAIN; 1784 goto fail; 1785 } 1786 ifp->if_capenable ^= IFCAP_TSO4; 1787 } 1788 if (mask & IFCAP_TSO6) { 1789 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1790 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1791 if_printf(ifp, "enable txcsum6 first.\n"); 1792 rc = EAGAIN; 1793 goto fail; 1794 } 1795 ifp->if_capenable ^= IFCAP_TSO6; 1796 } 1797 if (mask & IFCAP_LRO) { 1798 #if defined(INET) || defined(INET6) 1799 int i; 1800 struct sge_rxq *rxq; 1801 1802 ifp->if_capenable ^= IFCAP_LRO; 1803 for_each_rxq(vi, i, rxq) { 1804 if (ifp->if_capenable & IFCAP_LRO) 1805 rxq->iq.flags |= IQ_LRO_ENABLED; 1806 else 1807 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1808 } 1809 #endif 1810 } 1811 #ifdef TCP_OFFLOAD 1812 if (mask & IFCAP_TOE) { 1813 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1814 1815 rc = toe_capability(vi, enable); 1816 if (rc != 0) 1817 goto fail; 1818 1819 ifp->if_capenable ^= mask; 1820 } 1821 #endif 1822 if (mask & IFCAP_VLAN_HWTAGGING) { 1823 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1824 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1825 rc = update_mac_settings(ifp, XGMAC_VLANEX); 1826 } 1827 if (mask & IFCAP_VLAN_MTU) { 1828 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1829 1830 /* Need to find out how to disable auto-mtu-inflation */ 1831 } 1832 if (mask & IFCAP_VLAN_HWTSO) 1833 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1834 if (mask & IFCAP_VLAN_HWCSUM) 1835 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1836 #ifdef RATELIMIT 1837 if (mask & IFCAP_TXRTLMT) 1838 ifp->if_capenable ^= IFCAP_TXRTLMT; 1839 #endif 1840 if (mask & IFCAP_HWRXTSTMP) { 1841 int i; 1842 struct sge_rxq *rxq; 1843 1844 ifp->if_capenable ^= IFCAP_HWRXTSTMP; 1845 for_each_rxq(vi, i, rxq) { 1846 if (ifp->if_capenable & IFCAP_HWRXTSTMP) 1847 rxq->iq.flags |= IQ_RX_TIMESTAMP; 1848 else 1849 rxq->iq.flags &= ~IQ_RX_TIMESTAMP; 1850 } 1851 } 1852 1853 #ifdef VLAN_CAPABILITIES 1854 VLAN_CAPABILITIES(ifp); 1855 #endif 1856 fail: 1857 end_synchronized_op(sc, 0); 1858 break; 1859 1860 case SIOCSIFMEDIA: 1861 case SIOCGIFMEDIA: 1862 case SIOCGIFXMEDIA: 1863 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1864 break; 1865 1866 case SIOCGI2C: { 1867 struct ifi2creq i2c; 1868 1869 rc = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c)); 1870 if (rc != 0) 1871 break; 1872 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) { 1873 rc = EPERM; 1874 break; 1875 } 1876 if (i2c.len > sizeof(i2c.data)) { 1877 rc = EINVAL; 1878 break; 1879 } 1880 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4i2c"); 1881 if (rc) 1882 return (rc); 1883 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr, 1884 i2c.offset, i2c.len, &i2c.data[0]); 1885 end_synchronized_op(sc, 0); 1886 if (rc == 0) 1887 rc = copyout(&i2c, ifr_data_get_ptr(ifr), sizeof(i2c)); 1888 break; 1889 } 1890 1891 default: 1892 rc = ether_ioctl(ifp, cmd, data); 1893 } 1894 1895 return (rc); 1896 } 1897 1898 static int 1899 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1900 { 1901 struct vi_info *vi = ifp->if_softc; 1902 struct port_info *pi = vi->pi; 1903 struct adapter *sc = pi->adapter; 1904 struct sge_txq *txq; 1905 void *items[1]; 1906 int rc; 1907 1908 M_ASSERTPKTHDR(m); 1909 MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */ 1910 1911 if (__predict_false(pi->link_cfg.link_ok == false)) { 1912 m_freem(m); 1913 return (ENETDOWN); 1914 } 1915 1916 rc = parse_pkt(sc, &m); 1917 if (__predict_false(rc != 0)) { 1918 MPASS(m == NULL); /* was freed already */ 1919 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */ 1920 return (rc); 1921 } 1922 #ifdef RATELIMIT 1923 if (m->m_pkthdr.snd_tag != NULL) { 1924 /* EAGAIN tells the stack we are not the correct interface. */ 1925 if (__predict_false(ifp != m->m_pkthdr.snd_tag->ifp)) { 1926 m_freem(m); 1927 return (EAGAIN); 1928 } 1929 1930 return (ethofld_transmit(ifp, m)); 1931 } 1932 #endif 1933 1934 /* Select a txq. */ 1935 txq = &sc->sge.txq[vi->first_txq]; 1936 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 1937 txq += ((m->m_pkthdr.flowid % (vi->ntxq - vi->rsrv_noflowq)) + 1938 vi->rsrv_noflowq); 1939 1940 items[0] = m; 1941 rc = mp_ring_enqueue(txq->r, items, 1, 4096); 1942 if (__predict_false(rc != 0)) 1943 m_freem(m); 1944 1945 return (rc); 1946 } 1947 1948 static void 1949 cxgbe_qflush(struct ifnet *ifp) 1950 { 1951 struct vi_info *vi = ifp->if_softc; 1952 struct sge_txq *txq; 1953 int i; 1954 1955 /* queues do not exist if !VI_INIT_DONE. */ 1956 if (vi->flags & VI_INIT_DONE) { 1957 for_each_txq(vi, i, txq) { 1958 TXQ_LOCK(txq); 1959 txq->eq.flags |= EQ_QFLUSH; 1960 TXQ_UNLOCK(txq); 1961 while (!mp_ring_is_idle(txq->r)) { 1962 mp_ring_check_drainage(txq->r, 0); 1963 pause("qflush", 1); 1964 } 1965 TXQ_LOCK(txq); 1966 txq->eq.flags &= ~EQ_QFLUSH; 1967 TXQ_UNLOCK(txq); 1968 } 1969 } 1970 if_qflush(ifp); 1971 } 1972 1973 static uint64_t 1974 vi_get_counter(struct ifnet *ifp, ift_counter c) 1975 { 1976 struct vi_info *vi = ifp->if_softc; 1977 struct fw_vi_stats_vf *s = &vi->stats; 1978 1979 vi_refresh_stats(vi->pi->adapter, vi); 1980 1981 switch (c) { 1982 case IFCOUNTER_IPACKETS: 1983 return (s->rx_bcast_frames + s->rx_mcast_frames + 1984 s->rx_ucast_frames); 1985 case IFCOUNTER_IERRORS: 1986 return (s->rx_err_frames); 1987 case IFCOUNTER_OPACKETS: 1988 return (s->tx_bcast_frames + s->tx_mcast_frames + 1989 s->tx_ucast_frames + s->tx_offload_frames); 1990 case IFCOUNTER_OERRORS: 1991 return (s->tx_drop_frames); 1992 case IFCOUNTER_IBYTES: 1993 return (s->rx_bcast_bytes + s->rx_mcast_bytes + 1994 s->rx_ucast_bytes); 1995 case IFCOUNTER_OBYTES: 1996 return (s->tx_bcast_bytes + s->tx_mcast_bytes + 1997 s->tx_ucast_bytes + s->tx_offload_bytes); 1998 case IFCOUNTER_IMCASTS: 1999 return (s->rx_mcast_frames); 2000 case IFCOUNTER_OMCASTS: 2001 return (s->tx_mcast_frames); 2002 case IFCOUNTER_OQDROPS: { 2003 uint64_t drops; 2004 2005 drops = 0; 2006 if (vi->flags & VI_INIT_DONE) { 2007 int i; 2008 struct sge_txq *txq; 2009 2010 for_each_txq(vi, i, txq) 2011 drops += counter_u64_fetch(txq->r->drops); 2012 } 2013 2014 return (drops); 2015 2016 } 2017 2018 default: 2019 return (if_get_counter_default(ifp, c)); 2020 } 2021 } 2022 2023 uint64_t 2024 cxgbe_get_counter(struct ifnet *ifp, ift_counter c) 2025 { 2026 struct vi_info *vi = ifp->if_softc; 2027 struct port_info *pi = vi->pi; 2028 struct adapter *sc = pi->adapter; 2029 struct port_stats *s = &pi->stats; 2030 2031 if (pi->nvi > 1 || sc->flags & IS_VF) 2032 return (vi_get_counter(ifp, c)); 2033 2034 cxgbe_refresh_stats(sc, pi); 2035 2036 switch (c) { 2037 case IFCOUNTER_IPACKETS: 2038 return (s->rx_frames); 2039 2040 case IFCOUNTER_IERRORS: 2041 return (s->rx_jabber + s->rx_runt + s->rx_too_long + 2042 s->rx_fcs_err + s->rx_len_err); 2043 2044 case IFCOUNTER_OPACKETS: 2045 return (s->tx_frames); 2046 2047 case IFCOUNTER_OERRORS: 2048 return (s->tx_error_frames); 2049 2050 case IFCOUNTER_IBYTES: 2051 return (s->rx_octets); 2052 2053 case IFCOUNTER_OBYTES: 2054 return (s->tx_octets); 2055 2056 case IFCOUNTER_IMCASTS: 2057 return (s->rx_mcast_frames); 2058 2059 case IFCOUNTER_OMCASTS: 2060 return (s->tx_mcast_frames); 2061 2062 case IFCOUNTER_IQDROPS: 2063 return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 2064 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 2065 s->rx_trunc3 + pi->tnl_cong_drops); 2066 2067 case IFCOUNTER_OQDROPS: { 2068 uint64_t drops; 2069 2070 drops = s->tx_drop; 2071 if (vi->flags & VI_INIT_DONE) { 2072 int i; 2073 struct sge_txq *txq; 2074 2075 for_each_txq(vi, i, txq) 2076 drops += counter_u64_fetch(txq->r->drops); 2077 } 2078 2079 return (drops); 2080 2081 } 2082 2083 default: 2084 return (if_get_counter_default(ifp, c)); 2085 } 2086 } 2087 2088 /* 2089 * The kernel picks a media from the list we had provided but we still validate 2090 * the requeste. 2091 */ 2092 int 2093 cxgbe_media_change(struct ifnet *ifp) 2094 { 2095 struct vi_info *vi = ifp->if_softc; 2096 struct port_info *pi = vi->pi; 2097 struct ifmedia *ifm = &pi->media; 2098 struct link_config *lc = &pi->link_cfg; 2099 struct adapter *sc = pi->adapter; 2100 int rc; 2101 2102 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4mec"); 2103 if (rc != 0) 2104 return (rc); 2105 PORT_LOCK(pi); 2106 if (IFM_SUBTYPE(ifm->ifm_media) == IFM_AUTO) { 2107 /* ifconfig .. media autoselect */ 2108 if (!(lc->supported & FW_PORT_CAP32_ANEG)) { 2109 rc = ENOTSUP; /* AN not supported by transceiver */ 2110 goto done; 2111 } 2112 lc->requested_aneg = AUTONEG_ENABLE; 2113 lc->requested_speed = 0; 2114 lc->requested_fc |= PAUSE_AUTONEG; 2115 } else { 2116 lc->requested_aneg = AUTONEG_DISABLE; 2117 lc->requested_speed = 2118 ifmedia_baudrate(ifm->ifm_media) / 1000000; 2119 lc->requested_fc = 0; 2120 if (IFM_OPTIONS(ifm->ifm_media) & IFM_ETH_RXPAUSE) 2121 lc->requested_fc |= PAUSE_RX; 2122 if (IFM_OPTIONS(ifm->ifm_media) & IFM_ETH_TXPAUSE) 2123 lc->requested_fc |= PAUSE_TX; 2124 } 2125 if (pi->up_vis > 0) { 2126 fixup_link_config(pi); 2127 rc = apply_link_config(pi); 2128 } 2129 done: 2130 PORT_UNLOCK(pi); 2131 end_synchronized_op(sc, 0); 2132 return (rc); 2133 } 2134 2135 /* 2136 * Base media word (without ETHER, pause, link active, etc.) for the port at the 2137 * given speed. 2138 */ 2139 static int 2140 port_mword(struct port_info *pi, uint32_t speed) 2141 { 2142 2143 MPASS(speed & M_FW_PORT_CAP32_SPEED); 2144 MPASS(powerof2(speed)); 2145 2146 switch(pi->port_type) { 2147 case FW_PORT_TYPE_BT_SGMII: 2148 case FW_PORT_TYPE_BT_XFI: 2149 case FW_PORT_TYPE_BT_XAUI: 2150 /* BaseT */ 2151 switch (speed) { 2152 case FW_PORT_CAP32_SPEED_100M: 2153 return (IFM_100_T); 2154 case FW_PORT_CAP32_SPEED_1G: 2155 return (IFM_1000_T); 2156 case FW_PORT_CAP32_SPEED_10G: 2157 return (IFM_10G_T); 2158 } 2159 break; 2160 case FW_PORT_TYPE_KX4: 2161 if (speed == FW_PORT_CAP32_SPEED_10G) 2162 return (IFM_10G_KX4); 2163 break; 2164 case FW_PORT_TYPE_CX4: 2165 if (speed == FW_PORT_CAP32_SPEED_10G) 2166 return (IFM_10G_CX4); 2167 break; 2168 case FW_PORT_TYPE_KX: 2169 if (speed == FW_PORT_CAP32_SPEED_1G) 2170 return (IFM_1000_KX); 2171 break; 2172 case FW_PORT_TYPE_KR: 2173 case FW_PORT_TYPE_BP_AP: 2174 case FW_PORT_TYPE_BP4_AP: 2175 case FW_PORT_TYPE_BP40_BA: 2176 case FW_PORT_TYPE_KR4_100G: 2177 case FW_PORT_TYPE_KR_SFP28: 2178 case FW_PORT_TYPE_KR_XLAUI: 2179 switch (speed) { 2180 case FW_PORT_CAP32_SPEED_1G: 2181 return (IFM_1000_KX); 2182 case FW_PORT_CAP32_SPEED_10G: 2183 return (IFM_10G_KR); 2184 case FW_PORT_CAP32_SPEED_25G: 2185 return (IFM_25G_KR); 2186 case FW_PORT_CAP32_SPEED_40G: 2187 return (IFM_40G_KR4); 2188 case FW_PORT_CAP32_SPEED_50G: 2189 return (IFM_50G_KR2); 2190 case FW_PORT_CAP32_SPEED_100G: 2191 return (IFM_100G_KR4); 2192 } 2193 break; 2194 case FW_PORT_TYPE_FIBER_XFI: 2195 case FW_PORT_TYPE_FIBER_XAUI: 2196 case FW_PORT_TYPE_SFP: 2197 case FW_PORT_TYPE_QSFP_10G: 2198 case FW_PORT_TYPE_QSA: 2199 case FW_PORT_TYPE_QSFP: 2200 case FW_PORT_TYPE_CR4_QSFP: 2201 case FW_PORT_TYPE_CR_QSFP: 2202 case FW_PORT_TYPE_CR2_QSFP: 2203 case FW_PORT_TYPE_SFP28: 2204 /* Pluggable transceiver */ 2205 switch (pi->mod_type) { 2206 case FW_PORT_MOD_TYPE_LR: 2207 switch (speed) { 2208 case FW_PORT_CAP32_SPEED_1G: 2209 return (IFM_1000_LX); 2210 case FW_PORT_CAP32_SPEED_10G: 2211 return (IFM_10G_LR); 2212 case FW_PORT_CAP32_SPEED_25G: 2213 return (IFM_25G_LR); 2214 case FW_PORT_CAP32_SPEED_40G: 2215 return (IFM_40G_LR4); 2216 case FW_PORT_CAP32_SPEED_50G: 2217 return (IFM_50G_LR2); 2218 case FW_PORT_CAP32_SPEED_100G: 2219 return (IFM_100G_LR4); 2220 } 2221 break; 2222 case FW_PORT_MOD_TYPE_SR: 2223 switch (speed) { 2224 case FW_PORT_CAP32_SPEED_1G: 2225 return (IFM_1000_SX); 2226 case FW_PORT_CAP32_SPEED_10G: 2227 return (IFM_10G_SR); 2228 case FW_PORT_CAP32_SPEED_25G: 2229 return (IFM_25G_SR); 2230 case FW_PORT_CAP32_SPEED_40G: 2231 return (IFM_40G_SR4); 2232 case FW_PORT_CAP32_SPEED_50G: 2233 return (IFM_50G_SR2); 2234 case FW_PORT_CAP32_SPEED_100G: 2235 return (IFM_100G_SR4); 2236 } 2237 break; 2238 case FW_PORT_MOD_TYPE_ER: 2239 if (speed == FW_PORT_CAP32_SPEED_10G) 2240 return (IFM_10G_ER); 2241 break; 2242 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2243 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2244 switch (speed) { 2245 case FW_PORT_CAP32_SPEED_1G: 2246 return (IFM_1000_CX); 2247 case FW_PORT_CAP32_SPEED_10G: 2248 return (IFM_10G_TWINAX); 2249 case FW_PORT_CAP32_SPEED_25G: 2250 return (IFM_25G_CR); 2251 case FW_PORT_CAP32_SPEED_40G: 2252 return (IFM_40G_CR4); 2253 case FW_PORT_CAP32_SPEED_50G: 2254 return (IFM_50G_CR2); 2255 case FW_PORT_CAP32_SPEED_100G: 2256 return (IFM_100G_CR4); 2257 } 2258 break; 2259 case FW_PORT_MOD_TYPE_LRM: 2260 if (speed == FW_PORT_CAP32_SPEED_10G) 2261 return (IFM_10G_LRM); 2262 break; 2263 case FW_PORT_MOD_TYPE_NA: 2264 MPASS(0); /* Not pluggable? */ 2265 /* fall throough */ 2266 case FW_PORT_MOD_TYPE_ERROR: 2267 case FW_PORT_MOD_TYPE_UNKNOWN: 2268 case FW_PORT_MOD_TYPE_NOTSUPPORTED: 2269 break; 2270 case FW_PORT_MOD_TYPE_NONE: 2271 return (IFM_NONE); 2272 } 2273 break; 2274 case FW_PORT_TYPE_NONE: 2275 return (IFM_NONE); 2276 } 2277 2278 return (IFM_UNKNOWN); 2279 } 2280 2281 void 2282 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 2283 { 2284 struct vi_info *vi = ifp->if_softc; 2285 struct port_info *pi = vi->pi; 2286 struct adapter *sc = pi->adapter; 2287 struct link_config *lc = &pi->link_cfg; 2288 2289 if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4med") != 0) 2290 return; 2291 PORT_LOCK(pi); 2292 2293 if (pi->up_vis == 0) { 2294 /* 2295 * If all the interfaces are administratively down the firmware 2296 * does not report transceiver changes. Refresh port info here 2297 * so that ifconfig displays accurate ifmedia at all times. 2298 * This is the only reason we have a synchronized op in this 2299 * function. Just PORT_LOCK would have been enough otherwise. 2300 */ 2301 t4_update_port_info(pi); 2302 build_medialist(pi); 2303 } 2304 2305 /* ifm_status */ 2306 ifmr->ifm_status = IFM_AVALID; 2307 if (lc->link_ok == false) 2308 goto done; 2309 ifmr->ifm_status |= IFM_ACTIVE; 2310 2311 /* ifm_active */ 2312 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 2313 ifmr->ifm_active &= ~(IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE); 2314 if (lc->fc & PAUSE_RX) 2315 ifmr->ifm_active |= IFM_ETH_RXPAUSE; 2316 if (lc->fc & PAUSE_TX) 2317 ifmr->ifm_active |= IFM_ETH_TXPAUSE; 2318 ifmr->ifm_active |= port_mword(pi, speed_to_fwcap(lc->speed)); 2319 done: 2320 PORT_UNLOCK(pi); 2321 end_synchronized_op(sc, 0); 2322 } 2323 2324 static int 2325 vcxgbe_probe(device_t dev) 2326 { 2327 char buf[128]; 2328 struct vi_info *vi = device_get_softc(dev); 2329 2330 snprintf(buf, sizeof(buf), "port %d vi %td", vi->pi->port_id, 2331 vi - vi->pi->vi); 2332 device_set_desc_copy(dev, buf); 2333 2334 return (BUS_PROBE_DEFAULT); 2335 } 2336 2337 static int 2338 alloc_extra_vi(struct adapter *sc, struct port_info *pi, struct vi_info *vi) 2339 { 2340 int func, index, rc; 2341 uint32_t param, val; 2342 2343 ASSERT_SYNCHRONIZED_OP(sc); 2344 2345 index = vi - pi->vi; 2346 MPASS(index > 0); /* This function deals with _extra_ VIs only */ 2347 KASSERT(index < nitems(vi_mac_funcs), 2348 ("%s: VI %s doesn't have a MAC func", __func__, 2349 device_get_nameunit(vi->dev))); 2350 func = vi_mac_funcs[index]; 2351 rc = t4_alloc_vi_func(sc, sc->mbox, pi->tx_chan, sc->pf, 0, 1, 2352 vi->hw_addr, &vi->rss_size, func, 0); 2353 if (rc < 0) { 2354 device_printf(vi->dev, "failed to allocate virtual interface %d" 2355 "for port %d: %d\n", index, pi->port_id, -rc); 2356 return (-rc); 2357 } 2358 vi->viid = rc; 2359 if (chip_id(sc) <= CHELSIO_T5) 2360 vi->smt_idx = (rc & 0x7f) << 1; 2361 else 2362 vi->smt_idx = (rc & 0x7f); 2363 2364 if (vi->rss_size == 1) { 2365 /* 2366 * This VI didn't get a slice of the RSS table. Reduce the 2367 * number of VIs being created (hw.cxgbe.num_vis) or modify the 2368 * configuration file (nvi, rssnvi for this PF) if this is a 2369 * problem. 2370 */ 2371 device_printf(vi->dev, "RSS table not available.\n"); 2372 vi->rss_base = 0xffff; 2373 2374 return (0); 2375 } 2376 2377 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 2378 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_RSSINFO) | 2379 V_FW_PARAMS_PARAM_YZ(vi->viid); 2380 rc = t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2381 if (rc) 2382 vi->rss_base = 0xffff; 2383 else { 2384 MPASS((val >> 16) == vi->rss_size); 2385 vi->rss_base = val & 0xffff; 2386 } 2387 2388 return (0); 2389 } 2390 2391 static int 2392 vcxgbe_attach(device_t dev) 2393 { 2394 struct vi_info *vi; 2395 struct port_info *pi; 2396 struct adapter *sc; 2397 int rc; 2398 2399 vi = device_get_softc(dev); 2400 pi = vi->pi; 2401 sc = pi->adapter; 2402 2403 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4via"); 2404 if (rc) 2405 return (rc); 2406 rc = alloc_extra_vi(sc, pi, vi); 2407 end_synchronized_op(sc, 0); 2408 if (rc) 2409 return (rc); 2410 2411 rc = cxgbe_vi_attach(dev, vi); 2412 if (rc) { 2413 t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid); 2414 return (rc); 2415 } 2416 return (0); 2417 } 2418 2419 static int 2420 vcxgbe_detach(device_t dev) 2421 { 2422 struct vi_info *vi; 2423 struct adapter *sc; 2424 2425 vi = device_get_softc(dev); 2426 sc = vi->pi->adapter; 2427 2428 doom_vi(sc, vi); 2429 2430 cxgbe_vi_detach(vi); 2431 t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid); 2432 2433 end_synchronized_op(sc, 0); 2434 2435 return (0); 2436 } 2437 2438 void 2439 t4_fatal_err(struct adapter *sc) 2440 { 2441 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 2442 t4_intr_disable(sc); 2443 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 2444 device_get_nameunit(sc->dev)); 2445 if (t4_panic_on_fatal_err) 2446 panic("panic requested on fatal error"); 2447 } 2448 2449 void 2450 t4_add_adapter(struct adapter *sc) 2451 { 2452 sx_xlock(&t4_list_lock); 2453 SLIST_INSERT_HEAD(&t4_list, sc, link); 2454 sx_xunlock(&t4_list_lock); 2455 } 2456 2457 int 2458 t4_map_bars_0_and_4(struct adapter *sc) 2459 { 2460 sc->regs_rid = PCIR_BAR(0); 2461 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 2462 &sc->regs_rid, RF_ACTIVE); 2463 if (sc->regs_res == NULL) { 2464 device_printf(sc->dev, "cannot map registers.\n"); 2465 return (ENXIO); 2466 } 2467 sc->bt = rman_get_bustag(sc->regs_res); 2468 sc->bh = rman_get_bushandle(sc->regs_res); 2469 sc->mmio_len = rman_get_size(sc->regs_res); 2470 setbit(&sc->doorbells, DOORBELL_KDB); 2471 2472 sc->msix_rid = PCIR_BAR(4); 2473 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 2474 &sc->msix_rid, RF_ACTIVE); 2475 if (sc->msix_res == NULL) { 2476 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 2477 return (ENXIO); 2478 } 2479 2480 return (0); 2481 } 2482 2483 int 2484 t4_map_bar_2(struct adapter *sc) 2485 { 2486 2487 /* 2488 * T4: only iWARP driver uses the userspace doorbells. There is no need 2489 * to map it if RDMA is disabled. 2490 */ 2491 if (is_t4(sc) && sc->rdmacaps == 0) 2492 return (0); 2493 2494 sc->udbs_rid = PCIR_BAR(2); 2495 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 2496 &sc->udbs_rid, RF_ACTIVE); 2497 if (sc->udbs_res == NULL) { 2498 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 2499 return (ENXIO); 2500 } 2501 sc->udbs_base = rman_get_virtual(sc->udbs_res); 2502 2503 if (chip_id(sc) >= CHELSIO_T5) { 2504 setbit(&sc->doorbells, DOORBELL_UDB); 2505 #if defined(__i386__) || defined(__amd64__) 2506 if (t5_write_combine) { 2507 int rc, mode; 2508 2509 /* 2510 * Enable write combining on BAR2. This is the 2511 * userspace doorbell BAR and is split into 128B 2512 * (UDBS_SEG_SIZE) doorbell regions, each associated 2513 * with an egress queue. The first 64B has the doorbell 2514 * and the second 64B can be used to submit a tx work 2515 * request with an implicit doorbell. 2516 */ 2517 2518 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 2519 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 2520 if (rc == 0) { 2521 clrbit(&sc->doorbells, DOORBELL_UDB); 2522 setbit(&sc->doorbells, DOORBELL_WCWR); 2523 setbit(&sc->doorbells, DOORBELL_UDBWC); 2524 } else { 2525 device_printf(sc->dev, 2526 "couldn't enable write combining: %d\n", 2527 rc); 2528 } 2529 2530 mode = is_t5(sc) ? V_STATMODE(0) : V_T6_STATMODE(0); 2531 t4_write_reg(sc, A_SGE_STAT_CFG, 2532 V_STATSOURCE_T5(7) | mode); 2533 } 2534 #endif 2535 } 2536 sc->iwt.wc_en = isset(&sc->doorbells, DOORBELL_UDBWC) ? 1 : 0; 2537 2538 return (0); 2539 } 2540 2541 struct memwin_init { 2542 uint32_t base; 2543 uint32_t aperture; 2544 }; 2545 2546 static const struct memwin_init t4_memwin[NUM_MEMWIN] = { 2547 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 2548 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 2549 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 2550 }; 2551 2552 static const struct memwin_init t5_memwin[NUM_MEMWIN] = { 2553 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 2554 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 2555 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 2556 }; 2557 2558 static void 2559 setup_memwin(struct adapter *sc) 2560 { 2561 const struct memwin_init *mw_init; 2562 struct memwin *mw; 2563 int i; 2564 uint32_t bar0; 2565 2566 if (is_t4(sc)) { 2567 /* 2568 * Read low 32b of bar0 indirectly via the hardware backdoor 2569 * mechanism. Works from within PCI passthrough environments 2570 * too, where rman_get_start() can return a different value. We 2571 * need to program the T4 memory window decoders with the actual 2572 * addresses that will be coming across the PCIe link. 2573 */ 2574 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 2575 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 2576 2577 mw_init = &t4_memwin[0]; 2578 } else { 2579 /* T5+ use the relative offset inside the PCIe BAR */ 2580 bar0 = 0; 2581 2582 mw_init = &t5_memwin[0]; 2583 } 2584 2585 for (i = 0, mw = &sc->memwin[0]; i < NUM_MEMWIN; i++, mw_init++, mw++) { 2586 rw_init(&mw->mw_lock, "memory window access"); 2587 mw->mw_base = mw_init->base; 2588 mw->mw_aperture = mw_init->aperture; 2589 mw->mw_curpos = 0; 2590 t4_write_reg(sc, 2591 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 2592 (mw->mw_base + bar0) | V_BIR(0) | 2593 V_WINDOW(ilog2(mw->mw_aperture) - 10)); 2594 rw_wlock(&mw->mw_lock); 2595 position_memwin(sc, i, 0); 2596 rw_wunlock(&mw->mw_lock); 2597 } 2598 2599 /* flush */ 2600 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 2601 } 2602 2603 /* 2604 * Positions the memory window at the given address in the card's address space. 2605 * There are some alignment requirements and the actual position may be at an 2606 * address prior to the requested address. mw->mw_curpos always has the actual 2607 * position of the window. 2608 */ 2609 static void 2610 position_memwin(struct adapter *sc, int idx, uint32_t addr) 2611 { 2612 struct memwin *mw; 2613 uint32_t pf; 2614 uint32_t reg; 2615 2616 MPASS(idx >= 0 && idx < NUM_MEMWIN); 2617 mw = &sc->memwin[idx]; 2618 rw_assert(&mw->mw_lock, RA_WLOCKED); 2619 2620 if (is_t4(sc)) { 2621 pf = 0; 2622 mw->mw_curpos = addr & ~0xf; /* start must be 16B aligned */ 2623 } else { 2624 pf = V_PFNUM(sc->pf); 2625 mw->mw_curpos = addr & ~0x7f; /* start must be 128B aligned */ 2626 } 2627 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, idx); 2628 t4_write_reg(sc, reg, mw->mw_curpos | pf); 2629 t4_read_reg(sc, reg); /* flush */ 2630 } 2631 2632 int 2633 rw_via_memwin(struct adapter *sc, int idx, uint32_t addr, uint32_t *val, 2634 int len, int rw) 2635 { 2636 struct memwin *mw; 2637 uint32_t mw_end, v; 2638 2639 MPASS(idx >= 0 && idx < NUM_MEMWIN); 2640 2641 /* Memory can only be accessed in naturally aligned 4 byte units */ 2642 if (addr & 3 || len & 3 || len <= 0) 2643 return (EINVAL); 2644 2645 mw = &sc->memwin[idx]; 2646 while (len > 0) { 2647 rw_rlock(&mw->mw_lock); 2648 mw_end = mw->mw_curpos + mw->mw_aperture; 2649 if (addr >= mw_end || addr < mw->mw_curpos) { 2650 /* Will need to reposition the window */ 2651 if (!rw_try_upgrade(&mw->mw_lock)) { 2652 rw_runlock(&mw->mw_lock); 2653 rw_wlock(&mw->mw_lock); 2654 } 2655 rw_assert(&mw->mw_lock, RA_WLOCKED); 2656 position_memwin(sc, idx, addr); 2657 rw_downgrade(&mw->mw_lock); 2658 mw_end = mw->mw_curpos + mw->mw_aperture; 2659 } 2660 rw_assert(&mw->mw_lock, RA_RLOCKED); 2661 while (addr < mw_end && len > 0) { 2662 if (rw == 0) { 2663 v = t4_read_reg(sc, mw->mw_base + addr - 2664 mw->mw_curpos); 2665 *val++ = le32toh(v); 2666 } else { 2667 v = *val++; 2668 t4_write_reg(sc, mw->mw_base + addr - 2669 mw->mw_curpos, htole32(v)); 2670 } 2671 addr += 4; 2672 len -= 4; 2673 } 2674 rw_runlock(&mw->mw_lock); 2675 } 2676 2677 return (0); 2678 } 2679 2680 int 2681 alloc_atid_tab(struct tid_info *t, int flags) 2682 { 2683 int i; 2684 2685 MPASS(t->natids > 0); 2686 MPASS(t->atid_tab == NULL); 2687 2688 t->atid_tab = malloc(t->natids * sizeof(*t->atid_tab), M_CXGBE, 2689 M_ZERO | flags); 2690 if (t->atid_tab == NULL) 2691 return (ENOMEM); 2692 mtx_init(&t->atid_lock, "atid lock", NULL, MTX_DEF); 2693 t->afree = t->atid_tab; 2694 t->atids_in_use = 0; 2695 for (i = 1; i < t->natids; i++) 2696 t->atid_tab[i - 1].next = &t->atid_tab[i]; 2697 t->atid_tab[t->natids - 1].next = NULL; 2698 2699 return (0); 2700 } 2701 2702 void 2703 free_atid_tab(struct tid_info *t) 2704 { 2705 2706 KASSERT(t->atids_in_use == 0, 2707 ("%s: %d atids still in use.", __func__, t->atids_in_use)); 2708 2709 if (mtx_initialized(&t->atid_lock)) 2710 mtx_destroy(&t->atid_lock); 2711 free(t->atid_tab, M_CXGBE); 2712 t->atid_tab = NULL; 2713 } 2714 2715 int 2716 alloc_atid(struct adapter *sc, void *ctx) 2717 { 2718 struct tid_info *t = &sc->tids; 2719 int atid = -1; 2720 2721 mtx_lock(&t->atid_lock); 2722 if (t->afree) { 2723 union aopen_entry *p = t->afree; 2724 2725 atid = p - t->atid_tab; 2726 MPASS(atid <= M_TID_TID); 2727 t->afree = p->next; 2728 p->data = ctx; 2729 t->atids_in_use++; 2730 } 2731 mtx_unlock(&t->atid_lock); 2732 return (atid); 2733 } 2734 2735 void * 2736 lookup_atid(struct adapter *sc, int atid) 2737 { 2738 struct tid_info *t = &sc->tids; 2739 2740 return (t->atid_tab[atid].data); 2741 } 2742 2743 void 2744 free_atid(struct adapter *sc, int atid) 2745 { 2746 struct tid_info *t = &sc->tids; 2747 union aopen_entry *p = &t->atid_tab[atid]; 2748 2749 mtx_lock(&t->atid_lock); 2750 p->next = t->afree; 2751 t->afree = p; 2752 t->atids_in_use--; 2753 mtx_unlock(&t->atid_lock); 2754 } 2755 2756 static void 2757 queue_tid_release(struct adapter *sc, int tid) 2758 { 2759 2760 CXGBE_UNIMPLEMENTED("deferred tid release"); 2761 } 2762 2763 void 2764 release_tid(struct adapter *sc, int tid, struct sge_wrq *ctrlq) 2765 { 2766 struct wrqe *wr; 2767 struct cpl_tid_release *req; 2768 2769 wr = alloc_wrqe(sizeof(*req), ctrlq); 2770 if (wr == NULL) { 2771 queue_tid_release(sc, tid); /* defer */ 2772 return; 2773 } 2774 req = wrtod(wr); 2775 2776 INIT_TP_WR_MIT_CPL(req, CPL_TID_RELEASE, tid); 2777 2778 t4_wrq_tx(sc, wr); 2779 } 2780 2781 static int 2782 t4_range_cmp(const void *a, const void *b) 2783 { 2784 return ((const struct t4_range *)a)->start - 2785 ((const struct t4_range *)b)->start; 2786 } 2787 2788 /* 2789 * Verify that the memory range specified by the addr/len pair is valid within 2790 * the card's address space. 2791 */ 2792 static int 2793 validate_mem_range(struct adapter *sc, uint32_t addr, int len) 2794 { 2795 struct t4_range mem_ranges[4], *r, *next; 2796 uint32_t em, addr_len; 2797 int i, n, remaining; 2798 2799 /* Memory can only be accessed in naturally aligned 4 byte units */ 2800 if (addr & 3 || len & 3 || len <= 0) 2801 return (EINVAL); 2802 2803 /* Enabled memories */ 2804 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 2805 2806 r = &mem_ranges[0]; 2807 n = 0; 2808 bzero(r, sizeof(mem_ranges)); 2809 if (em & F_EDRAM0_ENABLE) { 2810 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 2811 r->size = G_EDRAM0_SIZE(addr_len) << 20; 2812 if (r->size > 0) { 2813 r->start = G_EDRAM0_BASE(addr_len) << 20; 2814 if (addr >= r->start && 2815 addr + len <= r->start + r->size) 2816 return (0); 2817 r++; 2818 n++; 2819 } 2820 } 2821 if (em & F_EDRAM1_ENABLE) { 2822 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 2823 r->size = G_EDRAM1_SIZE(addr_len) << 20; 2824 if (r->size > 0) { 2825 r->start = G_EDRAM1_BASE(addr_len) << 20; 2826 if (addr >= r->start && 2827 addr + len <= r->start + r->size) 2828 return (0); 2829 r++; 2830 n++; 2831 } 2832 } 2833 if (em & F_EXT_MEM_ENABLE) { 2834 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 2835 r->size = G_EXT_MEM_SIZE(addr_len) << 20; 2836 if (r->size > 0) { 2837 r->start = G_EXT_MEM_BASE(addr_len) << 20; 2838 if (addr >= r->start && 2839 addr + len <= r->start + r->size) 2840 return (0); 2841 r++; 2842 n++; 2843 } 2844 } 2845 if (is_t5(sc) && em & F_EXT_MEM1_ENABLE) { 2846 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 2847 r->size = G_EXT_MEM1_SIZE(addr_len) << 20; 2848 if (r->size > 0) { 2849 r->start = G_EXT_MEM1_BASE(addr_len) << 20; 2850 if (addr >= r->start && 2851 addr + len <= r->start + r->size) 2852 return (0); 2853 r++; 2854 n++; 2855 } 2856 } 2857 MPASS(n <= nitems(mem_ranges)); 2858 2859 if (n > 1) { 2860 /* Sort and merge the ranges. */ 2861 qsort(mem_ranges, n, sizeof(struct t4_range), t4_range_cmp); 2862 2863 /* Start from index 0 and examine the next n - 1 entries. */ 2864 r = &mem_ranges[0]; 2865 for (remaining = n - 1; remaining > 0; remaining--, r++) { 2866 2867 MPASS(r->size > 0); /* r is a valid entry. */ 2868 next = r + 1; 2869 MPASS(next->size > 0); /* and so is the next one. */ 2870 2871 while (r->start + r->size >= next->start) { 2872 /* Merge the next one into the current entry. */ 2873 r->size = max(r->start + r->size, 2874 next->start + next->size) - r->start; 2875 n--; /* One fewer entry in total. */ 2876 if (--remaining == 0) 2877 goto done; /* short circuit */ 2878 next++; 2879 } 2880 if (next != r + 1) { 2881 /* 2882 * Some entries were merged into r and next 2883 * points to the first valid entry that couldn't 2884 * be merged. 2885 */ 2886 MPASS(next->size > 0); /* must be valid */ 2887 memcpy(r + 1, next, remaining * sizeof(*r)); 2888 #ifdef INVARIANTS 2889 /* 2890 * This so that the foo->size assertion in the 2891 * next iteration of the loop do the right 2892 * thing for entries that were pulled up and are 2893 * no longer valid. 2894 */ 2895 MPASS(n < nitems(mem_ranges)); 2896 bzero(&mem_ranges[n], (nitems(mem_ranges) - n) * 2897 sizeof(struct t4_range)); 2898 #endif 2899 } 2900 } 2901 done: 2902 /* Done merging the ranges. */ 2903 MPASS(n > 0); 2904 r = &mem_ranges[0]; 2905 for (i = 0; i < n; i++, r++) { 2906 if (addr >= r->start && 2907 addr + len <= r->start + r->size) 2908 return (0); 2909 } 2910 } 2911 2912 return (EFAULT); 2913 } 2914 2915 static int 2916 fwmtype_to_hwmtype(int mtype) 2917 { 2918 2919 switch (mtype) { 2920 case FW_MEMTYPE_EDC0: 2921 return (MEM_EDC0); 2922 case FW_MEMTYPE_EDC1: 2923 return (MEM_EDC1); 2924 case FW_MEMTYPE_EXTMEM: 2925 return (MEM_MC0); 2926 case FW_MEMTYPE_EXTMEM1: 2927 return (MEM_MC1); 2928 default: 2929 panic("%s: cannot translate fw mtype %d.", __func__, mtype); 2930 } 2931 } 2932 2933 /* 2934 * Verify that the memory range specified by the memtype/offset/len pair is 2935 * valid and lies entirely within the memtype specified. The global address of 2936 * the start of the range is returned in addr. 2937 */ 2938 static int 2939 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 2940 uint32_t *addr) 2941 { 2942 uint32_t em, addr_len, maddr; 2943 2944 /* Memory can only be accessed in naturally aligned 4 byte units */ 2945 if (off & 3 || len & 3 || len == 0) 2946 return (EINVAL); 2947 2948 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 2949 switch (fwmtype_to_hwmtype(mtype)) { 2950 case MEM_EDC0: 2951 if (!(em & F_EDRAM0_ENABLE)) 2952 return (EINVAL); 2953 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 2954 maddr = G_EDRAM0_BASE(addr_len) << 20; 2955 break; 2956 case MEM_EDC1: 2957 if (!(em & F_EDRAM1_ENABLE)) 2958 return (EINVAL); 2959 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 2960 maddr = G_EDRAM1_BASE(addr_len) << 20; 2961 break; 2962 case MEM_MC: 2963 if (!(em & F_EXT_MEM_ENABLE)) 2964 return (EINVAL); 2965 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 2966 maddr = G_EXT_MEM_BASE(addr_len) << 20; 2967 break; 2968 case MEM_MC1: 2969 if (!is_t5(sc) || !(em & F_EXT_MEM1_ENABLE)) 2970 return (EINVAL); 2971 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 2972 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 2973 break; 2974 default: 2975 return (EINVAL); 2976 } 2977 2978 *addr = maddr + off; /* global address */ 2979 return (validate_mem_range(sc, *addr, len)); 2980 } 2981 2982 static int 2983 fixup_devlog_params(struct adapter *sc) 2984 { 2985 struct devlog_params *dparams = &sc->params.devlog; 2986 int rc; 2987 2988 rc = validate_mt_off_len(sc, dparams->memtype, dparams->start, 2989 dparams->size, &dparams->addr); 2990 2991 return (rc); 2992 } 2993 2994 static void 2995 update_nirq(struct intrs_and_queues *iaq, int nports) 2996 { 2997 int extra = T4_EXTRA_INTR; 2998 2999 iaq->nirq = extra; 3000 iaq->nirq += nports * (iaq->nrxq + iaq->nofldrxq); 3001 iaq->nirq += nports * (iaq->num_vis - 1) * 3002 max(iaq->nrxq_vi, iaq->nnmrxq_vi); 3003 iaq->nirq += nports * (iaq->num_vis - 1) * iaq->nofldrxq_vi; 3004 } 3005 3006 /* 3007 * Adjust requirements to fit the number of interrupts available. 3008 */ 3009 static void 3010 calculate_iaq(struct adapter *sc, struct intrs_and_queues *iaq, int itype, 3011 int navail) 3012 { 3013 int old_nirq; 3014 const int nports = sc->params.nports; 3015 3016 MPASS(nports > 0); 3017 MPASS(navail > 0); 3018 3019 bzero(iaq, sizeof(*iaq)); 3020 iaq->intr_type = itype; 3021 iaq->num_vis = t4_num_vis; 3022 iaq->ntxq = t4_ntxq; 3023 iaq->ntxq_vi = t4_ntxq_vi; 3024 iaq->nrxq = t4_nrxq; 3025 iaq->nrxq_vi = t4_nrxq_vi; 3026 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 3027 if (is_offload(sc) || is_ethoffload(sc)) { 3028 iaq->nofldtxq = t4_nofldtxq; 3029 iaq->nofldtxq_vi = t4_nofldtxq_vi; 3030 } 3031 #endif 3032 #ifdef TCP_OFFLOAD 3033 if (is_offload(sc)) { 3034 iaq->nofldrxq = t4_nofldrxq; 3035 iaq->nofldrxq_vi = t4_nofldrxq_vi; 3036 } 3037 #endif 3038 #ifdef DEV_NETMAP 3039 iaq->nnmtxq_vi = t4_nnmtxq_vi; 3040 iaq->nnmrxq_vi = t4_nnmrxq_vi; 3041 #endif 3042 3043 update_nirq(iaq, nports); 3044 if (iaq->nirq <= navail && 3045 (itype != INTR_MSI || powerof2(iaq->nirq))) { 3046 /* 3047 * This is the normal case -- there are enough interrupts for 3048 * everything. 3049 */ 3050 goto done; 3051 } 3052 3053 /* 3054 * If extra VIs have been configured try reducing their count and see if 3055 * that works. 3056 */ 3057 while (iaq->num_vis > 1) { 3058 iaq->num_vis--; 3059 update_nirq(iaq, nports); 3060 if (iaq->nirq <= navail && 3061 (itype != INTR_MSI || powerof2(iaq->nirq))) { 3062 device_printf(sc->dev, "virtual interfaces per port " 3063 "reduced to %d from %d. nrxq=%u, nofldrxq=%u, " 3064 "nrxq_vi=%u nofldrxq_vi=%u, nnmrxq_vi=%u. " 3065 "itype %d, navail %u, nirq %d.\n", 3066 iaq->num_vis, t4_num_vis, iaq->nrxq, iaq->nofldrxq, 3067 iaq->nrxq_vi, iaq->nofldrxq_vi, iaq->nnmrxq_vi, 3068 itype, navail, iaq->nirq); 3069 goto done; 3070 } 3071 } 3072 3073 /* 3074 * Extra VIs will not be created. Log a message if they were requested. 3075 */ 3076 MPASS(iaq->num_vis == 1); 3077 iaq->ntxq_vi = iaq->nrxq_vi = 0; 3078 iaq->nofldtxq_vi = iaq->nofldrxq_vi = 0; 3079 iaq->nnmtxq_vi = iaq->nnmrxq_vi = 0; 3080 if (iaq->num_vis != t4_num_vis) { 3081 device_printf(sc->dev, "extra virtual interfaces disabled. " 3082 "nrxq=%u, nofldrxq=%u, nrxq_vi=%u nofldrxq_vi=%u, " 3083 "nnmrxq_vi=%u. itype %d, navail %u, nirq %d.\n", 3084 iaq->nrxq, iaq->nofldrxq, iaq->nrxq_vi, iaq->nofldrxq_vi, 3085 iaq->nnmrxq_vi, itype, navail, iaq->nirq); 3086 } 3087 3088 /* 3089 * Keep reducing the number of NIC rx queues to the next lower power of 3090 * 2 (for even RSS distribution) and halving the TOE rx queues and see 3091 * if that works. 3092 */ 3093 do { 3094 if (iaq->nrxq > 1) { 3095 do { 3096 iaq->nrxq--; 3097 } while (!powerof2(iaq->nrxq)); 3098 } 3099 if (iaq->nofldrxq > 1) 3100 iaq->nofldrxq >>= 1; 3101 3102 old_nirq = iaq->nirq; 3103 update_nirq(iaq, nports); 3104 if (iaq->nirq <= navail && 3105 (itype != INTR_MSI || powerof2(iaq->nirq))) { 3106 device_printf(sc->dev, "running with reduced number of " 3107 "rx queues because of shortage of interrupts. " 3108 "nrxq=%u, nofldrxq=%u. " 3109 "itype %d, navail %u, nirq %d.\n", iaq->nrxq, 3110 iaq->nofldrxq, itype, navail, iaq->nirq); 3111 goto done; 3112 } 3113 } while (old_nirq != iaq->nirq); 3114 3115 /* One interrupt for everything. Ugh. */ 3116 device_printf(sc->dev, "running with minimal number of queues. " 3117 "itype %d, navail %u.\n", itype, navail); 3118 iaq->nirq = 1; 3119 MPASS(iaq->nrxq == 1); 3120 iaq->ntxq = 1; 3121 if (iaq->nofldrxq > 1) 3122 iaq->nofldtxq = 1; 3123 done: 3124 MPASS(iaq->num_vis > 0); 3125 if (iaq->num_vis > 1) { 3126 MPASS(iaq->nrxq_vi > 0); 3127 MPASS(iaq->ntxq_vi > 0); 3128 } 3129 MPASS(iaq->nirq > 0); 3130 MPASS(iaq->nrxq > 0); 3131 MPASS(iaq->ntxq > 0); 3132 if (itype == INTR_MSI) { 3133 MPASS(powerof2(iaq->nirq)); 3134 } 3135 } 3136 3137 static int 3138 cfg_itype_and_nqueues(struct adapter *sc, struct intrs_and_queues *iaq) 3139 { 3140 int rc, itype, navail, nalloc; 3141 3142 for (itype = INTR_MSIX; itype; itype >>= 1) { 3143 3144 if ((itype & t4_intr_types) == 0) 3145 continue; /* not allowed */ 3146 3147 if (itype == INTR_MSIX) 3148 navail = pci_msix_count(sc->dev); 3149 else if (itype == INTR_MSI) 3150 navail = pci_msi_count(sc->dev); 3151 else 3152 navail = 1; 3153 restart: 3154 if (navail == 0) 3155 continue; 3156 3157 calculate_iaq(sc, iaq, itype, navail); 3158 nalloc = iaq->nirq; 3159 rc = 0; 3160 if (itype == INTR_MSIX) 3161 rc = pci_alloc_msix(sc->dev, &nalloc); 3162 else if (itype == INTR_MSI) 3163 rc = pci_alloc_msi(sc->dev, &nalloc); 3164 3165 if (rc == 0 && nalloc > 0) { 3166 if (nalloc == iaq->nirq) 3167 return (0); 3168 3169 /* 3170 * Didn't get the number requested. Use whatever number 3171 * the kernel is willing to allocate. 3172 */ 3173 device_printf(sc->dev, "fewer vectors than requested, " 3174 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 3175 itype, iaq->nirq, nalloc); 3176 pci_release_msi(sc->dev); 3177 navail = nalloc; 3178 goto restart; 3179 } 3180 3181 device_printf(sc->dev, 3182 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 3183 itype, rc, iaq->nirq, nalloc); 3184 } 3185 3186 device_printf(sc->dev, 3187 "failed to find a usable interrupt type. " 3188 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 3189 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 3190 3191 return (ENXIO); 3192 } 3193 3194 #define FW_VERSION(chip) ( \ 3195 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ 3196 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ 3197 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ 3198 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) 3199 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) 3200 3201 struct fw_info { 3202 uint8_t chip; 3203 char *kld_name; 3204 char *fw_mod_name; 3205 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 3206 } fw_info[] = { 3207 { 3208 .chip = CHELSIO_T4, 3209 .kld_name = "t4fw_cfg", 3210 .fw_mod_name = "t4fw", 3211 .fw_hdr = { 3212 .chip = FW_HDR_CHIP_T4, 3213 .fw_ver = htobe32(FW_VERSION(T4)), 3214 .intfver_nic = FW_INTFVER(T4, NIC), 3215 .intfver_vnic = FW_INTFVER(T4, VNIC), 3216 .intfver_ofld = FW_INTFVER(T4, OFLD), 3217 .intfver_ri = FW_INTFVER(T4, RI), 3218 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 3219 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 3220 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 3221 .intfver_fcoe = FW_INTFVER(T4, FCOE), 3222 }, 3223 }, { 3224 .chip = CHELSIO_T5, 3225 .kld_name = "t5fw_cfg", 3226 .fw_mod_name = "t5fw", 3227 .fw_hdr = { 3228 .chip = FW_HDR_CHIP_T5, 3229 .fw_ver = htobe32(FW_VERSION(T5)), 3230 .intfver_nic = FW_INTFVER(T5, NIC), 3231 .intfver_vnic = FW_INTFVER(T5, VNIC), 3232 .intfver_ofld = FW_INTFVER(T5, OFLD), 3233 .intfver_ri = FW_INTFVER(T5, RI), 3234 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 3235 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 3236 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 3237 .intfver_fcoe = FW_INTFVER(T5, FCOE), 3238 }, 3239 }, { 3240 .chip = CHELSIO_T6, 3241 .kld_name = "t6fw_cfg", 3242 .fw_mod_name = "t6fw", 3243 .fw_hdr = { 3244 .chip = FW_HDR_CHIP_T6, 3245 .fw_ver = htobe32(FW_VERSION(T6)), 3246 .intfver_nic = FW_INTFVER(T6, NIC), 3247 .intfver_vnic = FW_INTFVER(T6, VNIC), 3248 .intfver_ofld = FW_INTFVER(T6, OFLD), 3249 .intfver_ri = FW_INTFVER(T6, RI), 3250 .intfver_iscsipdu = FW_INTFVER(T6, ISCSIPDU), 3251 .intfver_iscsi = FW_INTFVER(T6, ISCSI), 3252 .intfver_fcoepdu = FW_INTFVER(T6, FCOEPDU), 3253 .intfver_fcoe = FW_INTFVER(T6, FCOE), 3254 }, 3255 } 3256 }; 3257 3258 static struct fw_info * 3259 find_fw_info(int chip) 3260 { 3261 int i; 3262 3263 for (i = 0; i < nitems(fw_info); i++) { 3264 if (fw_info[i].chip == chip) 3265 return (&fw_info[i]); 3266 } 3267 return (NULL); 3268 } 3269 3270 /* 3271 * Is the given firmware API compatible with the one the driver was compiled 3272 * with? 3273 */ 3274 static int 3275 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 3276 { 3277 3278 /* short circuit if it's the exact same firmware version */ 3279 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 3280 return (1); 3281 3282 /* 3283 * XXX: Is this too conservative? Perhaps I should limit this to the 3284 * features that are supported in the driver. 3285 */ 3286 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 3287 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 3288 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 3289 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 3290 return (1); 3291 #undef SAME_INTF 3292 3293 return (0); 3294 } 3295 3296 /* 3297 * The firmware in the KLD is usable, but should it be installed? This routine 3298 * explains itself in detail if it indicates the KLD firmware should be 3299 * installed. 3300 */ 3301 static int 3302 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 3303 { 3304 const char *reason; 3305 3306 if (!card_fw_usable) { 3307 reason = "incompatible or unusable"; 3308 goto install; 3309 } 3310 3311 if (k > c) { 3312 reason = "older than the version bundled with this driver"; 3313 goto install; 3314 } 3315 3316 if (t4_fw_install == 2 && k != c) { 3317 reason = "different than the version bundled with this driver"; 3318 goto install; 3319 } 3320 3321 return (0); 3322 3323 install: 3324 if (t4_fw_install == 0) { 3325 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 3326 "but the driver is prohibited from installing a different " 3327 "firmware on the card.\n", 3328 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 3329 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 3330 3331 return (0); 3332 } 3333 3334 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 3335 "installing firmware %u.%u.%u.%u on card.\n", 3336 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 3337 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 3338 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 3339 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 3340 3341 return (1); 3342 } 3343 3344 /* 3345 * Establish contact with the firmware and determine if we are the master driver 3346 * or not, and whether we are responsible for chip initialization. 3347 */ 3348 static int 3349 prep_firmware(struct adapter *sc) 3350 { 3351 const struct firmware *fw = NULL, *default_cfg; 3352 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 3353 enum dev_state state; 3354 struct fw_info *fw_info; 3355 struct fw_hdr *card_fw; /* fw on the card */ 3356 const struct fw_hdr *kld_fw; /* fw in the KLD */ 3357 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 3358 against */ 3359 3360 /* This is the firmware whose headers the driver was compiled against */ 3361 fw_info = find_fw_info(chip_id(sc)); 3362 if (fw_info == NULL) { 3363 device_printf(sc->dev, 3364 "unable to look up firmware information for chip %d.\n", 3365 chip_id(sc)); 3366 return (EINVAL); 3367 } 3368 drv_fw = &fw_info->fw_hdr; 3369 3370 /* 3371 * The firmware KLD contains many modules. The KLD name is also the 3372 * name of the module that contains the default config file. 3373 */ 3374 default_cfg = firmware_get(fw_info->kld_name); 3375 3376 /* This is the firmware in the KLD */ 3377 fw = firmware_get(fw_info->fw_mod_name); 3378 if (fw != NULL) { 3379 kld_fw = (const void *)fw->data; 3380 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 3381 } else { 3382 kld_fw = NULL; 3383 kld_fw_usable = 0; 3384 } 3385 3386 /* Read the header of the firmware on the card */ 3387 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 3388 rc = -t4_read_flash(sc, FLASH_FW_START, 3389 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 3390 if (rc == 0) { 3391 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 3392 if (card_fw->fw_ver == be32toh(0xffffffff)) { 3393 uint32_t d = be32toh(kld_fw->fw_ver); 3394 3395 if (!kld_fw_usable) { 3396 device_printf(sc->dev, 3397 "no firmware on the card and no usable " 3398 "firmware bundled with the driver.\n"); 3399 rc = EIO; 3400 goto done; 3401 } else if (t4_fw_install == 0) { 3402 device_printf(sc->dev, 3403 "no firmware on the card and the driver " 3404 "is prohibited from installing new " 3405 "firmware.\n"); 3406 rc = EIO; 3407 goto done; 3408 } 3409 3410 device_printf(sc->dev, "no firmware on the card, " 3411 "installing firmware %d.%d.%d.%d\n", 3412 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 3413 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d)); 3414 rc = t4_fw_forceinstall(sc, fw->data, fw->datasize); 3415 if (rc < 0) { 3416 rc = -rc; 3417 device_printf(sc->dev, 3418 "firmware install failed: %d.\n", rc); 3419 goto done; 3420 } 3421 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 3422 card_fw_usable = 1; 3423 need_fw_reset = 0; 3424 } 3425 } else { 3426 device_printf(sc->dev, 3427 "Unable to read card's firmware header: %d\n", rc); 3428 card_fw_usable = 0; 3429 } 3430 3431 /* Contact firmware. */ 3432 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 3433 if (rc < 0 || state == DEV_STATE_ERR) { 3434 rc = -rc; 3435 device_printf(sc->dev, 3436 "failed to connect to the firmware: %d, %d.\n", rc, state); 3437 goto done; 3438 } 3439 pf = rc; 3440 if (pf == sc->mbox) 3441 sc->flags |= MASTER_PF; 3442 else if (state == DEV_STATE_UNINIT) { 3443 /* 3444 * We didn't get to be the master so we definitely won't be 3445 * configuring the chip. It's a bug if someone else hasn't 3446 * configured it already. 3447 */ 3448 device_printf(sc->dev, "couldn't be master(%d), " 3449 "device not already initialized either(%d).\n", rc, state); 3450 rc = EPROTO; 3451 goto done; 3452 } 3453 3454 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 3455 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) { 3456 /* 3457 * Common case: the firmware on the card is an exact match and 3458 * the KLD is an exact match too, or the KLD is 3459 * absent/incompatible. Note that t4_fw_install = 2 is ignored 3460 * here -- use cxgbetool loadfw if you want to reinstall the 3461 * same firmware as the one on the card. 3462 */ 3463 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 3464 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 3465 be32toh(card_fw->fw_ver))) { 3466 3467 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 3468 if (rc != 0) { 3469 device_printf(sc->dev, 3470 "failed to install firmware: %d\n", rc); 3471 goto done; 3472 } 3473 3474 /* Installed successfully, update the cached header too. */ 3475 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 3476 card_fw_usable = 1; 3477 need_fw_reset = 0; /* already reset as part of load_fw */ 3478 } 3479 3480 if (!card_fw_usable) { 3481 uint32_t d, c, k; 3482 3483 d = ntohl(drv_fw->fw_ver); 3484 c = ntohl(card_fw->fw_ver); 3485 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 3486 3487 device_printf(sc->dev, "Cannot find a usable firmware: " 3488 "fw_install %d, chip state %d, " 3489 "driver compiled with %d.%d.%d.%d, " 3490 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 3491 t4_fw_install, state, 3492 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 3493 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 3494 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 3495 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 3496 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 3497 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 3498 rc = EINVAL; 3499 goto done; 3500 } 3501 3502 /* Reset device */ 3503 if (need_fw_reset && 3504 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 3505 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 3506 if (rc != ETIMEDOUT && rc != EIO) 3507 t4_fw_bye(sc, sc->mbox); 3508 goto done; 3509 } 3510 sc->flags |= FW_OK; 3511 3512 rc = get_params__pre_init(sc); 3513 if (rc != 0) 3514 goto done; /* error message displayed already */ 3515 3516 /* Partition adapter resources as specified in the config file. */ 3517 if (state == DEV_STATE_UNINIT) { 3518 3519 KASSERT(sc->flags & MASTER_PF, 3520 ("%s: trying to change chip settings when not master.", 3521 __func__)); 3522 3523 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 3524 if (rc != 0) 3525 goto done; /* error message displayed already */ 3526 3527 t4_tweak_chip_settings(sc); 3528 3529 /* get basic stuff going */ 3530 rc = -t4_fw_initialize(sc, sc->mbox); 3531 if (rc != 0) { 3532 device_printf(sc->dev, "fw init failed: %d.\n", rc); 3533 goto done; 3534 } 3535 } else { 3536 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 3537 sc->cfcsum = 0; 3538 } 3539 3540 done: 3541 free(card_fw, M_CXGBE); 3542 if (fw != NULL) 3543 firmware_put(fw, FIRMWARE_UNLOAD); 3544 if (default_cfg != NULL) 3545 firmware_put(default_cfg, FIRMWARE_UNLOAD); 3546 3547 return (rc); 3548 } 3549 3550 #define FW_PARAM_DEV(param) \ 3551 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 3552 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 3553 #define FW_PARAM_PFVF(param) \ 3554 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 3555 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 3556 3557 /* 3558 * Partition chip resources for use between various PFs, VFs, etc. 3559 */ 3560 static int 3561 partition_resources(struct adapter *sc, const struct firmware *default_cfg, 3562 const char *name_prefix) 3563 { 3564 const struct firmware *cfg = NULL; 3565 int rc = 0; 3566 struct fw_caps_config_cmd caps; 3567 uint32_t mtype, moff, finicsum, cfcsum; 3568 3569 /* 3570 * Figure out what configuration file to use. Pick the default config 3571 * file for the card if the user hasn't specified one explicitly. 3572 */ 3573 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 3574 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 3575 /* Card specific overrides go here. */ 3576 if (pci_get_device(sc->dev) == 0x440a) 3577 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 3578 if (is_fpga(sc)) 3579 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 3580 } else if (strncmp(t4_cfg_file, BUILTIN_CF, sizeof(t4_cfg_file)) == 0) 3581 goto use_built_in_config; /* go straight to config. */ 3582 3583 /* 3584 * We need to load another module if the profile is anything except 3585 * "default" or "flash". 3586 */ 3587 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 3588 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 3589 char s[32]; 3590 3591 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 3592 cfg = firmware_get(s); 3593 if (cfg == NULL) { 3594 if (default_cfg != NULL) { 3595 device_printf(sc->dev, 3596 "unable to load module \"%s\" for " 3597 "configuration profile \"%s\", will use " 3598 "the default config file instead.\n", 3599 s, sc->cfg_file); 3600 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 3601 "%s", DEFAULT_CF); 3602 } else { 3603 device_printf(sc->dev, 3604 "unable to load module \"%s\" for " 3605 "configuration profile \"%s\", will use " 3606 "the config file on the card's flash " 3607 "instead.\n", s, sc->cfg_file); 3608 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 3609 "%s", FLASH_CF); 3610 } 3611 } 3612 } 3613 3614 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 3615 default_cfg == NULL) { 3616 device_printf(sc->dev, 3617 "default config file not available, will use the config " 3618 "file on the card's flash instead.\n"); 3619 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 3620 } 3621 3622 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 3623 u_int cflen; 3624 const uint32_t *cfdata; 3625 uint32_t param, val, addr; 3626 3627 KASSERT(cfg != NULL || default_cfg != NULL, 3628 ("%s: no config to upload", __func__)); 3629 3630 /* 3631 * Ask the firmware where it wants us to upload the config file. 3632 */ 3633 param = FW_PARAM_DEV(CF); 3634 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 3635 if (rc != 0) { 3636 /* No support for config file? Shouldn't happen. */ 3637 device_printf(sc->dev, 3638 "failed to query config file location: %d.\n", rc); 3639 goto done; 3640 } 3641 mtype = G_FW_PARAMS_PARAM_Y(val); 3642 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 3643 3644 /* 3645 * XXX: sheer laziness. We deliberately added 4 bytes of 3646 * useless stuffing/comments at the end of the config file so 3647 * it's ok to simply throw away the last remaining bytes when 3648 * the config file is not an exact multiple of 4. This also 3649 * helps with the validate_mt_off_len check. 3650 */ 3651 if (cfg != NULL) { 3652 cflen = cfg->datasize & ~3; 3653 cfdata = cfg->data; 3654 } else { 3655 cflen = default_cfg->datasize & ~3; 3656 cfdata = default_cfg->data; 3657 } 3658 3659 if (cflen > FLASH_CFG_MAX_SIZE) { 3660 device_printf(sc->dev, 3661 "config file too long (%d, max allowed is %d). " 3662 "Will try to use the config on the card, if any.\n", 3663 cflen, FLASH_CFG_MAX_SIZE); 3664 goto use_config_on_flash; 3665 } 3666 3667 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 3668 if (rc != 0) { 3669 device_printf(sc->dev, 3670 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 3671 "Will try to use the config on the card, if any.\n", 3672 __func__, mtype, moff, cflen, rc); 3673 goto use_config_on_flash; 3674 } 3675 write_via_memwin(sc, 2, addr, cfdata, cflen); 3676 } else { 3677 use_config_on_flash: 3678 mtype = FW_MEMTYPE_FLASH; 3679 moff = t4_flash_cfg_addr(sc); 3680 } 3681 3682 bzero(&caps, sizeof(caps)); 3683 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 3684 F_FW_CMD_REQUEST | F_FW_CMD_READ); 3685 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 3686 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 3687 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 3688 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 3689 if (rc != 0) { 3690 device_printf(sc->dev, 3691 "failed to pre-process config file: %d " 3692 "(mtype %d, moff 0x%x). Will reset the firmware and retry " 3693 "with the built-in configuration.\n", rc, mtype, moff); 3694 3695 rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST); 3696 if (rc != 0) { 3697 device_printf(sc->dev, 3698 "firmware reset failed: %d.\n", rc); 3699 if (rc != ETIMEDOUT && rc != EIO) { 3700 t4_fw_bye(sc, sc->mbox); 3701 sc->flags &= ~FW_OK; 3702 } 3703 goto done; 3704 } 3705 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", "built-in"); 3706 use_built_in_config: 3707 bzero(&caps, sizeof(caps)); 3708 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 3709 F_FW_CMD_REQUEST | F_FW_CMD_READ); 3710 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 3711 rc = t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 3712 if (rc != 0) { 3713 device_printf(sc->dev, 3714 "built-in configuration failed: %d.\n", rc); 3715 goto done; 3716 } 3717 } 3718 3719 finicsum = be32toh(caps.finicsum); 3720 cfcsum = be32toh(caps.cfcsum); 3721 if (finicsum != cfcsum) { 3722 device_printf(sc->dev, 3723 "WARNING: config file checksum mismatch: %08x %08x\n", 3724 finicsum, cfcsum); 3725 } 3726 sc->cfcsum = cfcsum; 3727 3728 #define LIMIT_CAPS(x) do { \ 3729 caps.x &= htobe16(t4_##x##_allowed); \ 3730 } while (0) 3731 3732 /* 3733 * Let the firmware know what features will (not) be used so it can tune 3734 * things accordingly. 3735 */ 3736 LIMIT_CAPS(nbmcaps); 3737 LIMIT_CAPS(linkcaps); 3738 LIMIT_CAPS(switchcaps); 3739 LIMIT_CAPS(niccaps); 3740 LIMIT_CAPS(toecaps); 3741 LIMIT_CAPS(rdmacaps); 3742 LIMIT_CAPS(cryptocaps); 3743 LIMIT_CAPS(iscsicaps); 3744 LIMIT_CAPS(fcoecaps); 3745 #undef LIMIT_CAPS 3746 3747 if (caps.niccaps & htobe16(FW_CAPS_CONFIG_NIC_HASHFILTER)) { 3748 /* 3749 * TOE and hashfilters are mutually exclusive. It is a config 3750 * file or firmware bug if both are reported as available. Try 3751 * to cope with the situation in non-debug builds by disabling 3752 * TOE. 3753 */ 3754 MPASS(caps.toecaps == 0); 3755 3756 caps.toecaps = 0; 3757 caps.rdmacaps = 0; 3758 caps.iscsicaps = 0; 3759 } 3760 3761 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 3762 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 3763 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 3764 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 3765 if (rc != 0) { 3766 device_printf(sc->dev, 3767 "failed to process config file: %d.\n", rc); 3768 } 3769 done: 3770 if (cfg != NULL) 3771 firmware_put(cfg, FIRMWARE_UNLOAD); 3772 return (rc); 3773 } 3774 3775 /* 3776 * Retrieve parameters that are needed (or nice to have) very early. 3777 */ 3778 static int 3779 get_params__pre_init(struct adapter *sc) 3780 { 3781 int rc; 3782 uint32_t param[2], val[2]; 3783 3784 t4_get_version_info(sc); 3785 3786 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 3787 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 3788 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 3789 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 3790 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 3791 3792 snprintf(sc->bs_version, sizeof(sc->bs_version), "%u.%u.%u.%u", 3793 G_FW_HDR_FW_VER_MAJOR(sc->params.bs_vers), 3794 G_FW_HDR_FW_VER_MINOR(sc->params.bs_vers), 3795 G_FW_HDR_FW_VER_MICRO(sc->params.bs_vers), 3796 G_FW_HDR_FW_VER_BUILD(sc->params.bs_vers)); 3797 3798 snprintf(sc->tp_version, sizeof(sc->tp_version), "%u.%u.%u.%u", 3799 G_FW_HDR_FW_VER_MAJOR(sc->params.tp_vers), 3800 G_FW_HDR_FW_VER_MINOR(sc->params.tp_vers), 3801 G_FW_HDR_FW_VER_MICRO(sc->params.tp_vers), 3802 G_FW_HDR_FW_VER_BUILD(sc->params.tp_vers)); 3803 3804 snprintf(sc->er_version, sizeof(sc->er_version), "%u.%u.%u.%u", 3805 G_FW_HDR_FW_VER_MAJOR(sc->params.er_vers), 3806 G_FW_HDR_FW_VER_MINOR(sc->params.er_vers), 3807 G_FW_HDR_FW_VER_MICRO(sc->params.er_vers), 3808 G_FW_HDR_FW_VER_BUILD(sc->params.er_vers)); 3809 3810 param[0] = FW_PARAM_DEV(PORTVEC); 3811 param[1] = FW_PARAM_DEV(CCLK); 3812 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 3813 if (rc != 0) { 3814 device_printf(sc->dev, 3815 "failed to query parameters (pre_init): %d.\n", rc); 3816 return (rc); 3817 } 3818 3819 sc->params.portvec = val[0]; 3820 sc->params.nports = bitcount32(val[0]); 3821 sc->params.vpd.cclk = val[1]; 3822 3823 /* Read device log parameters. */ 3824 rc = -t4_init_devlog_params(sc, 1); 3825 if (rc == 0) 3826 fixup_devlog_params(sc); 3827 else { 3828 device_printf(sc->dev, 3829 "failed to get devlog parameters: %d.\n", rc); 3830 rc = 0; /* devlog isn't critical for device operation */ 3831 } 3832 3833 return (rc); 3834 } 3835 3836 /* 3837 * Retrieve various parameters that are of interest to the driver. The device 3838 * has been initialized by the firmware at this point. 3839 */ 3840 static int 3841 get_params__post_init(struct adapter *sc) 3842 { 3843 int rc; 3844 uint32_t param[7], val[7]; 3845 struct fw_caps_config_cmd caps; 3846 3847 param[0] = FW_PARAM_PFVF(IQFLINT_START); 3848 param[1] = FW_PARAM_PFVF(EQ_START); 3849 param[2] = FW_PARAM_PFVF(FILTER_START); 3850 param[3] = FW_PARAM_PFVF(FILTER_END); 3851 param[4] = FW_PARAM_PFVF(L2T_START); 3852 param[5] = FW_PARAM_PFVF(L2T_END); 3853 param[6] = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 3854 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 3855 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_VDD); 3856 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 7, param, val); 3857 if (rc != 0) { 3858 device_printf(sc->dev, 3859 "failed to query parameters (post_init): %d.\n", rc); 3860 return (rc); 3861 } 3862 3863 sc->sge.iq_start = val[0]; 3864 sc->sge.eq_start = val[1]; 3865 if ((int)val[3] > (int)val[2]) { 3866 sc->tids.ftid_base = val[2]; 3867 sc->tids.ftid_end = val[3]; 3868 sc->tids.nftids = val[3] - val[2] + 1; 3869 } 3870 sc->vres.l2t.start = val[4]; 3871 sc->vres.l2t.size = val[5] - val[4] + 1; 3872 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 3873 ("%s: L2 table size (%u) larger than expected (%u)", 3874 __func__, sc->vres.l2t.size, L2T_SIZE)); 3875 sc->params.core_vdd = val[6]; 3876 3877 if (chip_id(sc) >= CHELSIO_T6) { 3878 3879 #ifdef INVARIANTS 3880 if (sc->params.fw_vers >= 3881 (V_FW_HDR_FW_VER_MAJOR(1) | V_FW_HDR_FW_VER_MINOR(20) | 3882 V_FW_HDR_FW_VER_MICRO(1) | V_FW_HDR_FW_VER_BUILD(0))) { 3883 /* 3884 * Note that the code to enable the region should run 3885 * before t4_fw_initialize and not here. This is just a 3886 * reminder to add said code. 3887 */ 3888 device_printf(sc->dev, 3889 "hpfilter region not enabled.\n"); 3890 } 3891 #endif 3892 3893 sc->tids.tid_base = t4_read_reg(sc, 3894 A_LE_DB_ACTIVE_TABLE_START_INDEX); 3895 3896 param[0] = FW_PARAM_PFVF(HPFILTER_START); 3897 param[1] = FW_PARAM_PFVF(HPFILTER_END); 3898 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 3899 if (rc != 0) { 3900 device_printf(sc->dev, 3901 "failed to query hpfilter parameters: %d.\n", rc); 3902 return (rc); 3903 } 3904 if ((int)val[1] > (int)val[0]) { 3905 sc->tids.hpftid_base = val[0]; 3906 sc->tids.hpftid_end = val[1]; 3907 sc->tids.nhpftids = val[1] - val[0] + 1; 3908 3909 /* 3910 * These should go off if the layout changes and the 3911 * driver needs to catch up. 3912 */ 3913 MPASS(sc->tids.hpftid_base == 0); 3914 MPASS(sc->tids.tid_base == sc->tids.nhpftids); 3915 } 3916 } 3917 3918 /* 3919 * MPSBGMAP is queried separately because only recent firmwares support 3920 * it as a parameter and we don't want the compound query above to fail 3921 * on older firmwares. 3922 */ 3923 param[0] = FW_PARAM_DEV(MPSBGMAP); 3924 val[0] = 0; 3925 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val); 3926 if (rc == 0) 3927 sc->params.mps_bg_map = val[0]; 3928 else 3929 sc->params.mps_bg_map = 0; 3930 3931 /* 3932 * Determine whether the firmware supports the filter2 work request. 3933 * This is queried separately for the same reason as MPSBGMAP above. 3934 */ 3935 param[0] = FW_PARAM_DEV(FILTER2_WR); 3936 val[0] = 0; 3937 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val); 3938 if (rc == 0) 3939 sc->params.filter2_wr_support = val[0] != 0; 3940 else 3941 sc->params.filter2_wr_support = 0; 3942 3943 /* get capabilites */ 3944 bzero(&caps, sizeof(caps)); 3945 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 3946 F_FW_CMD_REQUEST | F_FW_CMD_READ); 3947 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 3948 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 3949 if (rc != 0) { 3950 device_printf(sc->dev, 3951 "failed to get card capabilities: %d.\n", rc); 3952 return (rc); 3953 } 3954 3955 #define READ_CAPS(x) do { \ 3956 sc->x = htobe16(caps.x); \ 3957 } while (0) 3958 READ_CAPS(nbmcaps); 3959 READ_CAPS(linkcaps); 3960 READ_CAPS(switchcaps); 3961 READ_CAPS(niccaps); 3962 READ_CAPS(toecaps); 3963 READ_CAPS(rdmacaps); 3964 READ_CAPS(cryptocaps); 3965 READ_CAPS(iscsicaps); 3966 READ_CAPS(fcoecaps); 3967 3968 if (sc->niccaps & FW_CAPS_CONFIG_NIC_HASHFILTER) { 3969 MPASS(chip_id(sc) > CHELSIO_T4); 3970 MPASS(sc->toecaps == 0); 3971 sc->toecaps = 0; 3972 3973 param[0] = FW_PARAM_DEV(NTID); 3974 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val); 3975 if (rc != 0) { 3976 device_printf(sc->dev, 3977 "failed to query HASHFILTER parameters: %d.\n", rc); 3978 return (rc); 3979 } 3980 sc->tids.ntids = val[0]; 3981 if (sc->params.fw_vers < 3982 (V_FW_HDR_FW_VER_MAJOR(1) | V_FW_HDR_FW_VER_MINOR(20) | 3983 V_FW_HDR_FW_VER_MICRO(5) | V_FW_HDR_FW_VER_BUILD(0))) { 3984 MPASS(sc->tids.ntids >= sc->tids.nhpftids); 3985 sc->tids.ntids -= sc->tids.nhpftids; 3986 } 3987 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 3988 sc->params.hash_filter = 1; 3989 } 3990 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { 3991 param[0] = FW_PARAM_PFVF(ETHOFLD_START); 3992 param[1] = FW_PARAM_PFVF(ETHOFLD_END); 3993 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 3994 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); 3995 if (rc != 0) { 3996 device_printf(sc->dev, 3997 "failed to query NIC parameters: %d.\n", rc); 3998 return (rc); 3999 } 4000 if ((int)val[1] > (int)val[0]) { 4001 sc->tids.etid_base = val[0]; 4002 sc->tids.etid_end = val[1]; 4003 sc->tids.netids = val[1] - val[0] + 1; 4004 sc->params.eo_wr_cred = val[2]; 4005 sc->params.ethoffload = 1; 4006 } 4007 } 4008 if (sc->toecaps) { 4009 /* query offload-related parameters */ 4010 param[0] = FW_PARAM_DEV(NTID); 4011 param[1] = FW_PARAM_PFVF(SERVER_START); 4012 param[2] = FW_PARAM_PFVF(SERVER_END); 4013 param[3] = FW_PARAM_PFVF(TDDP_START); 4014 param[4] = FW_PARAM_PFVF(TDDP_END); 4015 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 4016 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 4017 if (rc != 0) { 4018 device_printf(sc->dev, 4019 "failed to query TOE parameters: %d.\n", rc); 4020 return (rc); 4021 } 4022 sc->tids.ntids = val[0]; 4023 if (sc->params.fw_vers < 4024 (V_FW_HDR_FW_VER_MAJOR(1) | V_FW_HDR_FW_VER_MINOR(20) | 4025 V_FW_HDR_FW_VER_MICRO(5) | V_FW_HDR_FW_VER_BUILD(0))) { 4026 MPASS(sc->tids.ntids >= sc->tids.nhpftids); 4027 sc->tids.ntids -= sc->tids.nhpftids; 4028 } 4029 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 4030 if ((int)val[2] > (int)val[1]) { 4031 sc->tids.stid_base = val[1]; 4032 sc->tids.nstids = val[2] - val[1] + 1; 4033 } 4034 sc->vres.ddp.start = val[3]; 4035 sc->vres.ddp.size = val[4] - val[3] + 1; 4036 sc->params.ofldq_wr_cred = val[5]; 4037 sc->params.offload = 1; 4038 } else { 4039 /* 4040 * The firmware attempts memfree TOE configuration for -SO cards 4041 * and will report toecaps=0 if it runs out of resources (this 4042 * depends on the config file). It may not report 0 for other 4043 * capabilities dependent on the TOE in this case. Set them to 4044 * 0 here so that the driver doesn't bother tracking resources 4045 * that will never be used. 4046 */ 4047 sc->iscsicaps = 0; 4048 sc->rdmacaps = 0; 4049 } 4050 if (sc->rdmacaps) { 4051 param[0] = FW_PARAM_PFVF(STAG_START); 4052 param[1] = FW_PARAM_PFVF(STAG_END); 4053 param[2] = FW_PARAM_PFVF(RQ_START); 4054 param[3] = FW_PARAM_PFVF(RQ_END); 4055 param[4] = FW_PARAM_PFVF(PBL_START); 4056 param[5] = FW_PARAM_PFVF(PBL_END); 4057 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 4058 if (rc != 0) { 4059 device_printf(sc->dev, 4060 "failed to query RDMA parameters(1): %d.\n", rc); 4061 return (rc); 4062 } 4063 sc->vres.stag.start = val[0]; 4064 sc->vres.stag.size = val[1] - val[0] + 1; 4065 sc->vres.rq.start = val[2]; 4066 sc->vres.rq.size = val[3] - val[2] + 1; 4067 sc->vres.pbl.start = val[4]; 4068 sc->vres.pbl.size = val[5] - val[4] + 1; 4069 4070 param[0] = FW_PARAM_PFVF(SQRQ_START); 4071 param[1] = FW_PARAM_PFVF(SQRQ_END); 4072 param[2] = FW_PARAM_PFVF(CQ_START); 4073 param[3] = FW_PARAM_PFVF(CQ_END); 4074 param[4] = FW_PARAM_PFVF(OCQ_START); 4075 param[5] = FW_PARAM_PFVF(OCQ_END); 4076 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 4077 if (rc != 0) { 4078 device_printf(sc->dev, 4079 "failed to query RDMA parameters(2): %d.\n", rc); 4080 return (rc); 4081 } 4082 sc->vres.qp.start = val[0]; 4083 sc->vres.qp.size = val[1] - val[0] + 1; 4084 sc->vres.cq.start = val[2]; 4085 sc->vres.cq.size = val[3] - val[2] + 1; 4086 sc->vres.ocq.start = val[4]; 4087 sc->vres.ocq.size = val[5] - val[4] + 1; 4088 4089 param[0] = FW_PARAM_PFVF(SRQ_START); 4090 param[1] = FW_PARAM_PFVF(SRQ_END); 4091 param[2] = FW_PARAM_DEV(MAXORDIRD_QP); 4092 param[3] = FW_PARAM_DEV(MAXIRD_ADAPTER); 4093 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 4, param, val); 4094 if (rc != 0) { 4095 device_printf(sc->dev, 4096 "failed to query RDMA parameters(3): %d.\n", rc); 4097 return (rc); 4098 } 4099 sc->vres.srq.start = val[0]; 4100 sc->vres.srq.size = val[1] - val[0] + 1; 4101 sc->params.max_ordird_qp = val[2]; 4102 sc->params.max_ird_adapter = val[3]; 4103 } 4104 if (sc->iscsicaps) { 4105 param[0] = FW_PARAM_PFVF(ISCSI_START); 4106 param[1] = FW_PARAM_PFVF(ISCSI_END); 4107 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 4108 if (rc != 0) { 4109 device_printf(sc->dev, 4110 "failed to query iSCSI parameters: %d.\n", rc); 4111 return (rc); 4112 } 4113 sc->vres.iscsi.start = val[0]; 4114 sc->vres.iscsi.size = val[1] - val[0] + 1; 4115 } 4116 if (sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS) { 4117 param[0] = FW_PARAM_PFVF(TLS_START); 4118 param[1] = FW_PARAM_PFVF(TLS_END); 4119 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 4120 if (rc != 0) { 4121 device_printf(sc->dev, 4122 "failed to query TLS parameters: %d.\n", rc); 4123 return (rc); 4124 } 4125 sc->vres.key.start = val[0]; 4126 sc->vres.key.size = val[1] - val[0] + 1; 4127 } 4128 4129 t4_init_sge_params(sc); 4130 4131 /* 4132 * We've got the params we wanted to query via the firmware. Now grab 4133 * some others directly from the chip. 4134 */ 4135 rc = t4_read_chip_settings(sc); 4136 4137 return (rc); 4138 } 4139 4140 static int 4141 set_params__post_init(struct adapter *sc) 4142 { 4143 uint32_t param, val; 4144 #ifdef TCP_OFFLOAD 4145 int i, v, shift; 4146 #endif 4147 4148 /* ask for encapsulated CPLs */ 4149 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 4150 val = 1; 4151 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 4152 4153 /* Enable 32b port caps if the firmware supports it. */ 4154 param = FW_PARAM_PFVF(PORT_CAPS32); 4155 val = 1; 4156 if (t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val) == 0) 4157 sc->params.port_caps32 = 1; 4158 4159 /* Let filter + maskhash steer to a part of the VI's RSS region. */ 4160 val = 1 << (G_MASKSIZE(t4_read_reg(sc, A_TP_RSS_CONFIG_TNL)) - 1); 4161 t4_set_reg_field(sc, A_TP_RSS_CONFIG_TNL, V_MASKFILTER(M_MASKFILTER), 4162 V_MASKFILTER(val - 1)); 4163 4164 #ifdef TCP_OFFLOAD 4165 /* 4166 * Override the TOE timers with user provided tunables. This is not the 4167 * recommended way to change the timers (the firmware config file is) so 4168 * these tunables are not documented. 4169 * 4170 * All the timer tunables are in microseconds. 4171 */ 4172 if (t4_toe_keepalive_idle != 0) { 4173 v = us_to_tcp_ticks(sc, t4_toe_keepalive_idle); 4174 v &= M_KEEPALIVEIDLE; 4175 t4_set_reg_field(sc, A_TP_KEEP_IDLE, 4176 V_KEEPALIVEIDLE(M_KEEPALIVEIDLE), V_KEEPALIVEIDLE(v)); 4177 } 4178 if (t4_toe_keepalive_interval != 0) { 4179 v = us_to_tcp_ticks(sc, t4_toe_keepalive_interval); 4180 v &= M_KEEPALIVEINTVL; 4181 t4_set_reg_field(sc, A_TP_KEEP_INTVL, 4182 V_KEEPALIVEINTVL(M_KEEPALIVEINTVL), V_KEEPALIVEINTVL(v)); 4183 } 4184 if (t4_toe_keepalive_count != 0) { 4185 v = t4_toe_keepalive_count & M_KEEPALIVEMAXR2; 4186 t4_set_reg_field(sc, A_TP_SHIFT_CNT, 4187 V_KEEPALIVEMAXR1(M_KEEPALIVEMAXR1) | 4188 V_KEEPALIVEMAXR2(M_KEEPALIVEMAXR2), 4189 V_KEEPALIVEMAXR1(1) | V_KEEPALIVEMAXR2(v)); 4190 } 4191 if (t4_toe_rexmt_min != 0) { 4192 v = us_to_tcp_ticks(sc, t4_toe_rexmt_min); 4193 v &= M_RXTMIN; 4194 t4_set_reg_field(sc, A_TP_RXT_MIN, 4195 V_RXTMIN(M_RXTMIN), V_RXTMIN(v)); 4196 } 4197 if (t4_toe_rexmt_max != 0) { 4198 v = us_to_tcp_ticks(sc, t4_toe_rexmt_max); 4199 v &= M_RXTMAX; 4200 t4_set_reg_field(sc, A_TP_RXT_MAX, 4201 V_RXTMAX(M_RXTMAX), V_RXTMAX(v)); 4202 } 4203 if (t4_toe_rexmt_count != 0) { 4204 v = t4_toe_rexmt_count & M_RXTSHIFTMAXR2; 4205 t4_set_reg_field(sc, A_TP_SHIFT_CNT, 4206 V_RXTSHIFTMAXR1(M_RXTSHIFTMAXR1) | 4207 V_RXTSHIFTMAXR2(M_RXTSHIFTMAXR2), 4208 V_RXTSHIFTMAXR1(1) | V_RXTSHIFTMAXR2(v)); 4209 } 4210 for (i = 0; i < nitems(t4_toe_rexmt_backoff); i++) { 4211 if (t4_toe_rexmt_backoff[i] != -1) { 4212 v = t4_toe_rexmt_backoff[i] & M_TIMERBACKOFFINDEX0; 4213 shift = (i & 3) << 3; 4214 t4_set_reg_field(sc, A_TP_TCP_BACKOFF_REG0 + (i & ~3), 4215 M_TIMERBACKOFFINDEX0 << shift, v << shift); 4216 } 4217 } 4218 #endif 4219 return (0); 4220 } 4221 4222 #undef FW_PARAM_PFVF 4223 #undef FW_PARAM_DEV 4224 4225 static void 4226 t4_set_desc(struct adapter *sc) 4227 { 4228 char buf[128]; 4229 struct adapter_params *p = &sc->params; 4230 4231 snprintf(buf, sizeof(buf), "Chelsio %s", p->vpd.id); 4232 4233 device_set_desc_copy(sc->dev, buf); 4234 } 4235 4236 static inline void 4237 ifmedia_add4(struct ifmedia *ifm, int m) 4238 { 4239 4240 ifmedia_add(ifm, m, 0, NULL); 4241 ifmedia_add(ifm, m | IFM_ETH_TXPAUSE, 0, NULL); 4242 ifmedia_add(ifm, m | IFM_ETH_RXPAUSE, 0, NULL); 4243 ifmedia_add(ifm, m | IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE, 0, NULL); 4244 } 4245 4246 /* 4247 * This is the selected media, which is not quite the same as the active media. 4248 * The media line in ifconfig is "media: Ethernet selected (active)" if selected 4249 * and active are not the same, and "media: Ethernet selected" otherwise. 4250 */ 4251 static void 4252 set_current_media(struct port_info *pi) 4253 { 4254 struct link_config *lc; 4255 struct ifmedia *ifm; 4256 int mword; 4257 u_int speed; 4258 4259 PORT_LOCK_ASSERT_OWNED(pi); 4260 4261 /* Leave current media alone if it's already set to IFM_NONE. */ 4262 ifm = &pi->media; 4263 if (ifm->ifm_cur != NULL && 4264 IFM_SUBTYPE(ifm->ifm_cur->ifm_media) == IFM_NONE) 4265 return; 4266 4267 lc = &pi->link_cfg; 4268 if (lc->requested_aneg != AUTONEG_DISABLE && 4269 lc->supported & FW_PORT_CAP32_ANEG) { 4270 ifmedia_set(ifm, IFM_ETHER | IFM_AUTO); 4271 return; 4272 } 4273 mword = IFM_ETHER | IFM_FDX; 4274 if (lc->requested_fc & PAUSE_TX) 4275 mword |= IFM_ETH_TXPAUSE; 4276 if (lc->requested_fc & PAUSE_RX) 4277 mword |= IFM_ETH_RXPAUSE; 4278 if (lc->requested_speed == 0) 4279 speed = port_top_speed(pi) * 1000; /* Gbps -> Mbps */ 4280 else 4281 speed = lc->requested_speed; 4282 mword |= port_mword(pi, speed_to_fwcap(speed)); 4283 ifmedia_set(ifm, mword); 4284 } 4285 4286 /* 4287 * Returns true if the ifmedia list for the port cannot change. 4288 */ 4289 static bool 4290 fixed_ifmedia(struct port_info *pi) 4291 { 4292 4293 return (pi->port_type == FW_PORT_TYPE_BT_SGMII || 4294 pi->port_type == FW_PORT_TYPE_BT_XFI || 4295 pi->port_type == FW_PORT_TYPE_BT_XAUI || 4296 pi->port_type == FW_PORT_TYPE_KX4 || 4297 pi->port_type == FW_PORT_TYPE_KX || 4298 pi->port_type == FW_PORT_TYPE_KR || 4299 pi->port_type == FW_PORT_TYPE_BP_AP || 4300 pi->port_type == FW_PORT_TYPE_BP4_AP || 4301 pi->port_type == FW_PORT_TYPE_BP40_BA || 4302 pi->port_type == FW_PORT_TYPE_KR4_100G || 4303 pi->port_type == FW_PORT_TYPE_KR_SFP28 || 4304 pi->port_type == FW_PORT_TYPE_KR_XLAUI); 4305 } 4306 4307 static void 4308 build_medialist(struct port_info *pi) 4309 { 4310 uint32_t ss, speed; 4311 int unknown, mword, bit; 4312 struct link_config *lc; 4313 struct ifmedia *ifm; 4314 4315 PORT_LOCK_ASSERT_OWNED(pi); 4316 4317 if (pi->flags & FIXED_IFMEDIA) 4318 return; 4319 4320 /* 4321 * Rebuild the ifmedia list. 4322 */ 4323 ifm = &pi->media; 4324 ifmedia_removeall(ifm); 4325 lc = &pi->link_cfg; 4326 ss = G_FW_PORT_CAP32_SPEED(lc->supported); /* Supported Speeds */ 4327 if (__predict_false(ss == 0)) { /* not supposed to happen. */ 4328 MPASS(ss != 0); 4329 no_media: 4330 MPASS(LIST_EMPTY(&ifm->ifm_list)); 4331 ifmedia_add(ifm, IFM_ETHER | IFM_NONE, 0, NULL); 4332 ifmedia_set(ifm, IFM_ETHER | IFM_NONE); 4333 return; 4334 } 4335 4336 unknown = 0; 4337 for (bit = S_FW_PORT_CAP32_SPEED; bit < fls(ss); bit++) { 4338 speed = 1 << bit; 4339 MPASS(speed & M_FW_PORT_CAP32_SPEED); 4340 if (ss & speed) { 4341 mword = port_mword(pi, speed); 4342 if (mword == IFM_NONE) { 4343 goto no_media; 4344 } else if (mword == IFM_UNKNOWN) 4345 unknown++; 4346 else 4347 ifmedia_add4(ifm, IFM_ETHER | IFM_FDX | mword); 4348 } 4349 } 4350 if (unknown > 0) /* Add one unknown for all unknown media types. */ 4351 ifmedia_add4(ifm, IFM_ETHER | IFM_FDX | IFM_UNKNOWN); 4352 if (lc->supported & FW_PORT_CAP32_ANEG) 4353 ifmedia_add(ifm, IFM_ETHER | IFM_AUTO, 0, NULL); 4354 4355 set_current_media(pi); 4356 } 4357 4358 /* 4359 * Initialize the requested fields in the link config based on driver tunables. 4360 */ 4361 static void 4362 init_link_config(struct port_info *pi) 4363 { 4364 struct link_config *lc = &pi->link_cfg; 4365 4366 PORT_LOCK_ASSERT_OWNED(pi); 4367 4368 lc->requested_speed = 0; 4369 4370 if (t4_autoneg == 0) 4371 lc->requested_aneg = AUTONEG_DISABLE; 4372 else if (t4_autoneg == 1) 4373 lc->requested_aneg = AUTONEG_ENABLE; 4374 else 4375 lc->requested_aneg = AUTONEG_AUTO; 4376 4377 lc->requested_fc = t4_pause_settings & (PAUSE_TX | PAUSE_RX | 4378 PAUSE_AUTONEG); 4379 4380 if (t4_fec == -1 || t4_fec & FEC_AUTO) 4381 lc->requested_fec = FEC_AUTO; 4382 else { 4383 lc->requested_fec = FEC_NONE; 4384 if (t4_fec & FEC_RS) 4385 lc->requested_fec |= FEC_RS; 4386 if (t4_fec & FEC_BASER_RS) 4387 lc->requested_fec |= FEC_BASER_RS; 4388 } 4389 } 4390 4391 /* 4392 * Makes sure that all requested settings comply with what's supported by the 4393 * port. Returns the number of settings that were invalid and had to be fixed. 4394 */ 4395 static int 4396 fixup_link_config(struct port_info *pi) 4397 { 4398 int n = 0; 4399 struct link_config *lc = &pi->link_cfg; 4400 uint32_t fwspeed; 4401 4402 PORT_LOCK_ASSERT_OWNED(pi); 4403 4404 /* Speed (when not autonegotiating) */ 4405 if (lc->requested_speed != 0) { 4406 fwspeed = speed_to_fwcap(lc->requested_speed); 4407 if ((fwspeed & lc->supported) == 0) { 4408 n++; 4409 lc->requested_speed = 0; 4410 } 4411 } 4412 4413 /* Link autonegotiation */ 4414 MPASS(lc->requested_aneg == AUTONEG_ENABLE || 4415 lc->requested_aneg == AUTONEG_DISABLE || 4416 lc->requested_aneg == AUTONEG_AUTO); 4417 if (lc->requested_aneg == AUTONEG_ENABLE && 4418 !(lc->supported & FW_PORT_CAP32_ANEG)) { 4419 n++; 4420 lc->requested_aneg = AUTONEG_AUTO; 4421 } 4422 4423 /* Flow control */ 4424 MPASS((lc->requested_fc & ~(PAUSE_TX | PAUSE_RX | PAUSE_AUTONEG)) == 0); 4425 if (lc->requested_fc & PAUSE_TX && 4426 !(lc->supported & FW_PORT_CAP32_FC_TX)) { 4427 n++; 4428 lc->requested_fc &= ~PAUSE_TX; 4429 } 4430 if (lc->requested_fc & PAUSE_RX && 4431 !(lc->supported & FW_PORT_CAP32_FC_RX)) { 4432 n++; 4433 lc->requested_fc &= ~PAUSE_RX; 4434 } 4435 if (!(lc->requested_fc & PAUSE_AUTONEG) && 4436 !(lc->supported & FW_PORT_CAP32_FORCE_PAUSE)) { 4437 n++; 4438 lc->requested_fc |= PAUSE_AUTONEG; 4439 } 4440 4441 /* FEC */ 4442 if ((lc->requested_fec & FEC_RS && 4443 !(lc->supported & FW_PORT_CAP32_FEC_RS)) || 4444 (lc->requested_fec & FEC_BASER_RS && 4445 !(lc->supported & FW_PORT_CAP32_FEC_BASER_RS))) { 4446 n++; 4447 lc->requested_fec = FEC_AUTO; 4448 } 4449 4450 return (n); 4451 } 4452 4453 /* 4454 * Apply the requested L1 settings, which are expected to be valid, to the 4455 * hardware. 4456 */ 4457 static int 4458 apply_link_config(struct port_info *pi) 4459 { 4460 struct adapter *sc = pi->adapter; 4461 struct link_config *lc = &pi->link_cfg; 4462 int rc; 4463 4464 #ifdef INVARIANTS 4465 ASSERT_SYNCHRONIZED_OP(sc); 4466 PORT_LOCK_ASSERT_OWNED(pi); 4467 4468 if (lc->requested_aneg == AUTONEG_ENABLE) 4469 MPASS(lc->supported & FW_PORT_CAP32_ANEG); 4470 if (!(lc->requested_fc & PAUSE_AUTONEG)) 4471 MPASS(lc->supported & FW_PORT_CAP32_FORCE_PAUSE); 4472 if (lc->requested_fc & PAUSE_TX) 4473 MPASS(lc->supported & FW_PORT_CAP32_FC_TX); 4474 if (lc->requested_fc & PAUSE_RX) 4475 MPASS(lc->supported & FW_PORT_CAP32_FC_RX); 4476 if (lc->requested_fec & FEC_RS) 4477 MPASS(lc->supported & FW_PORT_CAP32_FEC_RS); 4478 if (lc->requested_fec & FEC_BASER_RS) 4479 MPASS(lc->supported & FW_PORT_CAP32_FEC_BASER_RS); 4480 #endif 4481 rc = -t4_link_l1cfg(sc, sc->mbox, pi->tx_chan, lc); 4482 if (rc != 0) { 4483 /* Don't complain if the VF driver gets back an EPERM. */ 4484 if (!(sc->flags & IS_VF) || rc != FW_EPERM) 4485 device_printf(pi->dev, "l1cfg failed: %d\n", rc); 4486 } else { 4487 /* 4488 * An L1_CFG will almost always result in a link-change event if 4489 * the link is up, and the driver will refresh the actual 4490 * fec/fc/etc. when the notification is processed. If the link 4491 * is down then the actual settings are meaningless. 4492 * 4493 * This takes care of the case where a change in the L1 settings 4494 * may not result in a notification. 4495 */ 4496 if (lc->link_ok && !(lc->requested_fc & PAUSE_AUTONEG)) 4497 lc->fc = lc->requested_fc & (PAUSE_TX | PAUSE_RX); 4498 } 4499 return (rc); 4500 } 4501 4502 #define FW_MAC_EXACT_CHUNK 7 4503 4504 /* 4505 * Program the port's XGMAC based on parameters in ifnet. The caller also 4506 * indicates which parameters should be programmed (the rest are left alone). 4507 */ 4508 int 4509 update_mac_settings(struct ifnet *ifp, int flags) 4510 { 4511 int rc = 0; 4512 struct vi_info *vi = ifp->if_softc; 4513 struct port_info *pi = vi->pi; 4514 struct adapter *sc = pi->adapter; 4515 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 4516 4517 ASSERT_SYNCHRONIZED_OP(sc); 4518 KASSERT(flags, ("%s: not told what to update.", __func__)); 4519 4520 if (flags & XGMAC_MTU) 4521 mtu = ifp->if_mtu; 4522 4523 if (flags & XGMAC_PROMISC) 4524 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 4525 4526 if (flags & XGMAC_ALLMULTI) 4527 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 4528 4529 if (flags & XGMAC_VLANEX) 4530 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 4531 4532 if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) { 4533 rc = -t4_set_rxmode(sc, sc->mbox, vi->viid, mtu, promisc, 4534 allmulti, 1, vlanex, false); 4535 if (rc) { 4536 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, 4537 rc); 4538 return (rc); 4539 } 4540 } 4541 4542 if (flags & XGMAC_UCADDR) { 4543 uint8_t ucaddr[ETHER_ADDR_LEN]; 4544 4545 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 4546 rc = t4_change_mac(sc, sc->mbox, vi->viid, vi->xact_addr_filt, 4547 ucaddr, true, true); 4548 if (rc < 0) { 4549 rc = -rc; 4550 if_printf(ifp, "change_mac failed: %d\n", rc); 4551 return (rc); 4552 } else { 4553 vi->xact_addr_filt = rc; 4554 rc = 0; 4555 } 4556 } 4557 4558 if (flags & XGMAC_MCADDRS) { 4559 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 4560 int del = 1; 4561 uint64_t hash = 0; 4562 struct ifmultiaddr *ifma; 4563 int i = 0, j; 4564 4565 if_maddr_rlock(ifp); 4566 CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 4567 if (ifma->ifma_addr->sa_family != AF_LINK) 4568 continue; 4569 mcaddr[i] = 4570 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 4571 MPASS(ETHER_IS_MULTICAST(mcaddr[i])); 4572 i++; 4573 4574 if (i == FW_MAC_EXACT_CHUNK) { 4575 rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid, 4576 del, i, mcaddr, NULL, &hash, 0); 4577 if (rc < 0) { 4578 rc = -rc; 4579 for (j = 0; j < i; j++) { 4580 if_printf(ifp, 4581 "failed to add mc address" 4582 " %02x:%02x:%02x:" 4583 "%02x:%02x:%02x rc=%d\n", 4584 mcaddr[j][0], mcaddr[j][1], 4585 mcaddr[j][2], mcaddr[j][3], 4586 mcaddr[j][4], mcaddr[j][5], 4587 rc); 4588 } 4589 goto mcfail; 4590 } 4591 del = 0; 4592 i = 0; 4593 } 4594 } 4595 if (i > 0) { 4596 rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid, del, i, 4597 mcaddr, NULL, &hash, 0); 4598 if (rc < 0) { 4599 rc = -rc; 4600 for (j = 0; j < i; j++) { 4601 if_printf(ifp, 4602 "failed to add mc address" 4603 " %02x:%02x:%02x:" 4604 "%02x:%02x:%02x rc=%d\n", 4605 mcaddr[j][0], mcaddr[j][1], 4606 mcaddr[j][2], mcaddr[j][3], 4607 mcaddr[j][4], mcaddr[j][5], 4608 rc); 4609 } 4610 goto mcfail; 4611 } 4612 } 4613 4614 rc = -t4_set_addr_hash(sc, sc->mbox, vi->viid, 0, hash, 0); 4615 if (rc != 0) 4616 if_printf(ifp, "failed to set mc address hash: %d", rc); 4617 mcfail: 4618 if_maddr_runlock(ifp); 4619 } 4620 4621 return (rc); 4622 } 4623 4624 /* 4625 * {begin|end}_synchronized_op must be called from the same thread. 4626 */ 4627 int 4628 begin_synchronized_op(struct adapter *sc, struct vi_info *vi, int flags, 4629 char *wmesg) 4630 { 4631 int rc, pri; 4632 4633 #ifdef WITNESS 4634 /* the caller thinks it's ok to sleep, but is it really? */ 4635 if (flags & SLEEP_OK) 4636 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 4637 "begin_synchronized_op"); 4638 #endif 4639 4640 if (INTR_OK) 4641 pri = PCATCH; 4642 else 4643 pri = 0; 4644 4645 ADAPTER_LOCK(sc); 4646 for (;;) { 4647 4648 if (vi && IS_DOOMED(vi)) { 4649 rc = ENXIO; 4650 goto done; 4651 } 4652 4653 if (!IS_BUSY(sc)) { 4654 rc = 0; 4655 break; 4656 } 4657 4658 if (!(flags & SLEEP_OK)) { 4659 rc = EBUSY; 4660 goto done; 4661 } 4662 4663 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 4664 rc = EINTR; 4665 goto done; 4666 } 4667 } 4668 4669 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 4670 SET_BUSY(sc); 4671 #ifdef INVARIANTS 4672 sc->last_op = wmesg; 4673 sc->last_op_thr = curthread; 4674 sc->last_op_flags = flags; 4675 #endif 4676 4677 done: 4678 if (!(flags & HOLD_LOCK) || rc) 4679 ADAPTER_UNLOCK(sc); 4680 4681 return (rc); 4682 } 4683 4684 /* 4685 * Tell if_ioctl and if_init that the VI is going away. This is 4686 * special variant of begin_synchronized_op and must be paired with a 4687 * call to end_synchronized_op. 4688 */ 4689 void 4690 doom_vi(struct adapter *sc, struct vi_info *vi) 4691 { 4692 4693 ADAPTER_LOCK(sc); 4694 SET_DOOMED(vi); 4695 wakeup(&sc->flags); 4696 while (IS_BUSY(sc)) 4697 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 4698 SET_BUSY(sc); 4699 #ifdef INVARIANTS 4700 sc->last_op = "t4detach"; 4701 sc->last_op_thr = curthread; 4702 sc->last_op_flags = 0; 4703 #endif 4704 ADAPTER_UNLOCK(sc); 4705 } 4706 4707 /* 4708 * {begin|end}_synchronized_op must be called from the same thread. 4709 */ 4710 void 4711 end_synchronized_op(struct adapter *sc, int flags) 4712 { 4713 4714 if (flags & LOCK_HELD) 4715 ADAPTER_LOCK_ASSERT_OWNED(sc); 4716 else 4717 ADAPTER_LOCK(sc); 4718 4719 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 4720 CLR_BUSY(sc); 4721 wakeup(&sc->flags); 4722 ADAPTER_UNLOCK(sc); 4723 } 4724 4725 static int 4726 cxgbe_init_synchronized(struct vi_info *vi) 4727 { 4728 struct port_info *pi = vi->pi; 4729 struct adapter *sc = pi->adapter; 4730 struct ifnet *ifp = vi->ifp; 4731 int rc = 0, i; 4732 struct sge_txq *txq; 4733 4734 ASSERT_SYNCHRONIZED_OP(sc); 4735 4736 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 4737 return (0); /* already running */ 4738 4739 if (!(sc->flags & FULL_INIT_DONE) && 4740 ((rc = adapter_full_init(sc)) != 0)) 4741 return (rc); /* error message displayed already */ 4742 4743 if (!(vi->flags & VI_INIT_DONE) && 4744 ((rc = vi_full_init(vi)) != 0)) 4745 return (rc); /* error message displayed already */ 4746 4747 rc = update_mac_settings(ifp, XGMAC_ALL); 4748 if (rc) 4749 goto done; /* error message displayed already */ 4750 4751 PORT_LOCK(pi); 4752 if (pi->up_vis == 0) { 4753 t4_update_port_info(pi); 4754 fixup_link_config(pi); 4755 build_medialist(pi); 4756 apply_link_config(pi); 4757 } 4758 4759 rc = -t4_enable_vi(sc, sc->mbox, vi->viid, true, true); 4760 if (rc != 0) { 4761 if_printf(ifp, "enable_vi failed: %d\n", rc); 4762 PORT_UNLOCK(pi); 4763 goto done; 4764 } 4765 4766 /* 4767 * Can't fail from this point onwards. Review cxgbe_uninit_synchronized 4768 * if this changes. 4769 */ 4770 4771 for_each_txq(vi, i, txq) { 4772 TXQ_LOCK(txq); 4773 txq->eq.flags |= EQ_ENABLED; 4774 TXQ_UNLOCK(txq); 4775 } 4776 4777 /* 4778 * The first iq of the first port to come up is used for tracing. 4779 */ 4780 if (sc->traceq < 0 && IS_MAIN_VI(vi)) { 4781 sc->traceq = sc->sge.rxq[vi->first_rxq].iq.abs_id; 4782 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : 4783 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | 4784 V_QUEUENUMBER(sc->traceq)); 4785 pi->flags |= HAS_TRACEQ; 4786 } 4787 4788 /* all ok */ 4789 pi->up_vis++; 4790 ifp->if_drv_flags |= IFF_DRV_RUNNING; 4791 4792 if (pi->nvi > 1 || sc->flags & IS_VF) 4793 callout_reset(&vi->tick, hz, vi_tick, vi); 4794 else 4795 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 4796 PORT_UNLOCK(pi); 4797 done: 4798 if (rc != 0) 4799 cxgbe_uninit_synchronized(vi); 4800 4801 return (rc); 4802 } 4803 4804 /* 4805 * Idempotent. 4806 */ 4807 static int 4808 cxgbe_uninit_synchronized(struct vi_info *vi) 4809 { 4810 struct port_info *pi = vi->pi; 4811 struct adapter *sc = pi->adapter; 4812 struct ifnet *ifp = vi->ifp; 4813 int rc, i; 4814 struct sge_txq *txq; 4815 4816 ASSERT_SYNCHRONIZED_OP(sc); 4817 4818 if (!(vi->flags & VI_INIT_DONE)) { 4819 if (__predict_false(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 4820 KASSERT(0, ("uninited VI is running")); 4821 if_printf(ifp, "uninited VI with running ifnet. " 4822 "vi->flags 0x%016lx, if_flags 0x%08x, " 4823 "if_drv_flags 0x%08x\n", vi->flags, ifp->if_flags, 4824 ifp->if_drv_flags); 4825 } 4826 return (0); 4827 } 4828 4829 /* 4830 * Disable the VI so that all its data in either direction is discarded 4831 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 4832 * tick) intact as the TP can deliver negative advice or data that it's 4833 * holding in its RAM (for an offloaded connection) even after the VI is 4834 * disabled. 4835 */ 4836 rc = -t4_enable_vi(sc, sc->mbox, vi->viid, false, false); 4837 if (rc) { 4838 if_printf(ifp, "disable_vi failed: %d\n", rc); 4839 return (rc); 4840 } 4841 4842 for_each_txq(vi, i, txq) { 4843 TXQ_LOCK(txq); 4844 txq->eq.flags &= ~EQ_ENABLED; 4845 TXQ_UNLOCK(txq); 4846 } 4847 4848 PORT_LOCK(pi); 4849 if (pi->nvi > 1 || sc->flags & IS_VF) 4850 callout_stop(&vi->tick); 4851 else 4852 callout_stop(&pi->tick); 4853 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 4854 PORT_UNLOCK(pi); 4855 return (0); 4856 } 4857 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 4858 pi->up_vis--; 4859 if (pi->up_vis > 0) { 4860 PORT_UNLOCK(pi); 4861 return (0); 4862 } 4863 4864 pi->link_cfg.link_ok = false; 4865 pi->link_cfg.speed = 0; 4866 pi->link_cfg.link_down_rc = 255; 4867 t4_os_link_changed(pi); 4868 PORT_UNLOCK(pi); 4869 4870 return (0); 4871 } 4872 4873 /* 4874 * It is ok for this function to fail midway and return right away. t4_detach 4875 * will walk the entire sc->irq list and clean up whatever is valid. 4876 */ 4877 int 4878 t4_setup_intr_handlers(struct adapter *sc) 4879 { 4880 int rc, rid, p, q, v; 4881 char s[8]; 4882 struct irq *irq; 4883 struct port_info *pi; 4884 struct vi_info *vi; 4885 struct sge *sge = &sc->sge; 4886 struct sge_rxq *rxq; 4887 #ifdef TCP_OFFLOAD 4888 struct sge_ofld_rxq *ofld_rxq; 4889 #endif 4890 #ifdef DEV_NETMAP 4891 struct sge_nm_rxq *nm_rxq; 4892 #endif 4893 #ifdef RSS 4894 int nbuckets = rss_getnumbuckets(); 4895 #endif 4896 4897 /* 4898 * Setup interrupts. 4899 */ 4900 irq = &sc->irq[0]; 4901 rid = sc->intr_type == INTR_INTX ? 0 : 1; 4902 if (forwarding_intr_to_fwq(sc)) 4903 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all")); 4904 4905 /* Multiple interrupts. */ 4906 if (sc->flags & IS_VF) 4907 KASSERT(sc->intr_count >= T4VF_EXTRA_INTR + sc->params.nports, 4908 ("%s: too few intr.", __func__)); 4909 else 4910 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 4911 ("%s: too few intr.", __func__)); 4912 4913 /* The first one is always error intr on PFs */ 4914 if (!(sc->flags & IS_VF)) { 4915 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 4916 if (rc != 0) 4917 return (rc); 4918 irq++; 4919 rid++; 4920 } 4921 4922 /* The second one is always the firmware event queue (first on VFs) */ 4923 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sge->fwq, "evt"); 4924 if (rc != 0) 4925 return (rc); 4926 irq++; 4927 rid++; 4928 4929 for_each_port(sc, p) { 4930 pi = sc->port[p]; 4931 for_each_vi(pi, v, vi) { 4932 vi->first_intr = rid - 1; 4933 4934 if (vi->nnmrxq > 0) { 4935 int n = max(vi->nrxq, vi->nnmrxq); 4936 4937 rxq = &sge->rxq[vi->first_rxq]; 4938 #ifdef DEV_NETMAP 4939 nm_rxq = &sge->nm_rxq[vi->first_nm_rxq]; 4940 #endif 4941 for (q = 0; q < n; q++) { 4942 snprintf(s, sizeof(s), "%x%c%x", p, 4943 'a' + v, q); 4944 if (q < vi->nrxq) 4945 irq->rxq = rxq++; 4946 #ifdef DEV_NETMAP 4947 if (q < vi->nnmrxq) 4948 irq->nm_rxq = nm_rxq++; 4949 4950 if (irq->nm_rxq != NULL && 4951 irq->rxq == NULL) { 4952 /* Netmap rx only */ 4953 rc = t4_alloc_irq(sc, irq, rid, 4954 t4_nm_intr, irq->nm_rxq, s); 4955 } 4956 if (irq->nm_rxq != NULL && 4957 irq->rxq != NULL) { 4958 /* NIC and Netmap rx */ 4959 rc = t4_alloc_irq(sc, irq, rid, 4960 t4_vi_intr, irq, s); 4961 } 4962 #endif 4963 if (irq->rxq != NULL && 4964 irq->nm_rxq == NULL) { 4965 /* NIC rx only */ 4966 rc = t4_alloc_irq(sc, irq, rid, 4967 t4_intr, irq->rxq, s); 4968 } 4969 if (rc != 0) 4970 return (rc); 4971 #ifdef RSS 4972 if (q < vi->nrxq) { 4973 bus_bind_intr(sc->dev, irq->res, 4974 rss_getcpu(q % nbuckets)); 4975 } 4976 #endif 4977 irq++; 4978 rid++; 4979 vi->nintr++; 4980 } 4981 } else { 4982 for_each_rxq(vi, q, rxq) { 4983 snprintf(s, sizeof(s), "%x%c%x", p, 4984 'a' + v, q); 4985 rc = t4_alloc_irq(sc, irq, rid, 4986 t4_intr, rxq, s); 4987 if (rc != 0) 4988 return (rc); 4989 #ifdef RSS 4990 bus_bind_intr(sc->dev, irq->res, 4991 rss_getcpu(q % nbuckets)); 4992 #endif 4993 irq++; 4994 rid++; 4995 vi->nintr++; 4996 } 4997 } 4998 #ifdef TCP_OFFLOAD 4999 for_each_ofld_rxq(vi, q, ofld_rxq) { 5000 snprintf(s, sizeof(s), "%x%c%x", p, 'A' + v, q); 5001 rc = t4_alloc_irq(sc, irq, rid, t4_intr, 5002 ofld_rxq, s); 5003 if (rc != 0) 5004 return (rc); 5005 irq++; 5006 rid++; 5007 vi->nintr++; 5008 } 5009 #endif 5010 } 5011 } 5012 MPASS(irq == &sc->irq[sc->intr_count]); 5013 5014 return (0); 5015 } 5016 5017 int 5018 adapter_full_init(struct adapter *sc) 5019 { 5020 int rc, i; 5021 #ifdef RSS 5022 uint32_t raw_rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; 5023 uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; 5024 #endif 5025 5026 ASSERT_SYNCHRONIZED_OP(sc); 5027 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 5028 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 5029 ("%s: FULL_INIT_DONE already", __func__)); 5030 5031 /* 5032 * queues that belong to the adapter (not any particular port). 5033 */ 5034 rc = t4_setup_adapter_queues(sc); 5035 if (rc != 0) 5036 goto done; 5037 5038 for (i = 0; i < nitems(sc->tq); i++) { 5039 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 5040 taskqueue_thread_enqueue, &sc->tq[i]); 5041 if (sc->tq[i] == NULL) { 5042 device_printf(sc->dev, 5043 "failed to allocate task queue %d\n", i); 5044 rc = ENOMEM; 5045 goto done; 5046 } 5047 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 5048 device_get_nameunit(sc->dev), i); 5049 } 5050 #ifdef RSS 5051 MPASS(RSS_KEYSIZE == 40); 5052 rss_getkey((void *)&raw_rss_key[0]); 5053 for (i = 0; i < nitems(rss_key); i++) { 5054 rss_key[i] = htobe32(raw_rss_key[nitems(rss_key) - 1 - i]); 5055 } 5056 t4_write_rss_key(sc, &rss_key[0], -1, 1); 5057 #endif 5058 5059 if (!(sc->flags & IS_VF)) 5060 t4_intr_enable(sc); 5061 sc->flags |= FULL_INIT_DONE; 5062 done: 5063 if (rc != 0) 5064 adapter_full_uninit(sc); 5065 5066 return (rc); 5067 } 5068 5069 int 5070 adapter_full_uninit(struct adapter *sc) 5071 { 5072 int i; 5073 5074 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 5075 5076 t4_teardown_adapter_queues(sc); 5077 5078 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 5079 taskqueue_free(sc->tq[i]); 5080 sc->tq[i] = NULL; 5081 } 5082 5083 sc->flags &= ~FULL_INIT_DONE; 5084 5085 return (0); 5086 } 5087 5088 #ifdef RSS 5089 #define SUPPORTED_RSS_HASHTYPES (RSS_HASHTYPE_RSS_IPV4 | \ 5090 RSS_HASHTYPE_RSS_TCP_IPV4 | RSS_HASHTYPE_RSS_IPV6 | \ 5091 RSS_HASHTYPE_RSS_TCP_IPV6 | RSS_HASHTYPE_RSS_UDP_IPV4 | \ 5092 RSS_HASHTYPE_RSS_UDP_IPV6) 5093 5094 /* Translates kernel hash types to hardware. */ 5095 static int 5096 hashconfig_to_hashen(int hashconfig) 5097 { 5098 int hashen = 0; 5099 5100 if (hashconfig & RSS_HASHTYPE_RSS_IPV4) 5101 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN; 5102 if (hashconfig & RSS_HASHTYPE_RSS_IPV6) 5103 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN; 5104 if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV4) { 5105 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN | 5106 F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN; 5107 } 5108 if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV6) { 5109 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN | 5110 F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN; 5111 } 5112 if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV4) 5113 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN; 5114 if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV6) 5115 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN; 5116 5117 return (hashen); 5118 } 5119 5120 /* Translates hardware hash types to kernel. */ 5121 static int 5122 hashen_to_hashconfig(int hashen) 5123 { 5124 int hashconfig = 0; 5125 5126 if (hashen & F_FW_RSS_VI_CONFIG_CMD_UDPEN) { 5127 /* 5128 * If UDP hashing was enabled it must have been enabled for 5129 * either IPv4 or IPv6 (inclusive or). Enabling UDP without 5130 * enabling any 4-tuple hash is nonsense configuration. 5131 */ 5132 MPASS(hashen & (F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN | 5133 F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)); 5134 5135 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) 5136 hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV4; 5137 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) 5138 hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV6; 5139 } 5140 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) 5141 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV4; 5142 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) 5143 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV6; 5144 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN) 5145 hashconfig |= RSS_HASHTYPE_RSS_IPV4; 5146 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN) 5147 hashconfig |= RSS_HASHTYPE_RSS_IPV6; 5148 5149 return (hashconfig); 5150 } 5151 #endif 5152 5153 int 5154 vi_full_init(struct vi_info *vi) 5155 { 5156 struct adapter *sc = vi->pi->adapter; 5157 struct ifnet *ifp = vi->ifp; 5158 uint16_t *rss; 5159 struct sge_rxq *rxq; 5160 int rc, i, j; 5161 #ifdef RSS 5162 int nbuckets = rss_getnumbuckets(); 5163 int hashconfig = rss_gethashconfig(); 5164 int extra; 5165 #endif 5166 5167 ASSERT_SYNCHRONIZED_OP(sc); 5168 KASSERT((vi->flags & VI_INIT_DONE) == 0, 5169 ("%s: VI_INIT_DONE already", __func__)); 5170 5171 sysctl_ctx_init(&vi->ctx); 5172 vi->flags |= VI_SYSCTL_CTX; 5173 5174 /* 5175 * Allocate tx/rx/fl queues for this VI. 5176 */ 5177 rc = t4_setup_vi_queues(vi); 5178 if (rc != 0) 5179 goto done; /* error message displayed already */ 5180 5181 /* 5182 * Setup RSS for this VI. Save a copy of the RSS table for later use. 5183 */ 5184 if (vi->nrxq > vi->rss_size) { 5185 if_printf(ifp, "nrxq (%d) > hw RSS table size (%d); " 5186 "some queues will never receive traffic.\n", vi->nrxq, 5187 vi->rss_size); 5188 } else if (vi->rss_size % vi->nrxq) { 5189 if_printf(ifp, "nrxq (%d), hw RSS table size (%d); " 5190 "expect uneven traffic distribution.\n", vi->nrxq, 5191 vi->rss_size); 5192 } 5193 #ifdef RSS 5194 if (vi->nrxq != nbuckets) { 5195 if_printf(ifp, "nrxq (%d) != kernel RSS buckets (%d);" 5196 "performance will be impacted.\n", vi->nrxq, nbuckets); 5197 } 5198 #endif 5199 rss = malloc(vi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); 5200 for (i = 0; i < vi->rss_size;) { 5201 #ifdef RSS 5202 j = rss_get_indirection_to_bucket(i); 5203 j %= vi->nrxq; 5204 rxq = &sc->sge.rxq[vi->first_rxq + j]; 5205 rss[i++] = rxq->iq.abs_id; 5206 #else 5207 for_each_rxq(vi, j, rxq) { 5208 rss[i++] = rxq->iq.abs_id; 5209 if (i == vi->rss_size) 5210 break; 5211 } 5212 #endif 5213 } 5214 5215 rc = -t4_config_rss_range(sc, sc->mbox, vi->viid, 0, vi->rss_size, rss, 5216 vi->rss_size); 5217 if (rc != 0) { 5218 free(rss, M_CXGBE); 5219 if_printf(ifp, "rss_config failed: %d\n", rc); 5220 goto done; 5221 } 5222 5223 #ifdef RSS 5224 vi->hashen = hashconfig_to_hashen(hashconfig); 5225 5226 /* 5227 * We may have had to enable some hashes even though the global config 5228 * wants them disabled. This is a potential problem that must be 5229 * reported to the user. 5230 */ 5231 extra = hashen_to_hashconfig(vi->hashen) ^ hashconfig; 5232 5233 /* 5234 * If we consider only the supported hash types, then the enabled hashes 5235 * are a superset of the requested hashes. In other words, there cannot 5236 * be any supported hash that was requested but not enabled, but there 5237 * can be hashes that were not requested but had to be enabled. 5238 */ 5239 extra &= SUPPORTED_RSS_HASHTYPES; 5240 MPASS((extra & hashconfig) == 0); 5241 5242 if (extra) { 5243 if_printf(ifp, 5244 "global RSS config (0x%x) cannot be accommodated.\n", 5245 hashconfig); 5246 } 5247 if (extra & RSS_HASHTYPE_RSS_IPV4) 5248 if_printf(ifp, "IPv4 2-tuple hashing forced on.\n"); 5249 if (extra & RSS_HASHTYPE_RSS_TCP_IPV4) 5250 if_printf(ifp, "TCP/IPv4 4-tuple hashing forced on.\n"); 5251 if (extra & RSS_HASHTYPE_RSS_IPV6) 5252 if_printf(ifp, "IPv6 2-tuple hashing forced on.\n"); 5253 if (extra & RSS_HASHTYPE_RSS_TCP_IPV6) 5254 if_printf(ifp, "TCP/IPv6 4-tuple hashing forced on.\n"); 5255 if (extra & RSS_HASHTYPE_RSS_UDP_IPV4) 5256 if_printf(ifp, "UDP/IPv4 4-tuple hashing forced on.\n"); 5257 if (extra & RSS_HASHTYPE_RSS_UDP_IPV6) 5258 if_printf(ifp, "UDP/IPv6 4-tuple hashing forced on.\n"); 5259 #else 5260 vi->hashen = F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN | 5261 F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN | 5262 F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN | 5263 F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN | F_FW_RSS_VI_CONFIG_CMD_UDPEN; 5264 #endif 5265 rc = -t4_config_vi_rss(sc, sc->mbox, vi->viid, vi->hashen, rss[0], 0, 0); 5266 if (rc != 0) { 5267 free(rss, M_CXGBE); 5268 if_printf(ifp, "rss hash/defaultq config failed: %d\n", rc); 5269 goto done; 5270 } 5271 5272 vi->rss = rss; 5273 vi->flags |= VI_INIT_DONE; 5274 done: 5275 if (rc != 0) 5276 vi_full_uninit(vi); 5277 5278 return (rc); 5279 } 5280 5281 /* 5282 * Idempotent. 5283 */ 5284 int 5285 vi_full_uninit(struct vi_info *vi) 5286 { 5287 struct port_info *pi = vi->pi; 5288 struct adapter *sc = pi->adapter; 5289 int i; 5290 struct sge_rxq *rxq; 5291 struct sge_txq *txq; 5292 #ifdef TCP_OFFLOAD 5293 struct sge_ofld_rxq *ofld_rxq; 5294 struct sge_wrq *ofld_txq; 5295 #endif 5296 5297 if (vi->flags & VI_INIT_DONE) { 5298 5299 /* Need to quiesce queues. */ 5300 5301 /* XXX: Only for the first VI? */ 5302 if (IS_MAIN_VI(vi) && !(sc->flags & IS_VF)) 5303 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]); 5304 5305 for_each_txq(vi, i, txq) { 5306 quiesce_txq(sc, txq); 5307 } 5308 5309 #ifdef TCP_OFFLOAD 5310 for_each_ofld_txq(vi, i, ofld_txq) { 5311 quiesce_wrq(sc, ofld_txq); 5312 } 5313 #endif 5314 5315 for_each_rxq(vi, i, rxq) { 5316 quiesce_iq(sc, &rxq->iq); 5317 quiesce_fl(sc, &rxq->fl); 5318 } 5319 5320 #ifdef TCP_OFFLOAD 5321 for_each_ofld_rxq(vi, i, ofld_rxq) { 5322 quiesce_iq(sc, &ofld_rxq->iq); 5323 quiesce_fl(sc, &ofld_rxq->fl); 5324 } 5325 #endif 5326 free(vi->rss, M_CXGBE); 5327 free(vi->nm_rss, M_CXGBE); 5328 } 5329 5330 t4_teardown_vi_queues(vi); 5331 vi->flags &= ~VI_INIT_DONE; 5332 5333 return (0); 5334 } 5335 5336 static void 5337 quiesce_txq(struct adapter *sc, struct sge_txq *txq) 5338 { 5339 struct sge_eq *eq = &txq->eq; 5340 struct sge_qstat *spg = (void *)&eq->desc[eq->sidx]; 5341 5342 (void) sc; /* unused */ 5343 5344 #ifdef INVARIANTS 5345 TXQ_LOCK(txq); 5346 MPASS((eq->flags & EQ_ENABLED) == 0); 5347 TXQ_UNLOCK(txq); 5348 #endif 5349 5350 /* Wait for the mp_ring to empty. */ 5351 while (!mp_ring_is_idle(txq->r)) { 5352 mp_ring_check_drainage(txq->r, 0); 5353 pause("rquiesce", 1); 5354 } 5355 5356 /* Then wait for the hardware to finish. */ 5357 while (spg->cidx != htobe16(eq->pidx)) 5358 pause("equiesce", 1); 5359 5360 /* Finally, wait for the driver to reclaim all descriptors. */ 5361 while (eq->cidx != eq->pidx) 5362 pause("dquiesce", 1); 5363 } 5364 5365 static void 5366 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq) 5367 { 5368 5369 /* XXXTX */ 5370 } 5371 5372 static void 5373 quiesce_iq(struct adapter *sc, struct sge_iq *iq) 5374 { 5375 (void) sc; /* unused */ 5376 5377 /* Synchronize with the interrupt handler */ 5378 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 5379 pause("iqfree", 1); 5380 } 5381 5382 static void 5383 quiesce_fl(struct adapter *sc, struct sge_fl *fl) 5384 { 5385 mtx_lock(&sc->sfl_lock); 5386 FL_LOCK(fl); 5387 fl->flags |= FL_DOOMED; 5388 FL_UNLOCK(fl); 5389 callout_stop(&sc->sfl_callout); 5390 mtx_unlock(&sc->sfl_lock); 5391 5392 KASSERT((fl->flags & FL_STARVING) == 0, 5393 ("%s: still starving", __func__)); 5394 } 5395 5396 static int 5397 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 5398 driver_intr_t *handler, void *arg, char *name) 5399 { 5400 int rc; 5401 5402 irq->rid = rid; 5403 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 5404 RF_SHAREABLE | RF_ACTIVE); 5405 if (irq->res == NULL) { 5406 device_printf(sc->dev, 5407 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 5408 return (ENOMEM); 5409 } 5410 5411 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 5412 NULL, handler, arg, &irq->tag); 5413 if (rc != 0) { 5414 device_printf(sc->dev, 5415 "failed to setup interrupt for rid %d, name %s: %d\n", 5416 rid, name, rc); 5417 } else if (name) 5418 bus_describe_intr(sc->dev, irq->res, irq->tag, "%s", name); 5419 5420 return (rc); 5421 } 5422 5423 static int 5424 t4_free_irq(struct adapter *sc, struct irq *irq) 5425 { 5426 if (irq->tag) 5427 bus_teardown_intr(sc->dev, irq->res, irq->tag); 5428 if (irq->res) 5429 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 5430 5431 bzero(irq, sizeof(*irq)); 5432 5433 return (0); 5434 } 5435 5436 static void 5437 get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 5438 { 5439 5440 regs->version = chip_id(sc) | chip_rev(sc) << 10; 5441 t4_get_regs(sc, buf, regs->len); 5442 } 5443 5444 #define A_PL_INDIR_CMD 0x1f8 5445 5446 #define S_PL_AUTOINC 31 5447 #define M_PL_AUTOINC 0x1U 5448 #define V_PL_AUTOINC(x) ((x) << S_PL_AUTOINC) 5449 #define G_PL_AUTOINC(x) (((x) >> S_PL_AUTOINC) & M_PL_AUTOINC) 5450 5451 #define S_PL_VFID 20 5452 #define M_PL_VFID 0xffU 5453 #define V_PL_VFID(x) ((x) << S_PL_VFID) 5454 #define G_PL_VFID(x) (((x) >> S_PL_VFID) & M_PL_VFID) 5455 5456 #define S_PL_ADDR 0 5457 #define M_PL_ADDR 0xfffffU 5458 #define V_PL_ADDR(x) ((x) << S_PL_ADDR) 5459 #define G_PL_ADDR(x) (((x) >> S_PL_ADDR) & M_PL_ADDR) 5460 5461 #define A_PL_INDIR_DATA 0x1fc 5462 5463 static uint64_t 5464 read_vf_stat(struct adapter *sc, unsigned int viid, int reg) 5465 { 5466 u32 stats[2]; 5467 5468 mtx_assert(&sc->reg_lock, MA_OWNED); 5469 if (sc->flags & IS_VF) { 5470 stats[0] = t4_read_reg(sc, VF_MPS_REG(reg)); 5471 stats[1] = t4_read_reg(sc, VF_MPS_REG(reg + 4)); 5472 } else { 5473 t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) | 5474 V_PL_VFID(G_FW_VIID_VIN(viid)) | 5475 V_PL_ADDR(VF_MPS_REG(reg))); 5476 stats[0] = t4_read_reg(sc, A_PL_INDIR_DATA); 5477 stats[1] = t4_read_reg(sc, A_PL_INDIR_DATA); 5478 } 5479 return (((uint64_t)stats[1]) << 32 | stats[0]); 5480 } 5481 5482 static void 5483 t4_get_vi_stats(struct adapter *sc, unsigned int viid, 5484 struct fw_vi_stats_vf *stats) 5485 { 5486 5487 #define GET_STAT(name) \ 5488 read_vf_stat(sc, viid, A_MPS_VF_STAT_##name##_L) 5489 5490 stats->tx_bcast_bytes = GET_STAT(TX_VF_BCAST_BYTES); 5491 stats->tx_bcast_frames = GET_STAT(TX_VF_BCAST_FRAMES); 5492 stats->tx_mcast_bytes = GET_STAT(TX_VF_MCAST_BYTES); 5493 stats->tx_mcast_frames = GET_STAT(TX_VF_MCAST_FRAMES); 5494 stats->tx_ucast_bytes = GET_STAT(TX_VF_UCAST_BYTES); 5495 stats->tx_ucast_frames = GET_STAT(TX_VF_UCAST_FRAMES); 5496 stats->tx_drop_frames = GET_STAT(TX_VF_DROP_FRAMES); 5497 stats->tx_offload_bytes = GET_STAT(TX_VF_OFFLOAD_BYTES); 5498 stats->tx_offload_frames = GET_STAT(TX_VF_OFFLOAD_FRAMES); 5499 stats->rx_bcast_bytes = GET_STAT(RX_VF_BCAST_BYTES); 5500 stats->rx_bcast_frames = GET_STAT(RX_VF_BCAST_FRAMES); 5501 stats->rx_mcast_bytes = GET_STAT(RX_VF_MCAST_BYTES); 5502 stats->rx_mcast_frames = GET_STAT(RX_VF_MCAST_FRAMES); 5503 stats->rx_ucast_bytes = GET_STAT(RX_VF_UCAST_BYTES); 5504 stats->rx_ucast_frames = GET_STAT(RX_VF_UCAST_FRAMES); 5505 stats->rx_err_frames = GET_STAT(RX_VF_ERR_FRAMES); 5506 5507 #undef GET_STAT 5508 } 5509 5510 static void 5511 t4_clr_vi_stats(struct adapter *sc, unsigned int viid) 5512 { 5513 int reg; 5514 5515 t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) | 5516 V_PL_VFID(G_FW_VIID_VIN(viid)) | 5517 V_PL_ADDR(VF_MPS_REG(A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L))); 5518 for (reg = A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L; 5519 reg <= A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H; reg += 4) 5520 t4_write_reg(sc, A_PL_INDIR_DATA, 0); 5521 } 5522 5523 static void 5524 vi_refresh_stats(struct adapter *sc, struct vi_info *vi) 5525 { 5526 struct timeval tv; 5527 const struct timeval interval = {0, 250000}; /* 250ms */ 5528 5529 if (!(vi->flags & VI_INIT_DONE)) 5530 return; 5531 5532 getmicrotime(&tv); 5533 timevalsub(&tv, &interval); 5534 if (timevalcmp(&tv, &vi->last_refreshed, <)) 5535 return; 5536 5537 mtx_lock(&sc->reg_lock); 5538 t4_get_vi_stats(sc, vi->viid, &vi->stats); 5539 getmicrotime(&vi->last_refreshed); 5540 mtx_unlock(&sc->reg_lock); 5541 } 5542 5543 static void 5544 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi) 5545 { 5546 u_int i, v, tnl_cong_drops, bg_map; 5547 struct timeval tv; 5548 const struct timeval interval = {0, 250000}; /* 250ms */ 5549 5550 getmicrotime(&tv); 5551 timevalsub(&tv, &interval); 5552 if (timevalcmp(&tv, &pi->last_refreshed, <)) 5553 return; 5554 5555 tnl_cong_drops = 0; 5556 t4_get_port_stats(sc, pi->tx_chan, &pi->stats); 5557 bg_map = pi->mps_bg_map; 5558 while (bg_map) { 5559 i = ffs(bg_map) - 1; 5560 mtx_lock(&sc->reg_lock); 5561 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 1, 5562 A_TP_MIB_TNL_CNG_DROP_0 + i); 5563 mtx_unlock(&sc->reg_lock); 5564 tnl_cong_drops += v; 5565 bg_map &= ~(1 << i); 5566 } 5567 pi->tnl_cong_drops = tnl_cong_drops; 5568 getmicrotime(&pi->last_refreshed); 5569 } 5570 5571 static void 5572 cxgbe_tick(void *arg) 5573 { 5574 struct port_info *pi = arg; 5575 struct adapter *sc = pi->adapter; 5576 5577 PORT_LOCK_ASSERT_OWNED(pi); 5578 cxgbe_refresh_stats(sc, pi); 5579 5580 callout_schedule(&pi->tick, hz); 5581 } 5582 5583 void 5584 vi_tick(void *arg) 5585 { 5586 struct vi_info *vi = arg; 5587 struct adapter *sc = vi->pi->adapter; 5588 5589 vi_refresh_stats(sc, vi); 5590 5591 callout_schedule(&vi->tick, hz); 5592 } 5593 5594 /* 5595 * Should match fw_caps_config_<foo> enums in t4fw_interface.h 5596 */ 5597 static char *caps_decoder[] = { 5598 "\20\001IPMI\002NCSI", /* 0: NBM */ 5599 "\20\001PPP\002QFC\003DCBX", /* 1: link */ 5600 "\20\001INGRESS\002EGRESS", /* 2: switch */ 5601 "\20\001NIC\002VM\003IDS\004UM\005UM_ISGL" /* 3: NIC */ 5602 "\006HASHFILTER\007ETHOFLD", 5603 "\20\001TOE", /* 4: TOE */ 5604 "\20\001RDDP\002RDMAC", /* 5: RDMA */ 5605 "\20\001INITIATOR_PDU\002TARGET_PDU" /* 6: iSCSI */ 5606 "\003INITIATOR_CNXOFLD\004TARGET_CNXOFLD" 5607 "\005INITIATOR_SSNOFLD\006TARGET_SSNOFLD" 5608 "\007T10DIF" 5609 "\010INITIATOR_CMDOFLD\011TARGET_CMDOFLD", 5610 "\20\001LOOKASIDE\002TLSKEYS", /* 7: Crypto */ 5611 "\20\001INITIATOR\002TARGET\003CTRL_OFLD" /* 8: FCoE */ 5612 "\004PO_INITIATOR\005PO_TARGET", 5613 }; 5614 5615 void 5616 t4_sysctls(struct adapter *sc) 5617 { 5618 struct sysctl_ctx_list *ctx; 5619 struct sysctl_oid *oid; 5620 struct sysctl_oid_list *children, *c0; 5621 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 5622 5623 ctx = device_get_sysctl_ctx(sc->dev); 5624 5625 /* 5626 * dev.t4nex.X. 5627 */ 5628 oid = device_get_sysctl_tree(sc->dev); 5629 c0 = children = SYSCTL_CHILDREN(oid); 5630 5631 sc->sc_do_rxcopy = 1; 5632 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, 5633 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); 5634 5635 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 5636 sc->params.nports, "# of ports"); 5637 5638 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 5639 CTLTYPE_STRING | CTLFLAG_RD, doorbells, (uintptr_t)&sc->doorbells, 5640 sysctl_bitfield_8b, "A", "available doorbells"); 5641 5642 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 5643 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 5644 5645 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 5646 CTLTYPE_STRING | CTLFLAG_RD, sc->params.sge.timer_val, 5647 sizeof(sc->params.sge.timer_val), sysctl_int_array, "A", 5648 "interrupt holdoff timer values (us)"); 5649 5650 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 5651 CTLTYPE_STRING | CTLFLAG_RD, sc->params.sge.counter_val, 5652 sizeof(sc->params.sge.counter_val), sysctl_int_array, "A", 5653 "interrupt holdoff packet counter values"); 5654 5655 t4_sge_sysctls(sc, ctx, children); 5656 5657 sc->lro_timeout = 100; 5658 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, 5659 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); 5660 5661 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dflags", CTLFLAG_RW, 5662 &sc->debug_flags, 0, "flags to enable runtime debugging"); 5663 5664 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "tp_version", 5665 CTLFLAG_RD, sc->tp_version, 0, "TP microcode version"); 5666 5667 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 5668 CTLFLAG_RD, sc->fw_version, 0, "firmware version"); 5669 5670 if (sc->flags & IS_VF) 5671 return; 5672 5673 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 5674 NULL, chip_rev(sc), "chip hardware revision"); 5675 5676 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "sn", 5677 CTLFLAG_RD, sc->params.vpd.sn, 0, "serial number"); 5678 5679 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "pn", 5680 CTLFLAG_RD, sc->params.vpd.pn, 0, "part number"); 5681 5682 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "ec", 5683 CTLFLAG_RD, sc->params.vpd.ec, 0, "engineering change"); 5684 5685 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "md_version", 5686 CTLFLAG_RD, sc->params.vpd.md, 0, "manufacturing diags version"); 5687 5688 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "na", 5689 CTLFLAG_RD, sc->params.vpd.na, 0, "network address"); 5690 5691 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "er_version", CTLFLAG_RD, 5692 sc->er_version, 0, "expansion ROM version"); 5693 5694 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "bs_version", CTLFLAG_RD, 5695 sc->bs_version, 0, "bootstrap firmware version"); 5696 5697 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "scfg_version", CTLFLAG_RD, 5698 NULL, sc->params.scfg_vers, "serial config version"); 5699 5700 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "vpd_version", CTLFLAG_RD, 5701 NULL, sc->params.vpd_vers, "VPD version"); 5702 5703 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 5704 CTLFLAG_RD, sc->cfg_file, 0, "configuration file"); 5705 5706 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 5707 sc->cfcsum, "config file checksum"); 5708 5709 #define SYSCTL_CAP(name, n, text) \ 5710 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, #name, \ 5711 CTLTYPE_STRING | CTLFLAG_RD, caps_decoder[n], (uintptr_t)&sc->name, \ 5712 sysctl_bitfield_16b, "A", "available " text " capabilities") 5713 5714 SYSCTL_CAP(nbmcaps, 0, "NBM"); 5715 SYSCTL_CAP(linkcaps, 1, "link"); 5716 SYSCTL_CAP(switchcaps, 2, "switch"); 5717 SYSCTL_CAP(niccaps, 3, "NIC"); 5718 SYSCTL_CAP(toecaps, 4, "TCP offload"); 5719 SYSCTL_CAP(rdmacaps, 5, "RDMA"); 5720 SYSCTL_CAP(iscsicaps, 6, "iSCSI"); 5721 SYSCTL_CAP(cryptocaps, 7, "crypto"); 5722 SYSCTL_CAP(fcoecaps, 8, "FCoE"); 5723 #undef SYSCTL_CAP 5724 5725 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, 5726 NULL, sc->tids.nftids, "number of filters"); 5727 5728 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | 5729 CTLFLAG_RD, sc, 0, sysctl_temperature, "I", 5730 "chip temperature (in Celsius)"); 5731 5732 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "loadavg", CTLTYPE_STRING | 5733 CTLFLAG_RD, sc, 0, sysctl_loadavg, "A", 5734 "microprocessor load averages (debug firmwares only)"); 5735 5736 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_vdd", CTLFLAG_RD, 5737 &sc->params.core_vdd, 0, "core Vdd (in mV)"); 5738 5739 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "local_cpus", 5740 CTLTYPE_STRING | CTLFLAG_RD, sc, LOCAL_CPUS, 5741 sysctl_cpus, "A", "local CPUs"); 5742 5743 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "intr_cpus", 5744 CTLTYPE_STRING | CTLFLAG_RD, sc, INTR_CPUS, 5745 sysctl_cpus, "A", "preferred CPUs for interrupts"); 5746 5747 /* 5748 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 5749 */ 5750 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 5751 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 5752 "logs and miscellaneous information"); 5753 children = SYSCTL_CHILDREN(oid); 5754 5755 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 5756 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5757 sysctl_cctrl, "A", "congestion control"); 5758 5759 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 5760 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5761 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 5762 5763 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 5764 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 5765 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 5766 5767 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 5768 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 5769 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 5770 5771 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 5772 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 5773 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 5774 5775 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 5776 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 5777 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 5778 5779 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 5780 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 5781 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 5782 5783 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 5784 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5785 chip_id(sc) <= CHELSIO_T5 ? sysctl_cim_la : sysctl_cim_la_t6, 5786 "A", "CIM logic analyzer"); 5787 5788 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", 5789 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5790 sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); 5791 5792 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 5793 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 5794 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 5795 5796 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 5797 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 5798 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 5799 5800 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 5801 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 5802 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 5803 5804 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 5805 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 5806 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 5807 5808 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 5809 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 5810 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 5811 5812 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 5813 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 5814 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 5815 5816 if (chip_id(sc) > CHELSIO_T4) { 5817 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 5818 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 5819 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 5820 5821 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 5822 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 5823 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 5824 } 5825 5826 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", 5827 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5828 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); 5829 5830 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 5831 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5832 sysctl_cim_qcfg, "A", "CIM queue configuration"); 5833 5834 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 5835 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5836 sysctl_cpl_stats, "A", "CPL statistics"); 5837 5838 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 5839 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5840 sysctl_ddp_stats, "A", "non-TCP DDP statistics"); 5841 5842 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 5843 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5844 sysctl_devlog, "A", "firmware's device log"); 5845 5846 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 5847 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5848 sysctl_fcoe_stats, "A", "FCoE statistics"); 5849 5850 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 5851 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5852 sysctl_hw_sched, "A", "hardware scheduler "); 5853 5854 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 5855 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5856 sysctl_l2t, "A", "hardware L2 table"); 5857 5858 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "smt", 5859 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5860 sysctl_smt, "A", "hardware source MAC table"); 5861 5862 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 5863 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5864 sysctl_lb_stats, "A", "loopback statistics"); 5865 5866 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 5867 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5868 sysctl_meminfo, "A", "memory regions"); 5869 5870 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", 5871 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5872 chip_id(sc) <= CHELSIO_T5 ? sysctl_mps_tcam : sysctl_mps_tcam_t6, 5873 "A", "MPS TCAM entries"); 5874 5875 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 5876 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5877 sysctl_path_mtus, "A", "path MTUs"); 5878 5879 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 5880 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5881 sysctl_pm_stats, "A", "PM statistics"); 5882 5883 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 5884 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5885 sysctl_rdma_stats, "A", "RDMA statistics"); 5886 5887 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 5888 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5889 sysctl_tcp_stats, "A", "TCP statistics"); 5890 5891 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 5892 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5893 sysctl_tids, "A", "TID information"); 5894 5895 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 5896 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5897 sysctl_tp_err_stats, "A", "TP error statistics"); 5898 5899 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la_mask", 5900 CTLTYPE_INT | CTLFLAG_RW, sc, 0, sysctl_tp_la_mask, "I", 5901 "TP logic analyzer event capture mask"); 5902 5903 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", 5904 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5905 sysctl_tp_la, "A", "TP logic analyzer"); 5906 5907 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 5908 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5909 sysctl_tx_rate, "A", "Tx rate"); 5910 5911 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", 5912 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5913 sysctl_ulprx_la, "A", "ULPRX logic analyzer"); 5914 5915 if (chip_id(sc) >= CHELSIO_T5) { 5916 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 5917 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5918 sysctl_wcwr_stats, "A", "write combined work requests"); 5919 } 5920 5921 #ifdef TCP_OFFLOAD 5922 if (is_offload(sc)) { 5923 int i; 5924 char s[4]; 5925 5926 /* 5927 * dev.t4nex.X.toe. 5928 */ 5929 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 5930 NULL, "TOE parameters"); 5931 children = SYSCTL_CHILDREN(oid); 5932 5933 sc->tt.cong_algorithm = -1; 5934 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "cong_algorithm", 5935 CTLFLAG_RW, &sc->tt.cong_algorithm, 0, "congestion control " 5936 "(-1 = default, 0 = reno, 1 = tahoe, 2 = newreno, " 5937 "3 = highspeed)"); 5938 5939 sc->tt.sndbuf = 256 * 1024; 5940 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 5941 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 5942 5943 sc->tt.ddp = 0; 5944 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 5945 &sc->tt.ddp, 0, "DDP allowed"); 5946 5947 sc->tt.rx_coalesce = 1; 5948 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", 5949 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); 5950 5951 sc->tt.tls = 0; 5952 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tls", CTLFLAG_RW, 5953 &sc->tt.tls, 0, "Inline TLS allowed"); 5954 5955 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tls_rx_ports", 5956 CTLTYPE_INT | CTLFLAG_RW, sc, 0, sysctl_tls_rx_ports, 5957 "I", "TCP ports that use inline TLS+TOE RX"); 5958 5959 sc->tt.tx_align = 1; 5960 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align", 5961 CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload"); 5962 5963 sc->tt.tx_zcopy = 0; 5964 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_zcopy", 5965 CTLFLAG_RW, &sc->tt.tx_zcopy, 0, 5966 "Enable zero-copy aio_write(2)"); 5967 5968 sc->tt.cop_managed_offloading = !!t4_cop_managed_offloading; 5969 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 5970 "cop_managed_offloading", CTLFLAG_RW, 5971 &sc->tt.cop_managed_offloading, 0, 5972 "COP (Connection Offload Policy) controls all TOE offload"); 5973 5974 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "timer_tick", 5975 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, sysctl_tp_tick, "A", 5976 "TP timer tick (us)"); 5977 5978 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "timestamp_tick", 5979 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, sysctl_tp_tick, "A", 5980 "TCP timestamp tick (us)"); 5981 5982 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dack_tick", 5983 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, sysctl_tp_tick, "A", 5984 "DACK tick (us)"); 5985 5986 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dack_timer", 5987 CTLTYPE_UINT | CTLFLAG_RD, sc, 0, sysctl_tp_dack_timer, 5988 "IU", "DACK timer (us)"); 5989 5990 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rexmt_min", 5991 CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_RXT_MIN, 5992 sysctl_tp_timer, "LU", "Minimum retransmit interval (us)"); 5993 5994 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rexmt_max", 5995 CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_RXT_MAX, 5996 sysctl_tp_timer, "LU", "Maximum retransmit interval (us)"); 5997 5998 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "persist_min", 5999 CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_PERS_MIN, 6000 sysctl_tp_timer, "LU", "Persist timer min (us)"); 6001 6002 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "persist_max", 6003 CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_PERS_MAX, 6004 sysctl_tp_timer, "LU", "Persist timer max (us)"); 6005 6006 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "keepalive_idle", 6007 CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_KEEP_IDLE, 6008 sysctl_tp_timer, "LU", "Keepalive idle timer (us)"); 6009 6010 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "keepalive_interval", 6011 CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_KEEP_INTVL, 6012 sysctl_tp_timer, "LU", "Keepalive interval timer (us)"); 6013 6014 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "initial_srtt", 6015 CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_INIT_SRTT, 6016 sysctl_tp_timer, "LU", "Initial SRTT (us)"); 6017 6018 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "finwait2_timer", 6019 CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_FINWAIT2_TIMER, 6020 sysctl_tp_timer, "LU", "FINWAIT2 timer (us)"); 6021 6022 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "syn_rexmt_count", 6023 CTLTYPE_UINT | CTLFLAG_RD, sc, S_SYNSHIFTMAX, 6024 sysctl_tp_shift_cnt, "IU", 6025 "Number of SYN retransmissions before abort"); 6026 6027 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rexmt_count", 6028 CTLTYPE_UINT | CTLFLAG_RD, sc, S_RXTSHIFTMAXR2, 6029 sysctl_tp_shift_cnt, "IU", 6030 "Number of retransmissions before abort"); 6031 6032 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "keepalive_count", 6033 CTLTYPE_UINT | CTLFLAG_RD, sc, S_KEEPALIVEMAXR2, 6034 sysctl_tp_shift_cnt, "IU", 6035 "Number of keepalive probes before abort"); 6036 6037 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "rexmt_backoff", 6038 CTLFLAG_RD, NULL, "TOE retransmit backoffs"); 6039 children = SYSCTL_CHILDREN(oid); 6040 for (i = 0; i < 16; i++) { 6041 snprintf(s, sizeof(s), "%u", i); 6042 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, s, 6043 CTLTYPE_UINT | CTLFLAG_RD, sc, i, sysctl_tp_backoff, 6044 "IU", "TOE retransmit backoff"); 6045 } 6046 } 6047 #endif 6048 } 6049 6050 void 6051 vi_sysctls(struct vi_info *vi) 6052 { 6053 struct sysctl_ctx_list *ctx; 6054 struct sysctl_oid *oid; 6055 struct sysctl_oid_list *children; 6056 6057 ctx = device_get_sysctl_ctx(vi->dev); 6058 6059 /* 6060 * dev.v?(cxgbe|cxl).X. 6061 */ 6062 oid = device_get_sysctl_tree(vi->dev); 6063 children = SYSCTL_CHILDREN(oid); 6064 6065 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "viid", CTLFLAG_RD, NULL, 6066 vi->viid, "VI identifer"); 6067 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 6068 &vi->nrxq, 0, "# of rx queues"); 6069 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 6070 &vi->ntxq, 0, "# of tx queues"); 6071 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 6072 &vi->first_rxq, 0, "index of first rx queue"); 6073 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 6074 &vi->first_txq, 0, "index of first tx queue"); 6075 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "rss_base", CTLFLAG_RD, NULL, 6076 vi->rss_base, "start of RSS indirection table"); 6077 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "rss_size", CTLFLAG_RD, NULL, 6078 vi->rss_size, "size of RSS indirection table"); 6079 6080 if (IS_MAIN_VI(vi)) { 6081 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", 6082 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_noflowq, "IU", 6083 "Reserve queue 0 for non-flowid packets"); 6084 } 6085 6086 #ifdef TCP_OFFLOAD 6087 if (vi->nofldrxq != 0) { 6088 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 6089 &vi->nofldrxq, 0, 6090 "# of rx queues for offloaded TCP connections"); 6091 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 6092 &vi->nofldtxq, 0, 6093 "# of tx queues for offloaded TCP connections"); 6094 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 6095 CTLFLAG_RD, &vi->first_ofld_rxq, 0, 6096 "index of first TOE rx queue"); 6097 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 6098 CTLFLAG_RD, &vi->first_ofld_txq, 0, 6099 "index of first TOE tx queue"); 6100 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx_ofld", 6101 CTLTYPE_INT | CTLFLAG_RW, vi, 0, 6102 sysctl_holdoff_tmr_idx_ofld, "I", 6103 "holdoff timer index for TOE queues"); 6104 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx_ofld", 6105 CTLTYPE_INT | CTLFLAG_RW, vi, 0, 6106 sysctl_holdoff_pktc_idx_ofld, "I", 6107 "holdoff packet counter index for TOE queues"); 6108 } 6109 #endif 6110 #ifdef DEV_NETMAP 6111 if (vi->nnmrxq != 0) { 6112 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD, 6113 &vi->nnmrxq, 0, "# of netmap rx queues"); 6114 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD, 6115 &vi->nnmtxq, 0, "# of netmap tx queues"); 6116 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq", 6117 CTLFLAG_RD, &vi->first_nm_rxq, 0, 6118 "index of first netmap rx queue"); 6119 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq", 6120 CTLFLAG_RD, &vi->first_nm_txq, 0, 6121 "index of first netmap tx queue"); 6122 } 6123 #endif 6124 6125 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 6126 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_holdoff_tmr_idx, "I", 6127 "holdoff timer index"); 6128 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 6129 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_holdoff_pktc_idx, "I", 6130 "holdoff packet counter index"); 6131 6132 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 6133 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_qsize_rxq, "I", 6134 "rx queue size"); 6135 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 6136 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_qsize_txq, "I", 6137 "tx queue size"); 6138 } 6139 6140 static void 6141 cxgbe_sysctls(struct port_info *pi) 6142 { 6143 struct sysctl_ctx_list *ctx; 6144 struct sysctl_oid *oid; 6145 struct sysctl_oid_list *children, *children2; 6146 struct adapter *sc = pi->adapter; 6147 int i; 6148 char name[16]; 6149 static char *tc_flags = {"\20\1USER\2SYNC\3ASYNC\4ERR"}; 6150 6151 ctx = device_get_sysctl_ctx(pi->dev); 6152 6153 /* 6154 * dev.cxgbe.X. 6155 */ 6156 oid = device_get_sysctl_tree(pi->dev); 6157 children = SYSCTL_CHILDREN(oid); 6158 6159 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | 6160 CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); 6161 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { 6162 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 6163 CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I", 6164 "PHY temperature (in Celsius)"); 6165 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", 6166 CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I", 6167 "PHY firmware version"); 6168 } 6169 6170 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings", 6171 CTLTYPE_STRING | CTLFLAG_RW, pi, 0, sysctl_pause_settings, "A", 6172 "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)"); 6173 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fec", 6174 CTLTYPE_STRING | CTLFLAG_RW, pi, 0, sysctl_fec, "A", 6175 "Forward Error Correction (bit 0 = RS, bit 1 = BASER_RS)"); 6176 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "autoneg", 6177 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_autoneg, "I", 6178 "autonegotiation (-1 = not supported)"); 6179 6180 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "max_speed", CTLFLAG_RD, NULL, 6181 port_top_speed(pi), "max speed (in Gbps)"); 6182 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "mps_bg_map", CTLFLAG_RD, NULL, 6183 pi->mps_bg_map, "MPS buffer group map"); 6184 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_e_chan_map", CTLFLAG_RD, 6185 NULL, pi->rx_e_chan_map, "TP rx e-channel map"); 6186 6187 if (sc->flags & IS_VF) 6188 return; 6189 6190 /* 6191 * dev.(cxgbe|cxl).X.tc. 6192 */ 6193 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "tc", CTLFLAG_RD, NULL, 6194 "Tx scheduler traffic classes (cl_rl)"); 6195 children2 = SYSCTL_CHILDREN(oid); 6196 SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "pktsize", 6197 CTLFLAG_RW, &pi->sched_params->pktsize, 0, 6198 "pktsize for per-flow cl-rl (0 means up to the driver )"); 6199 SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "burstsize", 6200 CTLFLAG_RW, &pi->sched_params->burstsize, 0, 6201 "burstsize for per-flow cl-rl (0 means up to the driver)"); 6202 for (i = 0; i < sc->chip_params->nsched_cls; i++) { 6203 struct tx_cl_rl_params *tc = &pi->sched_params->cl_rl[i]; 6204 6205 snprintf(name, sizeof(name), "%d", i); 6206 children2 = SYSCTL_CHILDREN(SYSCTL_ADD_NODE(ctx, 6207 SYSCTL_CHILDREN(oid), OID_AUTO, name, CTLFLAG_RD, NULL, 6208 "traffic class")); 6209 SYSCTL_ADD_PROC(ctx, children2, OID_AUTO, "flags", 6210 CTLTYPE_STRING | CTLFLAG_RD, tc_flags, (uintptr_t)&tc->flags, 6211 sysctl_bitfield_8b, "A", "flags"); 6212 SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "refcount", 6213 CTLFLAG_RD, &tc->refcount, 0, "references to this class"); 6214 SYSCTL_ADD_PROC(ctx, children2, OID_AUTO, "params", 6215 CTLTYPE_STRING | CTLFLAG_RD, sc, (pi->port_id << 16) | i, 6216 sysctl_tc_params, "A", "traffic class parameters"); 6217 } 6218 6219 /* 6220 * dev.cxgbe.X.stats. 6221 */ 6222 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 6223 NULL, "port statistics"); 6224 children = SYSCTL_CHILDREN(oid); 6225 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD, 6226 &pi->tx_parse_error, 0, 6227 "# of tx packets with invalid length or # of segments"); 6228 6229 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 6230 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 6231 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ 6232 sysctl_handle_t4_reg64, "QU", desc) 6233 6234 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 6235 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 6236 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 6237 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 6238 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 6239 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 6240 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 6241 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 6242 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 6243 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 6244 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 6245 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 6246 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 6247 "# of tx frames in this range", 6248 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 6249 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 6250 "# of tx frames in this range", 6251 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 6252 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 6253 "# of tx frames in this range", 6254 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 6255 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 6256 "# of tx frames in this range", 6257 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 6258 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 6259 "# of tx frames in this range", 6260 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 6261 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 6262 "# of tx frames in this range", 6263 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 6264 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 6265 "# of tx frames in this range", 6266 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 6267 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 6268 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 6269 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 6270 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 6271 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 6272 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 6273 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 6274 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 6275 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 6276 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 6277 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 6278 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 6279 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 6280 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 6281 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 6282 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 6283 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 6284 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 6285 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 6286 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 6287 6288 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 6289 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 6290 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 6291 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 6292 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 6293 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 6294 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 6295 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 6296 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 6297 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 6298 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 6299 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 6300 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 6301 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 6302 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 6303 "# of frames received with bad FCS", 6304 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 6305 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 6306 "# of frames received with length error", 6307 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 6308 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 6309 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 6310 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 6311 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 6312 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 6313 "# of rx frames in this range", 6314 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 6315 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 6316 "# of rx frames in this range", 6317 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 6318 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 6319 "# of rx frames in this range", 6320 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 6321 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 6322 "# of rx frames in this range", 6323 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 6324 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 6325 "# of rx frames in this range", 6326 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 6327 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 6328 "# of rx frames in this range", 6329 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 6330 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 6331 "# of rx frames in this range", 6332 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 6333 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 6334 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 6335 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 6336 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 6337 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 6338 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 6339 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 6340 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 6341 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 6342 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 6343 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 6344 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 6345 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 6346 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 6347 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 6348 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 6349 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 6350 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 6351 6352 #undef SYSCTL_ADD_T4_REG64 6353 6354 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 6355 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 6356 &pi->stats.name, desc) 6357 6358 /* We get these from port_stats and they may be stale by up to 1s */ 6359 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 6360 "# drops due to buffer-group 0 overflows"); 6361 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 6362 "# drops due to buffer-group 1 overflows"); 6363 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 6364 "# drops due to buffer-group 2 overflows"); 6365 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 6366 "# drops due to buffer-group 3 overflows"); 6367 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 6368 "# of buffer-group 0 truncated packets"); 6369 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 6370 "# of buffer-group 1 truncated packets"); 6371 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 6372 "# of buffer-group 2 truncated packets"); 6373 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 6374 "# of buffer-group 3 truncated packets"); 6375 6376 #undef SYSCTL_ADD_T4_PORTSTAT 6377 6378 SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "tx_tls_records", 6379 CTLFLAG_RD, &pi->tx_tls_records, 6380 "# of TLS records transmitted"); 6381 SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "tx_tls_octets", 6382 CTLFLAG_RD, &pi->tx_tls_octets, 6383 "# of payload octets in transmitted TLS records"); 6384 SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "rx_tls_records", 6385 CTLFLAG_RD, &pi->rx_tls_records, 6386 "# of TLS records received"); 6387 SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "rx_tls_octets", 6388 CTLFLAG_RD, &pi->rx_tls_octets, 6389 "# of payload octets in received TLS records"); 6390 } 6391 6392 static int 6393 sysctl_int_array(SYSCTL_HANDLER_ARGS) 6394 { 6395 int rc, *i, space = 0; 6396 struct sbuf sb; 6397 6398 sbuf_new_for_sysctl(&sb, NULL, 64, req); 6399 for (i = arg1; arg2; arg2 -= sizeof(int), i++) { 6400 if (space) 6401 sbuf_printf(&sb, " "); 6402 sbuf_printf(&sb, "%d", *i); 6403 space = 1; 6404 } 6405 rc = sbuf_finish(&sb); 6406 sbuf_delete(&sb); 6407 return (rc); 6408 } 6409 6410 static int 6411 sysctl_bitfield_8b(SYSCTL_HANDLER_ARGS) 6412 { 6413 int rc; 6414 struct sbuf *sb; 6415 6416 rc = sysctl_wire_old_buffer(req, 0); 6417 if (rc != 0) 6418 return(rc); 6419 6420 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 6421 if (sb == NULL) 6422 return (ENOMEM); 6423 6424 sbuf_printf(sb, "%b", *(uint8_t *)(uintptr_t)arg2, (char *)arg1); 6425 rc = sbuf_finish(sb); 6426 sbuf_delete(sb); 6427 6428 return (rc); 6429 } 6430 6431 static int 6432 sysctl_bitfield_16b(SYSCTL_HANDLER_ARGS) 6433 { 6434 int rc; 6435 struct sbuf *sb; 6436 6437 rc = sysctl_wire_old_buffer(req, 0); 6438 if (rc != 0) 6439 return(rc); 6440 6441 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 6442 if (sb == NULL) 6443 return (ENOMEM); 6444 6445 sbuf_printf(sb, "%b", *(uint16_t *)(uintptr_t)arg2, (char *)arg1); 6446 rc = sbuf_finish(sb); 6447 sbuf_delete(sb); 6448 6449 return (rc); 6450 } 6451 6452 static int 6453 sysctl_btphy(SYSCTL_HANDLER_ARGS) 6454 { 6455 struct port_info *pi = arg1; 6456 int op = arg2; 6457 struct adapter *sc = pi->adapter; 6458 u_int v; 6459 int rc; 6460 6461 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, "t4btt"); 6462 if (rc) 6463 return (rc); 6464 /* XXX: magic numbers */ 6465 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, 6466 &v); 6467 end_synchronized_op(sc, 0); 6468 if (rc) 6469 return (rc); 6470 if (op == 0) 6471 v /= 256; 6472 6473 rc = sysctl_handle_int(oidp, &v, 0, req); 6474 return (rc); 6475 } 6476 6477 static int 6478 sysctl_noflowq(SYSCTL_HANDLER_ARGS) 6479 { 6480 struct vi_info *vi = arg1; 6481 int rc, val; 6482 6483 val = vi->rsrv_noflowq; 6484 rc = sysctl_handle_int(oidp, &val, 0, req); 6485 if (rc != 0 || req->newptr == NULL) 6486 return (rc); 6487 6488 if ((val >= 1) && (vi->ntxq > 1)) 6489 vi->rsrv_noflowq = 1; 6490 else 6491 vi->rsrv_noflowq = 0; 6492 6493 return (rc); 6494 } 6495 6496 static int 6497 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 6498 { 6499 struct vi_info *vi = arg1; 6500 struct adapter *sc = vi->pi->adapter; 6501 int idx, rc, i; 6502 struct sge_rxq *rxq; 6503 uint8_t v; 6504 6505 idx = vi->tmr_idx; 6506 6507 rc = sysctl_handle_int(oidp, &idx, 0, req); 6508 if (rc != 0 || req->newptr == NULL) 6509 return (rc); 6510 6511 if (idx < 0 || idx >= SGE_NTIMERS) 6512 return (EINVAL); 6513 6514 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 6515 "t4tmr"); 6516 if (rc) 6517 return (rc); 6518 6519 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(vi->pktc_idx != -1); 6520 for_each_rxq(vi, i, rxq) { 6521 #ifdef atomic_store_rel_8 6522 atomic_store_rel_8(&rxq->iq.intr_params, v); 6523 #else 6524 rxq->iq.intr_params = v; 6525 #endif 6526 } 6527 vi->tmr_idx = idx; 6528 6529 end_synchronized_op(sc, LOCK_HELD); 6530 return (0); 6531 } 6532 6533 static int 6534 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 6535 { 6536 struct vi_info *vi = arg1; 6537 struct adapter *sc = vi->pi->adapter; 6538 int idx, rc; 6539 6540 idx = vi->pktc_idx; 6541 6542 rc = sysctl_handle_int(oidp, &idx, 0, req); 6543 if (rc != 0 || req->newptr == NULL) 6544 return (rc); 6545 6546 if (idx < -1 || idx >= SGE_NCOUNTERS) 6547 return (EINVAL); 6548 6549 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 6550 "t4pktc"); 6551 if (rc) 6552 return (rc); 6553 6554 if (vi->flags & VI_INIT_DONE) 6555 rc = EBUSY; /* cannot be changed once the queues are created */ 6556 else 6557 vi->pktc_idx = idx; 6558 6559 end_synchronized_op(sc, LOCK_HELD); 6560 return (rc); 6561 } 6562 6563 static int 6564 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 6565 { 6566 struct vi_info *vi = arg1; 6567 struct adapter *sc = vi->pi->adapter; 6568 int qsize, rc; 6569 6570 qsize = vi->qsize_rxq; 6571 6572 rc = sysctl_handle_int(oidp, &qsize, 0, req); 6573 if (rc != 0 || req->newptr == NULL) 6574 return (rc); 6575 6576 if (qsize < 128 || (qsize & 7)) 6577 return (EINVAL); 6578 6579 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 6580 "t4rxqs"); 6581 if (rc) 6582 return (rc); 6583 6584 if (vi->flags & VI_INIT_DONE) 6585 rc = EBUSY; /* cannot be changed once the queues are created */ 6586 else 6587 vi->qsize_rxq = qsize; 6588 6589 end_synchronized_op(sc, LOCK_HELD); 6590 return (rc); 6591 } 6592 6593 static int 6594 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 6595 { 6596 struct vi_info *vi = arg1; 6597 struct adapter *sc = vi->pi->adapter; 6598 int qsize, rc; 6599 6600 qsize = vi->qsize_txq; 6601 6602 rc = sysctl_handle_int(oidp, &qsize, 0, req); 6603 if (rc != 0 || req->newptr == NULL) 6604 return (rc); 6605 6606 if (qsize < 128 || qsize > 65536) 6607 return (EINVAL); 6608 6609 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 6610 "t4txqs"); 6611 if (rc) 6612 return (rc); 6613 6614 if (vi->flags & VI_INIT_DONE) 6615 rc = EBUSY; /* cannot be changed once the queues are created */ 6616 else 6617 vi->qsize_txq = qsize; 6618 6619 end_synchronized_op(sc, LOCK_HELD); 6620 return (rc); 6621 } 6622 6623 static int 6624 sysctl_pause_settings(SYSCTL_HANDLER_ARGS) 6625 { 6626 struct port_info *pi = arg1; 6627 struct adapter *sc = pi->adapter; 6628 struct link_config *lc = &pi->link_cfg; 6629 int rc; 6630 6631 if (req->newptr == NULL) { 6632 struct sbuf *sb; 6633 static char *bits = "\20\1RX\2TX\3AUTO"; 6634 6635 rc = sysctl_wire_old_buffer(req, 0); 6636 if (rc != 0) 6637 return(rc); 6638 6639 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 6640 if (sb == NULL) 6641 return (ENOMEM); 6642 6643 if (lc->link_ok) { 6644 sbuf_printf(sb, "%b", (lc->fc & (PAUSE_TX | PAUSE_RX)) | 6645 (lc->requested_fc & PAUSE_AUTONEG), bits); 6646 } else { 6647 sbuf_printf(sb, "%b", lc->requested_fc & (PAUSE_TX | 6648 PAUSE_RX | PAUSE_AUTONEG), bits); 6649 } 6650 rc = sbuf_finish(sb); 6651 sbuf_delete(sb); 6652 } else { 6653 char s[2]; 6654 int n; 6655 6656 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX | 6657 PAUSE_AUTONEG)); 6658 s[1] = 0; 6659 6660 rc = sysctl_handle_string(oidp, s, sizeof(s), req); 6661 if (rc != 0) 6662 return(rc); 6663 6664 if (s[1] != 0) 6665 return (EINVAL); 6666 if (s[0] < '0' || s[0] > '9') 6667 return (EINVAL); /* not a number */ 6668 n = s[0] - '0'; 6669 if (n & ~(PAUSE_TX | PAUSE_RX | PAUSE_AUTONEG)) 6670 return (EINVAL); /* some other bit is set too */ 6671 6672 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, 6673 "t4PAUSE"); 6674 if (rc) 6675 return (rc); 6676 PORT_LOCK(pi); 6677 lc->requested_fc = n; 6678 fixup_link_config(pi); 6679 if (pi->up_vis > 0) 6680 rc = apply_link_config(pi); 6681 set_current_media(pi); 6682 PORT_UNLOCK(pi); 6683 end_synchronized_op(sc, 0); 6684 } 6685 6686 return (rc); 6687 } 6688 6689 static int 6690 sysctl_fec(SYSCTL_HANDLER_ARGS) 6691 { 6692 struct port_info *pi = arg1; 6693 struct adapter *sc = pi->adapter; 6694 struct link_config *lc = &pi->link_cfg; 6695 int rc; 6696 int8_t old; 6697 6698 if (req->newptr == NULL) { 6699 struct sbuf *sb; 6700 static char *bits = "\20\1RS\2BASE-R\3RSVD1\4RSVD2\5RSVD3\6AUTO"; 6701 6702 rc = sysctl_wire_old_buffer(req, 0); 6703 if (rc != 0) 6704 return(rc); 6705 6706 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 6707 if (sb == NULL) 6708 return (ENOMEM); 6709 6710 /* 6711 * Display the requested_fec when the link is down -- the actual 6712 * FEC makes sense only when the link is up. 6713 */ 6714 if (lc->link_ok) { 6715 sbuf_printf(sb, "%b", (lc->fec & M_FW_PORT_CAP32_FEC) | 6716 (lc->requested_fec & FEC_AUTO), bits); 6717 } else { 6718 sbuf_printf(sb, "%b", lc->requested_fec, bits); 6719 } 6720 rc = sbuf_finish(sb); 6721 sbuf_delete(sb); 6722 } else { 6723 char s[3]; 6724 int n; 6725 6726 snprintf(s, sizeof(s), "%d", 6727 lc->requested_fec == FEC_AUTO ? -1 : 6728 lc->requested_fec & M_FW_PORT_CAP32_FEC); 6729 6730 rc = sysctl_handle_string(oidp, s, sizeof(s), req); 6731 if (rc != 0) 6732 return(rc); 6733 6734 n = strtol(&s[0], NULL, 0); 6735 if (n < 0 || n & FEC_AUTO) 6736 n = FEC_AUTO; 6737 else { 6738 if (n & ~M_FW_PORT_CAP32_FEC) 6739 return (EINVAL);/* some other bit is set too */ 6740 if (!powerof2(n)) 6741 return (EINVAL);/* one bit can be set at most */ 6742 } 6743 6744 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, 6745 "t4fec"); 6746 if (rc) 6747 return (rc); 6748 PORT_LOCK(pi); 6749 old = lc->requested_fec; 6750 if (n == FEC_AUTO) 6751 lc->requested_fec = FEC_AUTO; 6752 else if (n == 0) 6753 lc->requested_fec = FEC_NONE; 6754 else { 6755 if ((lc->supported | V_FW_PORT_CAP32_FEC(n)) != 6756 lc->supported) { 6757 rc = ENOTSUP; 6758 goto done; 6759 } 6760 lc->requested_fec = n; 6761 } 6762 fixup_link_config(pi); 6763 if (pi->up_vis > 0) { 6764 rc = apply_link_config(pi); 6765 if (rc != 0) { 6766 lc->requested_fec = old; 6767 if (rc == FW_EPROTO) 6768 rc = ENOTSUP; 6769 } 6770 } 6771 done: 6772 PORT_UNLOCK(pi); 6773 end_synchronized_op(sc, 0); 6774 } 6775 6776 return (rc); 6777 } 6778 6779 static int 6780 sysctl_autoneg(SYSCTL_HANDLER_ARGS) 6781 { 6782 struct port_info *pi = arg1; 6783 struct adapter *sc = pi->adapter; 6784 struct link_config *lc = &pi->link_cfg; 6785 int rc, val; 6786 6787 if (lc->supported & FW_PORT_CAP32_ANEG) 6788 val = lc->requested_aneg == AUTONEG_DISABLE ? 0 : 1; 6789 else 6790 val = -1; 6791 rc = sysctl_handle_int(oidp, &val, 0, req); 6792 if (rc != 0 || req->newptr == NULL) 6793 return (rc); 6794 if (val == 0) 6795 val = AUTONEG_DISABLE; 6796 else if (val == 1) 6797 val = AUTONEG_ENABLE; 6798 else 6799 val = AUTONEG_AUTO; 6800 6801 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, 6802 "t4aneg"); 6803 if (rc) 6804 return (rc); 6805 PORT_LOCK(pi); 6806 if (val == AUTONEG_ENABLE && !(lc->supported & FW_PORT_CAP32_ANEG)) { 6807 rc = ENOTSUP; 6808 goto done; 6809 } 6810 lc->requested_aneg = val; 6811 fixup_link_config(pi); 6812 if (pi->up_vis > 0) 6813 rc = apply_link_config(pi); 6814 set_current_media(pi); 6815 done: 6816 PORT_UNLOCK(pi); 6817 end_synchronized_op(sc, 0); 6818 return (rc); 6819 } 6820 6821 static int 6822 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 6823 { 6824 struct adapter *sc = arg1; 6825 int reg = arg2; 6826 uint64_t val; 6827 6828 val = t4_read_reg64(sc, reg); 6829 6830 return (sysctl_handle_64(oidp, &val, 0, req)); 6831 } 6832 6833 static int 6834 sysctl_temperature(SYSCTL_HANDLER_ARGS) 6835 { 6836 struct adapter *sc = arg1; 6837 int rc, t; 6838 uint32_t param, val; 6839 6840 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); 6841 if (rc) 6842 return (rc); 6843 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 6844 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 6845 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); 6846 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 6847 end_synchronized_op(sc, 0); 6848 if (rc) 6849 return (rc); 6850 6851 /* unknown is returned as 0 but we display -1 in that case */ 6852 t = val == 0 ? -1 : val; 6853 6854 rc = sysctl_handle_int(oidp, &t, 0, req); 6855 return (rc); 6856 } 6857 6858 static int 6859 sysctl_loadavg(SYSCTL_HANDLER_ARGS) 6860 { 6861 struct adapter *sc = arg1; 6862 struct sbuf *sb; 6863 int rc; 6864 uint32_t param, val; 6865 6866 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4lavg"); 6867 if (rc) 6868 return (rc); 6869 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 6870 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_LOAD); 6871 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 6872 end_synchronized_op(sc, 0); 6873 if (rc) 6874 return (rc); 6875 6876 rc = sysctl_wire_old_buffer(req, 0); 6877 if (rc != 0) 6878 return (rc); 6879 6880 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6881 if (sb == NULL) 6882 return (ENOMEM); 6883 6884 if (val == 0xffffffff) { 6885 /* Only debug and custom firmwares report load averages. */ 6886 sbuf_printf(sb, "not available"); 6887 } else { 6888 sbuf_printf(sb, "%d %d %d", val & 0xff, (val >> 8) & 0xff, 6889 (val >> 16) & 0xff); 6890 } 6891 rc = sbuf_finish(sb); 6892 sbuf_delete(sb); 6893 6894 return (rc); 6895 } 6896 6897 static int 6898 sysctl_cctrl(SYSCTL_HANDLER_ARGS) 6899 { 6900 struct adapter *sc = arg1; 6901 struct sbuf *sb; 6902 int rc, i; 6903 uint16_t incr[NMTUS][NCCTRL_WIN]; 6904 static const char *dec_fac[] = { 6905 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 6906 "0.9375" 6907 }; 6908 6909 rc = sysctl_wire_old_buffer(req, 0); 6910 if (rc != 0) 6911 return (rc); 6912 6913 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6914 if (sb == NULL) 6915 return (ENOMEM); 6916 6917 t4_read_cong_tbl(sc, incr); 6918 6919 for (i = 0; i < NCCTRL_WIN; ++i) { 6920 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 6921 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 6922 incr[5][i], incr[6][i], incr[7][i]); 6923 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 6924 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 6925 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 6926 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 6927 } 6928 6929 rc = sbuf_finish(sb); 6930 sbuf_delete(sb); 6931 6932 return (rc); 6933 } 6934 6935 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 6936 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 6937 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 6938 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 6939 }; 6940 6941 static int 6942 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 6943 { 6944 struct adapter *sc = arg1; 6945 struct sbuf *sb; 6946 int rc, i, n, qid = arg2; 6947 uint32_t *buf, *p; 6948 char *qtype; 6949 u_int cim_num_obq = sc->chip_params->cim_num_obq; 6950 6951 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 6952 ("%s: bad qid %d\n", __func__, qid)); 6953 6954 if (qid < CIM_NUM_IBQ) { 6955 /* inbound queue */ 6956 qtype = "IBQ"; 6957 n = 4 * CIM_IBQ_SIZE; 6958 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 6959 rc = t4_read_cim_ibq(sc, qid, buf, n); 6960 } else { 6961 /* outbound queue */ 6962 qtype = "OBQ"; 6963 qid -= CIM_NUM_IBQ; 6964 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 6965 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 6966 rc = t4_read_cim_obq(sc, qid, buf, n); 6967 } 6968 6969 if (rc < 0) { 6970 rc = -rc; 6971 goto done; 6972 } 6973 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 6974 6975 rc = sysctl_wire_old_buffer(req, 0); 6976 if (rc != 0) 6977 goto done; 6978 6979 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 6980 if (sb == NULL) { 6981 rc = ENOMEM; 6982 goto done; 6983 } 6984 6985 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 6986 for (i = 0, p = buf; i < n; i += 16, p += 4) 6987 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 6988 p[2], p[3]); 6989 6990 rc = sbuf_finish(sb); 6991 sbuf_delete(sb); 6992 done: 6993 free(buf, M_CXGBE); 6994 return (rc); 6995 } 6996 6997 static int 6998 sysctl_cim_la(SYSCTL_HANDLER_ARGS) 6999 { 7000 struct adapter *sc = arg1; 7001 u_int cfg; 7002 struct sbuf *sb; 7003 uint32_t *buf, *p; 7004 int rc; 7005 7006 MPASS(chip_id(sc) <= CHELSIO_T5); 7007 7008 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 7009 if (rc != 0) 7010 return (rc); 7011 7012 rc = sysctl_wire_old_buffer(req, 0); 7013 if (rc != 0) 7014 return (rc); 7015 7016 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7017 if (sb == NULL) 7018 return (ENOMEM); 7019 7020 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 7021 M_ZERO | M_WAITOK); 7022 7023 rc = -t4_cim_read_la(sc, buf, NULL); 7024 if (rc != 0) 7025 goto done; 7026 7027 sbuf_printf(sb, "Status Data PC%s", 7028 cfg & F_UPDBGLACAPTPCONLY ? "" : 7029 " LS0Stat LS0Addr LS0Data"); 7030 7031 for (p = buf; p <= &buf[sc->params.cim_la_size - 8]; p += 8) { 7032 if (cfg & F_UPDBGLACAPTPCONLY) { 7033 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 7034 p[6], p[7]); 7035 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 7036 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 7037 p[4] & 0xff, p[5] >> 8); 7038 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 7039 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 7040 p[1] & 0xf, p[2] >> 4); 7041 } else { 7042 sbuf_printf(sb, 7043 "\n %02x %x%07x %x%07x %08x %08x " 7044 "%08x%08x%08x%08x", 7045 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 7046 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 7047 p[6], p[7]); 7048 } 7049 } 7050 7051 rc = sbuf_finish(sb); 7052 sbuf_delete(sb); 7053 done: 7054 free(buf, M_CXGBE); 7055 return (rc); 7056 } 7057 7058 static int 7059 sysctl_cim_la_t6(SYSCTL_HANDLER_ARGS) 7060 { 7061 struct adapter *sc = arg1; 7062 u_int cfg; 7063 struct sbuf *sb; 7064 uint32_t *buf, *p; 7065 int rc; 7066 7067 MPASS(chip_id(sc) > CHELSIO_T5); 7068 7069 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 7070 if (rc != 0) 7071 return (rc); 7072 7073 rc = sysctl_wire_old_buffer(req, 0); 7074 if (rc != 0) 7075 return (rc); 7076 7077 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7078 if (sb == NULL) 7079 return (ENOMEM); 7080 7081 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 7082 M_ZERO | M_WAITOK); 7083 7084 rc = -t4_cim_read_la(sc, buf, NULL); 7085 if (rc != 0) 7086 goto done; 7087 7088 sbuf_printf(sb, "Status Inst Data PC%s", 7089 cfg & F_UPDBGLACAPTPCONLY ? "" : 7090 " LS0Stat LS0Addr LS0Data LS1Stat LS1Addr LS1Data"); 7091 7092 for (p = buf; p <= &buf[sc->params.cim_la_size - 10]; p += 10) { 7093 if (cfg & F_UPDBGLACAPTPCONLY) { 7094 sbuf_printf(sb, "\n %02x %08x %08x %08x", 7095 p[3] & 0xff, p[2], p[1], p[0]); 7096 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x %02x%06x", 7097 (p[6] >> 8) & 0xff, p[6] & 0xff, p[5] >> 8, 7098 p[5] & 0xff, p[4] >> 8, p[4] & 0xff, p[3] >> 8); 7099 sbuf_printf(sb, "\n %02x %04x%04x %04x%04x %04x%04x", 7100 (p[9] >> 16) & 0xff, p[9] & 0xffff, p[8] >> 16, 7101 p[8] & 0xffff, p[7] >> 16, p[7] & 0xffff, 7102 p[6] >> 16); 7103 } else { 7104 sbuf_printf(sb, "\n %02x %04x%04x %04x%04x %04x%04x " 7105 "%08x %08x %08x %08x %08x %08x", 7106 (p[9] >> 16) & 0xff, 7107 p[9] & 0xffff, p[8] >> 16, 7108 p[8] & 0xffff, p[7] >> 16, 7109 p[7] & 0xffff, p[6] >> 16, 7110 p[2], p[1], p[0], p[5], p[4], p[3]); 7111 } 7112 } 7113 7114 rc = sbuf_finish(sb); 7115 sbuf_delete(sb); 7116 done: 7117 free(buf, M_CXGBE); 7118 return (rc); 7119 } 7120 7121 static int 7122 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) 7123 { 7124 struct adapter *sc = arg1; 7125 u_int i; 7126 struct sbuf *sb; 7127 uint32_t *buf, *p; 7128 int rc; 7129 7130 rc = sysctl_wire_old_buffer(req, 0); 7131 if (rc != 0) 7132 return (rc); 7133 7134 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7135 if (sb == NULL) 7136 return (ENOMEM); 7137 7138 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, 7139 M_ZERO | M_WAITOK); 7140 7141 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); 7142 p = buf; 7143 7144 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 7145 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], 7146 p[1], p[0]); 7147 } 7148 7149 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); 7150 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 7151 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", 7152 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, 7153 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, 7154 (p[1] >> 2) | ((p[2] & 3) << 30), 7155 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, 7156 p[0] & 1); 7157 } 7158 7159 rc = sbuf_finish(sb); 7160 sbuf_delete(sb); 7161 free(buf, M_CXGBE); 7162 return (rc); 7163 } 7164 7165 static int 7166 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) 7167 { 7168 struct adapter *sc = arg1; 7169 u_int i; 7170 struct sbuf *sb; 7171 uint32_t *buf, *p; 7172 int rc; 7173 7174 rc = sysctl_wire_old_buffer(req, 0); 7175 if (rc != 0) 7176 return (rc); 7177 7178 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7179 if (sb == NULL) 7180 return (ENOMEM); 7181 7182 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, 7183 M_ZERO | M_WAITOK); 7184 7185 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); 7186 p = buf; 7187 7188 sbuf_printf(sb, "Cntl ID DataBE Addr Data"); 7189 for (i = 0; i < CIM_PIFLA_SIZE; i++, p += 6) { 7190 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", 7191 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, 7192 p[4], p[3], p[2], p[1], p[0]); 7193 } 7194 7195 sbuf_printf(sb, "\n\nCntl ID Data"); 7196 for (i = 0; i < CIM_PIFLA_SIZE; i++, p += 6) { 7197 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", 7198 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); 7199 } 7200 7201 rc = sbuf_finish(sb); 7202 sbuf_delete(sb); 7203 free(buf, M_CXGBE); 7204 return (rc); 7205 } 7206 7207 static int 7208 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 7209 { 7210 struct adapter *sc = arg1; 7211 struct sbuf *sb; 7212 int rc, i; 7213 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 7214 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 7215 uint16_t thres[CIM_NUM_IBQ]; 7216 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 7217 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 7218 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 7219 7220 cim_num_obq = sc->chip_params->cim_num_obq; 7221 if (is_t4(sc)) { 7222 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 7223 obq_rdaddr = A_UP_OBQ_0_REALADDR; 7224 } else { 7225 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 7226 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 7227 } 7228 nq = CIM_NUM_IBQ + cim_num_obq; 7229 7230 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 7231 if (rc == 0) 7232 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 7233 if (rc != 0) 7234 return (rc); 7235 7236 t4_read_cimq_cfg(sc, base, size, thres); 7237 7238 rc = sysctl_wire_old_buffer(req, 0); 7239 if (rc != 0) 7240 return (rc); 7241 7242 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 7243 if (sb == NULL) 7244 return (ENOMEM); 7245 7246 sbuf_printf(sb, 7247 " Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 7248 7249 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 7250 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 7251 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 7252 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 7253 G_QUEREMFLITS(p[2]) * 16); 7254 for ( ; i < nq; i++, p += 4, wr += 2) 7255 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 7256 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 7257 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 7258 G_QUEREMFLITS(p[2]) * 16); 7259 7260 rc = sbuf_finish(sb); 7261 sbuf_delete(sb); 7262 7263 return (rc); 7264 } 7265 7266 static int 7267 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 7268 { 7269 struct adapter *sc = arg1; 7270 struct sbuf *sb; 7271 int rc; 7272 struct tp_cpl_stats stats; 7273 7274 rc = sysctl_wire_old_buffer(req, 0); 7275 if (rc != 0) 7276 return (rc); 7277 7278 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7279 if (sb == NULL) 7280 return (ENOMEM); 7281 7282 mtx_lock(&sc->reg_lock); 7283 t4_tp_get_cpl_stats(sc, &stats, 0); 7284 mtx_unlock(&sc->reg_lock); 7285 7286 if (sc->chip_params->nchan > 2) { 7287 sbuf_printf(sb, " channel 0 channel 1" 7288 " channel 2 channel 3"); 7289 sbuf_printf(sb, "\nCPL requests: %10u %10u %10u %10u", 7290 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 7291 sbuf_printf(sb, "\nCPL responses: %10u %10u %10u %10u", 7292 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 7293 } else { 7294 sbuf_printf(sb, " channel 0 channel 1"); 7295 sbuf_printf(sb, "\nCPL requests: %10u %10u", 7296 stats.req[0], stats.req[1]); 7297 sbuf_printf(sb, "\nCPL responses: %10u %10u", 7298 stats.rsp[0], stats.rsp[1]); 7299 } 7300 7301 rc = sbuf_finish(sb); 7302 sbuf_delete(sb); 7303 7304 return (rc); 7305 } 7306 7307 static int 7308 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 7309 { 7310 struct adapter *sc = arg1; 7311 struct sbuf *sb; 7312 int rc; 7313 struct tp_usm_stats stats; 7314 7315 rc = sysctl_wire_old_buffer(req, 0); 7316 if (rc != 0) 7317 return(rc); 7318 7319 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7320 if (sb == NULL) 7321 return (ENOMEM); 7322 7323 t4_get_usm_stats(sc, &stats, 1); 7324 7325 sbuf_printf(sb, "Frames: %u\n", stats.frames); 7326 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 7327 sbuf_printf(sb, "Drops: %u", stats.drops); 7328 7329 rc = sbuf_finish(sb); 7330 sbuf_delete(sb); 7331 7332 return (rc); 7333 } 7334 7335 static const char * const devlog_level_strings[] = { 7336 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 7337 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 7338 [FW_DEVLOG_LEVEL_ERR] = "ERR", 7339 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 7340 [FW_DEVLOG_LEVEL_INFO] = "INFO", 7341 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 7342 }; 7343 7344 static const char * const devlog_facility_strings[] = { 7345 [FW_DEVLOG_FACILITY_CORE] = "CORE", 7346 [FW_DEVLOG_FACILITY_CF] = "CF", 7347 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 7348 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 7349 [FW_DEVLOG_FACILITY_RES] = "RES", 7350 [FW_DEVLOG_FACILITY_HW] = "HW", 7351 [FW_DEVLOG_FACILITY_FLR] = "FLR", 7352 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 7353 [FW_DEVLOG_FACILITY_PHY] = "PHY", 7354 [FW_DEVLOG_FACILITY_MAC] = "MAC", 7355 [FW_DEVLOG_FACILITY_PORT] = "PORT", 7356 [FW_DEVLOG_FACILITY_VI] = "VI", 7357 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 7358 [FW_DEVLOG_FACILITY_ACL] = "ACL", 7359 [FW_DEVLOG_FACILITY_TM] = "TM", 7360 [FW_DEVLOG_FACILITY_QFC] = "QFC", 7361 [FW_DEVLOG_FACILITY_DCB] = "DCB", 7362 [FW_DEVLOG_FACILITY_ETH] = "ETH", 7363 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 7364 [FW_DEVLOG_FACILITY_RI] = "RI", 7365 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 7366 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 7367 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 7368 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE", 7369 [FW_DEVLOG_FACILITY_CHNET] = "CHNET", 7370 }; 7371 7372 static int 7373 sysctl_devlog(SYSCTL_HANDLER_ARGS) 7374 { 7375 struct adapter *sc = arg1; 7376 struct devlog_params *dparams = &sc->params.devlog; 7377 struct fw_devlog_e *buf, *e; 7378 int i, j, rc, nentries, first = 0; 7379 struct sbuf *sb; 7380 uint64_t ftstamp = UINT64_MAX; 7381 7382 if (dparams->addr == 0) 7383 return (ENXIO); 7384 7385 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 7386 if (buf == NULL) 7387 return (ENOMEM); 7388 7389 rc = read_via_memwin(sc, 1, dparams->addr, (void *)buf, dparams->size); 7390 if (rc != 0) 7391 goto done; 7392 7393 nentries = dparams->size / sizeof(struct fw_devlog_e); 7394 for (i = 0; i < nentries; i++) { 7395 e = &buf[i]; 7396 7397 if (e->timestamp == 0) 7398 break; /* end */ 7399 7400 e->timestamp = be64toh(e->timestamp); 7401 e->seqno = be32toh(e->seqno); 7402 for (j = 0; j < 8; j++) 7403 e->params[j] = be32toh(e->params[j]); 7404 7405 if (e->timestamp < ftstamp) { 7406 ftstamp = e->timestamp; 7407 first = i; 7408 } 7409 } 7410 7411 if (buf[first].timestamp == 0) 7412 goto done; /* nothing in the log */ 7413 7414 rc = sysctl_wire_old_buffer(req, 0); 7415 if (rc != 0) 7416 goto done; 7417 7418 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7419 if (sb == NULL) { 7420 rc = ENOMEM; 7421 goto done; 7422 } 7423 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 7424 "Seq#", "Tstamp", "Level", "Facility", "Message"); 7425 7426 i = first; 7427 do { 7428 e = &buf[i]; 7429 if (e->timestamp == 0) 7430 break; /* end */ 7431 7432 sbuf_printf(sb, "%10d %15ju %8s %8s ", 7433 e->seqno, e->timestamp, 7434 (e->level < nitems(devlog_level_strings) ? 7435 devlog_level_strings[e->level] : "UNKNOWN"), 7436 (e->facility < nitems(devlog_facility_strings) ? 7437 devlog_facility_strings[e->facility] : "UNKNOWN")); 7438 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 7439 e->params[2], e->params[3], e->params[4], 7440 e->params[5], e->params[6], e->params[7]); 7441 7442 if (++i == nentries) 7443 i = 0; 7444 } while (i != first); 7445 7446 rc = sbuf_finish(sb); 7447 sbuf_delete(sb); 7448 done: 7449 free(buf, M_CXGBE); 7450 return (rc); 7451 } 7452 7453 static int 7454 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 7455 { 7456 struct adapter *sc = arg1; 7457 struct sbuf *sb; 7458 int rc; 7459 struct tp_fcoe_stats stats[MAX_NCHAN]; 7460 int i, nchan = sc->chip_params->nchan; 7461 7462 rc = sysctl_wire_old_buffer(req, 0); 7463 if (rc != 0) 7464 return (rc); 7465 7466 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7467 if (sb == NULL) 7468 return (ENOMEM); 7469 7470 for (i = 0; i < nchan; i++) 7471 t4_get_fcoe_stats(sc, i, &stats[i], 1); 7472 7473 if (nchan > 2) { 7474 sbuf_printf(sb, " channel 0 channel 1" 7475 " channel 2 channel 3"); 7476 sbuf_printf(sb, "\noctetsDDP: %16ju %16ju %16ju %16ju", 7477 stats[0].octets_ddp, stats[1].octets_ddp, 7478 stats[2].octets_ddp, stats[3].octets_ddp); 7479 sbuf_printf(sb, "\nframesDDP: %16u %16u %16u %16u", 7480 stats[0].frames_ddp, stats[1].frames_ddp, 7481 stats[2].frames_ddp, stats[3].frames_ddp); 7482 sbuf_printf(sb, "\nframesDrop: %16u %16u %16u %16u", 7483 stats[0].frames_drop, stats[1].frames_drop, 7484 stats[2].frames_drop, stats[3].frames_drop); 7485 } else { 7486 sbuf_printf(sb, " channel 0 channel 1"); 7487 sbuf_printf(sb, "\noctetsDDP: %16ju %16ju", 7488 stats[0].octets_ddp, stats[1].octets_ddp); 7489 sbuf_printf(sb, "\nframesDDP: %16u %16u", 7490 stats[0].frames_ddp, stats[1].frames_ddp); 7491 sbuf_printf(sb, "\nframesDrop: %16u %16u", 7492 stats[0].frames_drop, stats[1].frames_drop); 7493 } 7494 7495 rc = sbuf_finish(sb); 7496 sbuf_delete(sb); 7497 7498 return (rc); 7499 } 7500 7501 static int 7502 sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 7503 { 7504 struct adapter *sc = arg1; 7505 struct sbuf *sb; 7506 int rc, i; 7507 unsigned int map, kbps, ipg, mode; 7508 unsigned int pace_tab[NTX_SCHED]; 7509 7510 rc = sysctl_wire_old_buffer(req, 0); 7511 if (rc != 0) 7512 return (rc); 7513 7514 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7515 if (sb == NULL) 7516 return (ENOMEM); 7517 7518 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 7519 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 7520 t4_read_pace_tbl(sc, pace_tab); 7521 7522 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 7523 "Class IPG (0.1 ns) Flow IPG (us)"); 7524 7525 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 7526 t4_get_tx_sched(sc, i, &kbps, &ipg, 1); 7527 sbuf_printf(sb, "\n %u %-5s %u ", i, 7528 (mode & (1 << i)) ? "flow" : "class", map & 3); 7529 if (kbps) 7530 sbuf_printf(sb, "%9u ", kbps); 7531 else 7532 sbuf_printf(sb, " disabled "); 7533 7534 if (ipg) 7535 sbuf_printf(sb, "%13u ", ipg); 7536 else 7537 sbuf_printf(sb, " disabled "); 7538 7539 if (pace_tab[i]) 7540 sbuf_printf(sb, "%10u", pace_tab[i]); 7541 else 7542 sbuf_printf(sb, " disabled"); 7543 } 7544 7545 rc = sbuf_finish(sb); 7546 sbuf_delete(sb); 7547 7548 return (rc); 7549 } 7550 7551 static int 7552 sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 7553 { 7554 struct adapter *sc = arg1; 7555 struct sbuf *sb; 7556 int rc, i, j; 7557 uint64_t *p0, *p1; 7558 struct lb_port_stats s[2]; 7559 static const char *stat_name[] = { 7560 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 7561 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 7562 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 7563 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 7564 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 7565 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 7566 "BG2FramesTrunc:", "BG3FramesTrunc:" 7567 }; 7568 7569 rc = sysctl_wire_old_buffer(req, 0); 7570 if (rc != 0) 7571 return (rc); 7572 7573 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7574 if (sb == NULL) 7575 return (ENOMEM); 7576 7577 memset(s, 0, sizeof(s)); 7578 7579 for (i = 0; i < sc->chip_params->nchan; i += 2) { 7580 t4_get_lb_stats(sc, i, &s[0]); 7581 t4_get_lb_stats(sc, i + 1, &s[1]); 7582 7583 p0 = &s[0].octets; 7584 p1 = &s[1].octets; 7585 sbuf_printf(sb, "%s Loopback %u" 7586 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 7587 7588 for (j = 0; j < nitems(stat_name); j++) 7589 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 7590 *p0++, *p1++); 7591 } 7592 7593 rc = sbuf_finish(sb); 7594 sbuf_delete(sb); 7595 7596 return (rc); 7597 } 7598 7599 static int 7600 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) 7601 { 7602 int rc = 0; 7603 struct port_info *pi = arg1; 7604 struct link_config *lc = &pi->link_cfg; 7605 struct sbuf *sb; 7606 7607 rc = sysctl_wire_old_buffer(req, 0); 7608 if (rc != 0) 7609 return(rc); 7610 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); 7611 if (sb == NULL) 7612 return (ENOMEM); 7613 7614 if (lc->link_ok || lc->link_down_rc == 255) 7615 sbuf_printf(sb, "n/a"); 7616 else 7617 sbuf_printf(sb, "%s", t4_link_down_rc_str(lc->link_down_rc)); 7618 7619 rc = sbuf_finish(sb); 7620 sbuf_delete(sb); 7621 7622 return (rc); 7623 } 7624 7625 struct mem_desc { 7626 unsigned int base; 7627 unsigned int limit; 7628 unsigned int idx; 7629 }; 7630 7631 static int 7632 mem_desc_cmp(const void *a, const void *b) 7633 { 7634 return ((const struct mem_desc *)a)->base - 7635 ((const struct mem_desc *)b)->base; 7636 } 7637 7638 static void 7639 mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 7640 unsigned int to) 7641 { 7642 unsigned int size; 7643 7644 if (from == to) 7645 return; 7646 7647 size = to - from + 1; 7648 if (size == 0) 7649 return; 7650 7651 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 7652 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 7653 } 7654 7655 static int 7656 sysctl_meminfo(SYSCTL_HANDLER_ARGS) 7657 { 7658 struct adapter *sc = arg1; 7659 struct sbuf *sb; 7660 int rc, i, n; 7661 uint32_t lo, hi, used, alloc; 7662 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 7663 static const char *region[] = { 7664 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 7665 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 7666 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 7667 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 7668 "RQUDP region:", "PBL region:", "TXPBL region:", 7669 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 7670 "On-chip queues:", "TLS keys:", 7671 }; 7672 struct mem_desc avail[4]; 7673 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 7674 struct mem_desc *md = mem; 7675 7676 rc = sysctl_wire_old_buffer(req, 0); 7677 if (rc != 0) 7678 return (rc); 7679 7680 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7681 if (sb == NULL) 7682 return (ENOMEM); 7683 7684 for (i = 0; i < nitems(mem); i++) { 7685 mem[i].limit = 0; 7686 mem[i].idx = i; 7687 } 7688 7689 /* Find and sort the populated memory ranges */ 7690 i = 0; 7691 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 7692 if (lo & F_EDRAM0_ENABLE) { 7693 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 7694 avail[i].base = G_EDRAM0_BASE(hi) << 20; 7695 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 7696 avail[i].idx = 0; 7697 i++; 7698 } 7699 if (lo & F_EDRAM1_ENABLE) { 7700 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 7701 avail[i].base = G_EDRAM1_BASE(hi) << 20; 7702 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 7703 avail[i].idx = 1; 7704 i++; 7705 } 7706 if (lo & F_EXT_MEM_ENABLE) { 7707 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 7708 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 7709 avail[i].limit = avail[i].base + 7710 (G_EXT_MEM_SIZE(hi) << 20); 7711 avail[i].idx = is_t5(sc) ? 3 : 2; /* Call it MC0 for T5 */ 7712 i++; 7713 } 7714 if (is_t5(sc) && lo & F_EXT_MEM1_ENABLE) { 7715 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 7716 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 7717 avail[i].limit = avail[i].base + 7718 (G_EXT_MEM1_SIZE(hi) << 20); 7719 avail[i].idx = 4; 7720 i++; 7721 } 7722 if (!i) /* no memory available */ 7723 return 0; 7724 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 7725 7726 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 7727 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 7728 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 7729 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 7730 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 7731 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 7732 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 7733 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 7734 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 7735 7736 /* the next few have explicit upper bounds */ 7737 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 7738 md->limit = md->base - 1 + 7739 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 7740 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 7741 md++; 7742 7743 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 7744 md->limit = md->base - 1 + 7745 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 7746 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 7747 md++; 7748 7749 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 7750 if (chip_id(sc) <= CHELSIO_T5) 7751 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 7752 else 7753 md->base = t4_read_reg(sc, A_LE_DB_HASH_TBL_BASE_ADDR); 7754 md->limit = 0; 7755 } else { 7756 md->base = 0; 7757 md->idx = nitems(region); /* hide it */ 7758 } 7759 md++; 7760 7761 #define ulp_region(reg) \ 7762 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 7763 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 7764 7765 ulp_region(RX_ISCSI); 7766 ulp_region(RX_TDDP); 7767 ulp_region(TX_TPT); 7768 ulp_region(RX_STAG); 7769 ulp_region(RX_RQ); 7770 ulp_region(RX_RQUDP); 7771 ulp_region(RX_PBL); 7772 ulp_region(TX_PBL); 7773 #undef ulp_region 7774 7775 md->base = 0; 7776 md->idx = nitems(region); 7777 if (!is_t4(sc)) { 7778 uint32_t size = 0; 7779 uint32_t sge_ctrl = t4_read_reg(sc, A_SGE_CONTROL2); 7780 uint32_t fifo_size = t4_read_reg(sc, A_SGE_DBVFIFO_SIZE); 7781 7782 if (is_t5(sc)) { 7783 if (sge_ctrl & F_VFIFO_ENABLE) 7784 size = G_DBVFIFO_SIZE(fifo_size); 7785 } else 7786 size = G_T6_DBVFIFO_SIZE(fifo_size); 7787 7788 if (size) { 7789 md->base = G_BASEADDR(t4_read_reg(sc, 7790 A_SGE_DBVFIFO_BADDR)); 7791 md->limit = md->base + (size << 2) - 1; 7792 } 7793 } 7794 md++; 7795 7796 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 7797 md->limit = 0; 7798 md++; 7799 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 7800 md->limit = 0; 7801 md++; 7802 7803 md->base = sc->vres.ocq.start; 7804 if (sc->vres.ocq.size) 7805 md->limit = md->base + sc->vres.ocq.size - 1; 7806 else 7807 md->idx = nitems(region); /* hide it */ 7808 md++; 7809 7810 md->base = sc->vres.key.start; 7811 if (sc->vres.key.size) 7812 md->limit = md->base + sc->vres.key.size - 1; 7813 else 7814 md->idx = nitems(region); /* hide it */ 7815 md++; 7816 7817 /* add any address-space holes, there can be up to 3 */ 7818 for (n = 0; n < i - 1; n++) 7819 if (avail[n].limit < avail[n + 1].base) 7820 (md++)->base = avail[n].limit; 7821 if (avail[n].limit) 7822 (md++)->base = avail[n].limit; 7823 7824 n = md - mem; 7825 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 7826 7827 for (lo = 0; lo < i; lo++) 7828 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 7829 avail[lo].limit - 1); 7830 7831 sbuf_printf(sb, "\n"); 7832 for (i = 0; i < n; i++) { 7833 if (mem[i].idx >= nitems(region)) 7834 continue; /* skip holes */ 7835 if (!mem[i].limit) 7836 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 7837 mem_region_show(sb, region[mem[i].idx], mem[i].base, 7838 mem[i].limit); 7839 } 7840 7841 sbuf_printf(sb, "\n"); 7842 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 7843 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 7844 mem_region_show(sb, "uP RAM:", lo, hi); 7845 7846 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 7847 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 7848 mem_region_show(sb, "uP Extmem2:", lo, hi); 7849 7850 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 7851 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 7852 G_PMRXMAXPAGE(lo), 7853 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 7854 (lo & F_PMRXNUMCHN) ? 2 : 1); 7855 7856 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 7857 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 7858 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 7859 G_PMTXMAXPAGE(lo), 7860 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 7861 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 7862 sbuf_printf(sb, "%u p-structs\n", 7863 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 7864 7865 for (i = 0; i < 4; i++) { 7866 if (chip_id(sc) > CHELSIO_T5) 7867 lo = t4_read_reg(sc, A_MPS_RX_MAC_BG_PG_CNT0 + i * 4); 7868 else 7869 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 7870 if (is_t5(sc)) { 7871 used = G_T5_USED(lo); 7872 alloc = G_T5_ALLOC(lo); 7873 } else { 7874 used = G_USED(lo); 7875 alloc = G_ALLOC(lo); 7876 } 7877 /* For T6 these are MAC buffer groups */ 7878 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 7879 i, used, alloc); 7880 } 7881 for (i = 0; i < sc->chip_params->nchan; i++) { 7882 if (chip_id(sc) > CHELSIO_T5) 7883 lo = t4_read_reg(sc, A_MPS_RX_LPBK_BG_PG_CNT0 + i * 4); 7884 else 7885 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 7886 if (is_t5(sc)) { 7887 used = G_T5_USED(lo); 7888 alloc = G_T5_ALLOC(lo); 7889 } else { 7890 used = G_USED(lo); 7891 alloc = G_ALLOC(lo); 7892 } 7893 /* For T6 these are MAC buffer groups */ 7894 sbuf_printf(sb, 7895 "\nLoopback %d using %u pages out of %u allocated", 7896 i, used, alloc); 7897 } 7898 7899 rc = sbuf_finish(sb); 7900 sbuf_delete(sb); 7901 7902 return (rc); 7903 } 7904 7905 static inline void 7906 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) 7907 { 7908 *mask = x | y; 7909 y = htobe64(y); 7910 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); 7911 } 7912 7913 static int 7914 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) 7915 { 7916 struct adapter *sc = arg1; 7917 struct sbuf *sb; 7918 int rc, i; 7919 7920 MPASS(chip_id(sc) <= CHELSIO_T5); 7921 7922 rc = sysctl_wire_old_buffer(req, 0); 7923 if (rc != 0) 7924 return (rc); 7925 7926 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7927 if (sb == NULL) 7928 return (ENOMEM); 7929 7930 sbuf_printf(sb, 7931 "Idx Ethernet address Mask Vld Ports PF" 7932 " VF Replication P0 P1 P2 P3 ML"); 7933 for (i = 0; i < sc->chip_params->mps_tcam_size; i++) { 7934 uint64_t tcamx, tcamy, mask; 7935 uint32_t cls_lo, cls_hi; 7936 uint8_t addr[ETHER_ADDR_LEN]; 7937 7938 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); 7939 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); 7940 if (tcamx & tcamy) 7941 continue; 7942 tcamxy2valmask(tcamx, tcamy, addr, &mask); 7943 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 7944 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 7945 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" 7946 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], 7947 addr[3], addr[4], addr[5], (uintmax_t)mask, 7948 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', 7949 G_PORTMAP(cls_hi), G_PF(cls_lo), 7950 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); 7951 7952 if (cls_lo & F_REPLICATE) { 7953 struct fw_ldst_cmd ldst_cmd; 7954 7955 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 7956 ldst_cmd.op_to_addrspace = 7957 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 7958 F_FW_CMD_REQUEST | F_FW_CMD_READ | 7959 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 7960 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 7961 ldst_cmd.u.mps.rplc.fid_idx = 7962 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 7963 V_FW_LDST_CMD_IDX(i)); 7964 7965 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 7966 "t4mps"); 7967 if (rc) 7968 break; 7969 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 7970 sizeof(ldst_cmd), &ldst_cmd); 7971 end_synchronized_op(sc, 0); 7972 7973 if (rc != 0) { 7974 sbuf_printf(sb, "%36d", rc); 7975 rc = 0; 7976 } else { 7977 sbuf_printf(sb, " %08x %08x %08x %08x", 7978 be32toh(ldst_cmd.u.mps.rplc.rplc127_96), 7979 be32toh(ldst_cmd.u.mps.rplc.rplc95_64), 7980 be32toh(ldst_cmd.u.mps.rplc.rplc63_32), 7981 be32toh(ldst_cmd.u.mps.rplc.rplc31_0)); 7982 } 7983 } else 7984 sbuf_printf(sb, "%36s", ""); 7985 7986 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), 7987 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), 7988 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); 7989 } 7990 7991 if (rc) 7992 (void) sbuf_finish(sb); 7993 else 7994 rc = sbuf_finish(sb); 7995 sbuf_delete(sb); 7996 7997 return (rc); 7998 } 7999 8000 static int 8001 sysctl_mps_tcam_t6(SYSCTL_HANDLER_ARGS) 8002 { 8003 struct adapter *sc = arg1; 8004 struct sbuf *sb; 8005 int rc, i; 8006 8007 MPASS(chip_id(sc) > CHELSIO_T5); 8008 8009 rc = sysctl_wire_old_buffer(req, 0); 8010 if (rc != 0) 8011 return (rc); 8012 8013 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 8014 if (sb == NULL) 8015 return (ENOMEM); 8016 8017 sbuf_printf(sb, "Idx Ethernet address Mask VNI Mask" 8018 " IVLAN Vld DIP_Hit Lookup Port Vld Ports PF VF" 8019 " Replication" 8020 " P0 P1 P2 P3 ML\n"); 8021 8022 for (i = 0; i < sc->chip_params->mps_tcam_size; i++) { 8023 uint8_t dip_hit, vlan_vld, lookup_type, port_num; 8024 uint16_t ivlan; 8025 uint64_t tcamx, tcamy, val, mask; 8026 uint32_t cls_lo, cls_hi, ctl, data2, vnix, vniy; 8027 uint8_t addr[ETHER_ADDR_LEN]; 8028 8029 ctl = V_CTLREQID(1) | V_CTLCMDTYPE(0) | V_CTLXYBITSEL(0); 8030 if (i < 256) 8031 ctl |= V_CTLTCAMINDEX(i) | V_CTLTCAMSEL(0); 8032 else 8033 ctl |= V_CTLTCAMINDEX(i - 256) | V_CTLTCAMSEL(1); 8034 t4_write_reg(sc, A_MPS_CLS_TCAM_DATA2_CTL, ctl); 8035 val = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA1_REQ_ID1); 8036 tcamy = G_DMACH(val) << 32; 8037 tcamy |= t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA0_REQ_ID1); 8038 data2 = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA2_REQ_ID1); 8039 lookup_type = G_DATALKPTYPE(data2); 8040 port_num = G_DATAPORTNUM(data2); 8041 if (lookup_type && lookup_type != M_DATALKPTYPE) { 8042 /* Inner header VNI */ 8043 vniy = ((data2 & F_DATAVIDH2) << 23) | 8044 (G_DATAVIDH1(data2) << 16) | G_VIDL(val); 8045 dip_hit = data2 & F_DATADIPHIT; 8046 vlan_vld = 0; 8047 } else { 8048 vniy = 0; 8049 dip_hit = 0; 8050 vlan_vld = data2 & F_DATAVIDH2; 8051 ivlan = G_VIDL(val); 8052 } 8053 8054 ctl |= V_CTLXYBITSEL(1); 8055 t4_write_reg(sc, A_MPS_CLS_TCAM_DATA2_CTL, ctl); 8056 val = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA1_REQ_ID1); 8057 tcamx = G_DMACH(val) << 32; 8058 tcamx |= t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA0_REQ_ID1); 8059 data2 = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA2_REQ_ID1); 8060 if (lookup_type && lookup_type != M_DATALKPTYPE) { 8061 /* Inner header VNI mask */ 8062 vnix = ((data2 & F_DATAVIDH2) << 23) | 8063 (G_DATAVIDH1(data2) << 16) | G_VIDL(val); 8064 } else 8065 vnix = 0; 8066 8067 if (tcamx & tcamy) 8068 continue; 8069 tcamxy2valmask(tcamx, tcamy, addr, &mask); 8070 8071 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 8072 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 8073 8074 if (lookup_type && lookup_type != M_DATALKPTYPE) { 8075 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x " 8076 "%012jx %06x %06x - - %3c" 8077 " 'I' %4x %3c %#x%4u%4d", i, addr[0], 8078 addr[1], addr[2], addr[3], addr[4], addr[5], 8079 (uintmax_t)mask, vniy, vnix, dip_hit ? 'Y' : 'N', 8080 port_num, cls_lo & F_T6_SRAM_VLD ? 'Y' : 'N', 8081 G_PORTMAP(cls_hi), G_T6_PF(cls_lo), 8082 cls_lo & F_T6_VF_VALID ? G_T6_VF(cls_lo) : -1); 8083 } else { 8084 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x " 8085 "%012jx - - ", i, addr[0], addr[1], 8086 addr[2], addr[3], addr[4], addr[5], 8087 (uintmax_t)mask); 8088 8089 if (vlan_vld) 8090 sbuf_printf(sb, "%4u Y ", ivlan); 8091 else 8092 sbuf_printf(sb, " - N "); 8093 8094 sbuf_printf(sb, "- %3c %4x %3c %#x%4u%4d", 8095 lookup_type ? 'I' : 'O', port_num, 8096 cls_lo & F_T6_SRAM_VLD ? 'Y' : 'N', 8097 G_PORTMAP(cls_hi), G_T6_PF(cls_lo), 8098 cls_lo & F_T6_VF_VALID ? G_T6_VF(cls_lo) : -1); 8099 } 8100 8101 8102 if (cls_lo & F_T6_REPLICATE) { 8103 struct fw_ldst_cmd ldst_cmd; 8104 8105 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 8106 ldst_cmd.op_to_addrspace = 8107 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 8108 F_FW_CMD_REQUEST | F_FW_CMD_READ | 8109 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 8110 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 8111 ldst_cmd.u.mps.rplc.fid_idx = 8112 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 8113 V_FW_LDST_CMD_IDX(i)); 8114 8115 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 8116 "t6mps"); 8117 if (rc) 8118 break; 8119 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 8120 sizeof(ldst_cmd), &ldst_cmd); 8121 end_synchronized_op(sc, 0); 8122 8123 if (rc != 0) { 8124 sbuf_printf(sb, "%72d", rc); 8125 rc = 0; 8126 } else { 8127 sbuf_printf(sb, " %08x %08x %08x %08x" 8128 " %08x %08x %08x %08x", 8129 be32toh(ldst_cmd.u.mps.rplc.rplc255_224), 8130 be32toh(ldst_cmd.u.mps.rplc.rplc223_192), 8131 be32toh(ldst_cmd.u.mps.rplc.rplc191_160), 8132 be32toh(ldst_cmd.u.mps.rplc.rplc159_128), 8133 be32toh(ldst_cmd.u.mps.rplc.rplc127_96), 8134 be32toh(ldst_cmd.u.mps.rplc.rplc95_64), 8135 be32toh(ldst_cmd.u.mps.rplc.rplc63_32), 8136 be32toh(ldst_cmd.u.mps.rplc.rplc31_0)); 8137 } 8138 } else 8139 sbuf_printf(sb, "%72s", ""); 8140 8141 sbuf_printf(sb, "%4u%3u%3u%3u %#x", 8142 G_T6_SRAM_PRIO0(cls_lo), G_T6_SRAM_PRIO1(cls_lo), 8143 G_T6_SRAM_PRIO2(cls_lo), G_T6_SRAM_PRIO3(cls_lo), 8144 (cls_lo >> S_T6_MULTILISTEN0) & 0xf); 8145 } 8146 8147 if (rc) 8148 (void) sbuf_finish(sb); 8149 else 8150 rc = sbuf_finish(sb); 8151 sbuf_delete(sb); 8152 8153 return (rc); 8154 } 8155 8156 static int 8157 sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 8158 { 8159 struct adapter *sc = arg1; 8160 struct sbuf *sb; 8161 int rc; 8162 uint16_t mtus[NMTUS]; 8163 8164 rc = sysctl_wire_old_buffer(req, 0); 8165 if (rc != 0) 8166 return (rc); 8167 8168 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 8169 if (sb == NULL) 8170 return (ENOMEM); 8171 8172 t4_read_mtu_tbl(sc, mtus, NULL); 8173 8174 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 8175 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 8176 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 8177 mtus[14], mtus[15]); 8178 8179 rc = sbuf_finish(sb); 8180 sbuf_delete(sb); 8181 8182 return (rc); 8183 } 8184 8185 static int 8186 sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 8187 { 8188 struct adapter *sc = arg1; 8189 struct sbuf *sb; 8190 int rc, i; 8191 uint32_t tx_cnt[MAX_PM_NSTATS], rx_cnt[MAX_PM_NSTATS]; 8192 uint64_t tx_cyc[MAX_PM_NSTATS], rx_cyc[MAX_PM_NSTATS]; 8193 static const char *tx_stats[MAX_PM_NSTATS] = { 8194 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:", 8195 "Tx FIFO wait", NULL, "Tx latency" 8196 }; 8197 static const char *rx_stats[MAX_PM_NSTATS] = { 8198 "Read:", "Write bypass:", "Write mem:", "Flush:", 8199 "Rx FIFO wait", NULL, "Rx latency" 8200 }; 8201 8202 rc = sysctl_wire_old_buffer(req, 0); 8203 if (rc != 0) 8204 return (rc); 8205 8206 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 8207 if (sb == NULL) 8208 return (ENOMEM); 8209 8210 t4_pmtx_get_stats(sc, tx_cnt, tx_cyc); 8211 t4_pmrx_get_stats(sc, rx_cnt, rx_cyc); 8212 8213 sbuf_printf(sb, " Tx pcmds Tx bytes"); 8214 for (i = 0; i < 4; i++) { 8215 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i], 8216 tx_cyc[i]); 8217 } 8218 8219 sbuf_printf(sb, "\n Rx pcmds Rx bytes"); 8220 for (i = 0; i < 4; i++) { 8221 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i], 8222 rx_cyc[i]); 8223 } 8224 8225 if (chip_id(sc) > CHELSIO_T5) { 8226 sbuf_printf(sb, 8227 "\n Total wait Total occupancy"); 8228 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i], 8229 tx_cyc[i]); 8230 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i], 8231 rx_cyc[i]); 8232 8233 i += 2; 8234 MPASS(i < nitems(tx_stats)); 8235 8236 sbuf_printf(sb, 8237 "\n Reads Total wait"); 8238 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i], 8239 tx_cyc[i]); 8240 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i], 8241 rx_cyc[i]); 8242 } 8243 8244 rc = sbuf_finish(sb); 8245 sbuf_delete(sb); 8246 8247 return (rc); 8248 } 8249 8250 static int 8251 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 8252 { 8253 struct adapter *sc = arg1; 8254 struct sbuf *sb; 8255 int rc; 8256 struct tp_rdma_stats stats; 8257 8258 rc = sysctl_wire_old_buffer(req, 0); 8259 if (rc != 0) 8260 return (rc); 8261 8262 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 8263 if (sb == NULL) 8264 return (ENOMEM); 8265 8266 mtx_lock(&sc->reg_lock); 8267 t4_tp_get_rdma_stats(sc, &stats, 0); 8268 mtx_unlock(&sc->reg_lock); 8269 8270 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 8271 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 8272 8273 rc = sbuf_finish(sb); 8274 sbuf_delete(sb); 8275 8276 return (rc); 8277 } 8278 8279 static int 8280 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 8281 { 8282 struct adapter *sc = arg1; 8283 struct sbuf *sb; 8284 int rc; 8285 struct tp_tcp_stats v4, v6; 8286 8287 rc = sysctl_wire_old_buffer(req, 0); 8288 if (rc != 0) 8289 return (rc); 8290 8291 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 8292 if (sb == NULL) 8293 return (ENOMEM); 8294 8295 mtx_lock(&sc->reg_lock); 8296 t4_tp_get_tcp_stats(sc, &v4, &v6, 0); 8297 mtx_unlock(&sc->reg_lock); 8298 8299 sbuf_printf(sb, 8300 " IP IPv6\n"); 8301 sbuf_printf(sb, "OutRsts: %20u %20u\n", 8302 v4.tcp_out_rsts, v6.tcp_out_rsts); 8303 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 8304 v4.tcp_in_segs, v6.tcp_in_segs); 8305 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 8306 v4.tcp_out_segs, v6.tcp_out_segs); 8307 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 8308 v4.tcp_retrans_segs, v6.tcp_retrans_segs); 8309 8310 rc = sbuf_finish(sb); 8311 sbuf_delete(sb); 8312 8313 return (rc); 8314 } 8315 8316 static int 8317 sysctl_tids(SYSCTL_HANDLER_ARGS) 8318 { 8319 struct adapter *sc = arg1; 8320 struct sbuf *sb; 8321 int rc; 8322 struct tid_info *t = &sc->tids; 8323 8324 rc = sysctl_wire_old_buffer(req, 0); 8325 if (rc != 0) 8326 return (rc); 8327 8328 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 8329 if (sb == NULL) 8330 return (ENOMEM); 8331 8332 if (t->natids) { 8333 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 8334 t->atids_in_use); 8335 } 8336 8337 if (t->nhpftids) { 8338 sbuf_printf(sb, "HPFTID range: %u-%u, in use: %u\n", 8339 t->hpftid_base, t->hpftid_end, t->hpftids_in_use); 8340 } 8341 8342 if (t->ntids) { 8343 sbuf_printf(sb, "TID range: "); 8344 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 8345 uint32_t b, hb; 8346 8347 if (chip_id(sc) <= CHELSIO_T5) { 8348 b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 8349 hb = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 8350 } else { 8351 b = t4_read_reg(sc, A_LE_DB_SRVR_START_INDEX); 8352 hb = t4_read_reg(sc, A_T6_LE_DB_HASH_TID_BASE); 8353 } 8354 8355 if (b) 8356 sbuf_printf(sb, "%u-%u, ", t->tid_base, b - 1); 8357 sbuf_printf(sb, "%u-%u", hb, t->ntids - 1); 8358 } else 8359 sbuf_printf(sb, "%u-%u", t->tid_base, t->ntids - 1); 8360 sbuf_printf(sb, ", in use: %u\n", 8361 atomic_load_acq_int(&t->tids_in_use)); 8362 } 8363 8364 if (t->nstids) { 8365 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 8366 t->stid_base + t->nstids - 1, t->stids_in_use); 8367 } 8368 8369 if (t->nftids) { 8370 sbuf_printf(sb, "FTID range: %u-%u, in use: %u\n", t->ftid_base, 8371 t->ftid_end, t->ftids_in_use); 8372 } 8373 8374 if (t->netids) { 8375 sbuf_printf(sb, "ETID range: %u-%u, in use: %u\n", t->etid_base, 8376 t->etid_base + t->netids - 1, t->etids_in_use); 8377 } 8378 8379 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 8380 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 8381 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 8382 8383 rc = sbuf_finish(sb); 8384 sbuf_delete(sb); 8385 8386 return (rc); 8387 } 8388 8389 static int 8390 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 8391 { 8392 struct adapter *sc = arg1; 8393 struct sbuf *sb; 8394 int rc; 8395 struct tp_err_stats stats; 8396 8397 rc = sysctl_wire_old_buffer(req, 0); 8398 if (rc != 0) 8399 return (rc); 8400 8401 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 8402 if (sb == NULL) 8403 return (ENOMEM); 8404 8405 mtx_lock(&sc->reg_lock); 8406 t4_tp_get_err_stats(sc, &stats, 0); 8407 mtx_unlock(&sc->reg_lock); 8408 8409 if (sc->chip_params->nchan > 2) { 8410 sbuf_printf(sb, " channel 0 channel 1" 8411 " channel 2 channel 3\n"); 8412 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 8413 stats.mac_in_errs[0], stats.mac_in_errs[1], 8414 stats.mac_in_errs[2], stats.mac_in_errs[3]); 8415 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 8416 stats.hdr_in_errs[0], stats.hdr_in_errs[1], 8417 stats.hdr_in_errs[2], stats.hdr_in_errs[3]); 8418 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 8419 stats.tcp_in_errs[0], stats.tcp_in_errs[1], 8420 stats.tcp_in_errs[2], stats.tcp_in_errs[3]); 8421 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 8422 stats.tcp6_in_errs[0], stats.tcp6_in_errs[1], 8423 stats.tcp6_in_errs[2], stats.tcp6_in_errs[3]); 8424 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 8425 stats.tnl_cong_drops[0], stats.tnl_cong_drops[1], 8426 stats.tnl_cong_drops[2], stats.tnl_cong_drops[3]); 8427 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 8428 stats.tnl_tx_drops[0], stats.tnl_tx_drops[1], 8429 stats.tnl_tx_drops[2], stats.tnl_tx_drops[3]); 8430 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 8431 stats.ofld_vlan_drops[0], stats.ofld_vlan_drops[1], 8432 stats.ofld_vlan_drops[2], stats.ofld_vlan_drops[3]); 8433 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 8434 stats.ofld_chan_drops[0], stats.ofld_chan_drops[1], 8435 stats.ofld_chan_drops[2], stats.ofld_chan_drops[3]); 8436 } else { 8437 sbuf_printf(sb, " channel 0 channel 1\n"); 8438 sbuf_printf(sb, "macInErrs: %10u %10u\n", 8439 stats.mac_in_errs[0], stats.mac_in_errs[1]); 8440 sbuf_printf(sb, "hdrInErrs: %10u %10u\n", 8441 stats.hdr_in_errs[0], stats.hdr_in_errs[1]); 8442 sbuf_printf(sb, "tcpInErrs: %10u %10u\n", 8443 stats.tcp_in_errs[0], stats.tcp_in_errs[1]); 8444 sbuf_printf(sb, "tcp6InErrs: %10u %10u\n", 8445 stats.tcp6_in_errs[0], stats.tcp6_in_errs[1]); 8446 sbuf_printf(sb, "tnlCongDrops: %10u %10u\n", 8447 stats.tnl_cong_drops[0], stats.tnl_cong_drops[1]); 8448 sbuf_printf(sb, "tnlTxDrops: %10u %10u\n", 8449 stats.tnl_tx_drops[0], stats.tnl_tx_drops[1]); 8450 sbuf_printf(sb, "ofldVlanDrops: %10u %10u\n", 8451 stats.ofld_vlan_drops[0], stats.ofld_vlan_drops[1]); 8452 sbuf_printf(sb, "ofldChanDrops: %10u %10u\n\n", 8453 stats.ofld_chan_drops[0], stats.ofld_chan_drops[1]); 8454 } 8455 8456 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 8457 stats.ofld_no_neigh, stats.ofld_cong_defer); 8458 8459 rc = sbuf_finish(sb); 8460 sbuf_delete(sb); 8461 8462 return (rc); 8463 } 8464 8465 static int 8466 sysctl_tp_la_mask(SYSCTL_HANDLER_ARGS) 8467 { 8468 struct adapter *sc = arg1; 8469 struct tp_params *tpp = &sc->params.tp; 8470 u_int mask; 8471 int rc; 8472 8473 mask = tpp->la_mask >> 16; 8474 rc = sysctl_handle_int(oidp, &mask, 0, req); 8475 if (rc != 0 || req->newptr == NULL) 8476 return (rc); 8477 if (mask > 0xffff) 8478 return (EINVAL); 8479 tpp->la_mask = mask << 16; 8480 t4_set_reg_field(sc, A_TP_DBG_LA_CONFIG, 0xffff0000U, tpp->la_mask); 8481 8482 return (0); 8483 } 8484 8485 struct field_desc { 8486 const char *name; 8487 u_int start; 8488 u_int width; 8489 }; 8490 8491 static void 8492 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) 8493 { 8494 char buf[32]; 8495 int line_size = 0; 8496 8497 while (f->name) { 8498 uint64_t mask = (1ULL << f->width) - 1; 8499 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, 8500 ((uintmax_t)v >> f->start) & mask); 8501 8502 if (line_size + len >= 79) { 8503 line_size = 8; 8504 sbuf_printf(sb, "\n "); 8505 } 8506 sbuf_printf(sb, "%s ", buf); 8507 line_size += len + 1; 8508 f++; 8509 } 8510 sbuf_printf(sb, "\n"); 8511 } 8512 8513 static const struct field_desc tp_la0[] = { 8514 { "RcfOpCodeOut", 60, 4 }, 8515 { "State", 56, 4 }, 8516 { "WcfState", 52, 4 }, 8517 { "RcfOpcSrcOut", 50, 2 }, 8518 { "CRxError", 49, 1 }, 8519 { "ERxError", 48, 1 }, 8520 { "SanityFailed", 47, 1 }, 8521 { "SpuriousMsg", 46, 1 }, 8522 { "FlushInputMsg", 45, 1 }, 8523 { "FlushInputCpl", 44, 1 }, 8524 { "RssUpBit", 43, 1 }, 8525 { "RssFilterHit", 42, 1 }, 8526 { "Tid", 32, 10 }, 8527 { "InitTcb", 31, 1 }, 8528 { "LineNumber", 24, 7 }, 8529 { "Emsg", 23, 1 }, 8530 { "EdataOut", 22, 1 }, 8531 { "Cmsg", 21, 1 }, 8532 { "CdataOut", 20, 1 }, 8533 { "EreadPdu", 19, 1 }, 8534 { "CreadPdu", 18, 1 }, 8535 { "TunnelPkt", 17, 1 }, 8536 { "RcfPeerFin", 16, 1 }, 8537 { "RcfReasonOut", 12, 4 }, 8538 { "TxCchannel", 10, 2 }, 8539 { "RcfTxChannel", 8, 2 }, 8540 { "RxEchannel", 6, 2 }, 8541 { "RcfRxChannel", 5, 1 }, 8542 { "RcfDataOutSrdy", 4, 1 }, 8543 { "RxDvld", 3, 1 }, 8544 { "RxOoDvld", 2, 1 }, 8545 { "RxCongestion", 1, 1 }, 8546 { "TxCongestion", 0, 1 }, 8547 { NULL } 8548 }; 8549 8550 static const struct field_desc tp_la1[] = { 8551 { "CplCmdIn", 56, 8 }, 8552 { "CplCmdOut", 48, 8 }, 8553 { "ESynOut", 47, 1 }, 8554 { "EAckOut", 46, 1 }, 8555 { "EFinOut", 45, 1 }, 8556 { "ERstOut", 44, 1 }, 8557 { "SynIn", 43, 1 }, 8558 { "AckIn", 42, 1 }, 8559 { "FinIn", 41, 1 }, 8560 { "RstIn", 40, 1 }, 8561 { "DataIn", 39, 1 }, 8562 { "DataInVld", 38, 1 }, 8563 { "PadIn", 37, 1 }, 8564 { "RxBufEmpty", 36, 1 }, 8565 { "RxDdp", 35, 1 }, 8566 { "RxFbCongestion", 34, 1 }, 8567 { "TxFbCongestion", 33, 1 }, 8568 { "TxPktSumSrdy", 32, 1 }, 8569 { "RcfUlpType", 28, 4 }, 8570 { "Eread", 27, 1 }, 8571 { "Ebypass", 26, 1 }, 8572 { "Esave", 25, 1 }, 8573 { "Static0", 24, 1 }, 8574 { "Cread", 23, 1 }, 8575 { "Cbypass", 22, 1 }, 8576 { "Csave", 21, 1 }, 8577 { "CPktOut", 20, 1 }, 8578 { "RxPagePoolFull", 18, 2 }, 8579 { "RxLpbkPkt", 17, 1 }, 8580 { "TxLpbkPkt", 16, 1 }, 8581 { "RxVfValid", 15, 1 }, 8582 { "SynLearned", 14, 1 }, 8583 { "SetDelEntry", 13, 1 }, 8584 { "SetInvEntry", 12, 1 }, 8585 { "CpcmdDvld", 11, 1 }, 8586 { "CpcmdSave", 10, 1 }, 8587 { "RxPstructsFull", 8, 2 }, 8588 { "EpcmdDvld", 7, 1 }, 8589 { "EpcmdFlush", 6, 1 }, 8590 { "EpcmdTrimPrefix", 5, 1 }, 8591 { "EpcmdTrimPostfix", 4, 1 }, 8592 { "ERssIp4Pkt", 3, 1 }, 8593 { "ERssIp6Pkt", 2, 1 }, 8594 { "ERssTcpUdpPkt", 1, 1 }, 8595 { "ERssFceFipPkt", 0, 1 }, 8596 { NULL } 8597 }; 8598 8599 static const struct field_desc tp_la2[] = { 8600 { "CplCmdIn", 56, 8 }, 8601 { "MpsVfVld", 55, 1 }, 8602 { "MpsPf", 52, 3 }, 8603 { "MpsVf", 44, 8 }, 8604 { "SynIn", 43, 1 }, 8605 { "AckIn", 42, 1 }, 8606 { "FinIn", 41, 1 }, 8607 { "RstIn", 40, 1 }, 8608 { "DataIn", 39, 1 }, 8609 { "DataInVld", 38, 1 }, 8610 { "PadIn", 37, 1 }, 8611 { "RxBufEmpty", 36, 1 }, 8612 { "RxDdp", 35, 1 }, 8613 { "RxFbCongestion", 34, 1 }, 8614 { "TxFbCongestion", 33, 1 }, 8615 { "TxPktSumSrdy", 32, 1 }, 8616 { "RcfUlpType", 28, 4 }, 8617 { "Eread", 27, 1 }, 8618 { "Ebypass", 26, 1 }, 8619 { "Esave", 25, 1 }, 8620 { "Static0", 24, 1 }, 8621 { "Cread", 23, 1 }, 8622 { "Cbypass", 22, 1 }, 8623 { "Csave", 21, 1 }, 8624 { "CPktOut", 20, 1 }, 8625 { "RxPagePoolFull", 18, 2 }, 8626 { "RxLpbkPkt", 17, 1 }, 8627 { "TxLpbkPkt", 16, 1 }, 8628 { "RxVfValid", 15, 1 }, 8629 { "SynLearned", 14, 1 }, 8630 { "SetDelEntry", 13, 1 }, 8631 { "SetInvEntry", 12, 1 }, 8632 { "CpcmdDvld", 11, 1 }, 8633 { "CpcmdSave", 10, 1 }, 8634 { "RxPstructsFull", 8, 2 }, 8635 { "EpcmdDvld", 7, 1 }, 8636 { "EpcmdFlush", 6, 1 }, 8637 { "EpcmdTrimPrefix", 5, 1 }, 8638 { "EpcmdTrimPostfix", 4, 1 }, 8639 { "ERssIp4Pkt", 3, 1 }, 8640 { "ERssIp6Pkt", 2, 1 }, 8641 { "ERssTcpUdpPkt", 1, 1 }, 8642 { "ERssFceFipPkt", 0, 1 }, 8643 { NULL } 8644 }; 8645 8646 static void 8647 tp_la_show(struct sbuf *sb, uint64_t *p, int idx) 8648 { 8649 8650 field_desc_show(sb, *p, tp_la0); 8651 } 8652 8653 static void 8654 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) 8655 { 8656 8657 if (idx) 8658 sbuf_printf(sb, "\n"); 8659 field_desc_show(sb, p[0], tp_la0); 8660 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 8661 field_desc_show(sb, p[1], tp_la0); 8662 } 8663 8664 static void 8665 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) 8666 { 8667 8668 if (idx) 8669 sbuf_printf(sb, "\n"); 8670 field_desc_show(sb, p[0], tp_la0); 8671 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 8672 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); 8673 } 8674 8675 static int 8676 sysctl_tp_la(SYSCTL_HANDLER_ARGS) 8677 { 8678 struct adapter *sc = arg1; 8679 struct sbuf *sb; 8680 uint64_t *buf, *p; 8681 int rc; 8682 u_int i, inc; 8683 void (*show_func)(struct sbuf *, uint64_t *, int); 8684 8685 rc = sysctl_wire_old_buffer(req, 0); 8686 if (rc != 0) 8687 return (rc); 8688 8689 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 8690 if (sb == NULL) 8691 return (ENOMEM); 8692 8693 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); 8694 8695 t4_tp_read_la(sc, buf, NULL); 8696 p = buf; 8697 8698 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { 8699 case 2: 8700 inc = 2; 8701 show_func = tp_la_show2; 8702 break; 8703 case 3: 8704 inc = 2; 8705 show_func = tp_la_show3; 8706 break; 8707 default: 8708 inc = 1; 8709 show_func = tp_la_show; 8710 } 8711 8712 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) 8713 (*show_func)(sb, p, i); 8714 8715 rc = sbuf_finish(sb); 8716 sbuf_delete(sb); 8717 free(buf, M_CXGBE); 8718 return (rc); 8719 } 8720 8721 static int 8722 sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 8723 { 8724 struct adapter *sc = arg1; 8725 struct sbuf *sb; 8726 int rc; 8727 u64 nrate[MAX_NCHAN], orate[MAX_NCHAN]; 8728 8729 rc = sysctl_wire_old_buffer(req, 0); 8730 if (rc != 0) 8731 return (rc); 8732 8733 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 8734 if (sb == NULL) 8735 return (ENOMEM); 8736 8737 t4_get_chan_txrate(sc, nrate, orate); 8738 8739 if (sc->chip_params->nchan > 2) { 8740 sbuf_printf(sb, " channel 0 channel 1" 8741 " channel 2 channel 3\n"); 8742 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 8743 nrate[0], nrate[1], nrate[2], nrate[3]); 8744 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 8745 orate[0], orate[1], orate[2], orate[3]); 8746 } else { 8747 sbuf_printf(sb, " channel 0 channel 1\n"); 8748 sbuf_printf(sb, "NIC B/s: %10ju %10ju\n", 8749 nrate[0], nrate[1]); 8750 sbuf_printf(sb, "Offload B/s: %10ju %10ju", 8751 orate[0], orate[1]); 8752 } 8753 8754 rc = sbuf_finish(sb); 8755 sbuf_delete(sb); 8756 8757 return (rc); 8758 } 8759 8760 static int 8761 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) 8762 { 8763 struct adapter *sc = arg1; 8764 struct sbuf *sb; 8765 uint32_t *buf, *p; 8766 int rc, i; 8767 8768 rc = sysctl_wire_old_buffer(req, 0); 8769 if (rc != 0) 8770 return (rc); 8771 8772 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 8773 if (sb == NULL) 8774 return (ENOMEM); 8775 8776 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, 8777 M_ZERO | M_WAITOK); 8778 8779 t4_ulprx_read_la(sc, buf); 8780 p = buf; 8781 8782 sbuf_printf(sb, " Pcmd Type Message" 8783 " Data"); 8784 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { 8785 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", 8786 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); 8787 } 8788 8789 rc = sbuf_finish(sb); 8790 sbuf_delete(sb); 8791 free(buf, M_CXGBE); 8792 return (rc); 8793 } 8794 8795 static int 8796 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 8797 { 8798 struct adapter *sc = arg1; 8799 struct sbuf *sb; 8800 int rc, v; 8801 8802 MPASS(chip_id(sc) >= CHELSIO_T5); 8803 8804 rc = sysctl_wire_old_buffer(req, 0); 8805 if (rc != 0) 8806 return (rc); 8807 8808 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 8809 if (sb == NULL) 8810 return (ENOMEM); 8811 8812 v = t4_read_reg(sc, A_SGE_STAT_CFG); 8813 if (G_STATSOURCE_T5(v) == 7) { 8814 int mode; 8815 8816 mode = is_t5(sc) ? G_STATMODE(v) : G_T6_STATMODE(v); 8817 if (mode == 0) { 8818 sbuf_printf(sb, "total %d, incomplete %d", 8819 t4_read_reg(sc, A_SGE_STAT_TOTAL), 8820 t4_read_reg(sc, A_SGE_STAT_MATCH)); 8821 } else if (mode == 1) { 8822 sbuf_printf(sb, "total %d, data overflow %d", 8823 t4_read_reg(sc, A_SGE_STAT_TOTAL), 8824 t4_read_reg(sc, A_SGE_STAT_MATCH)); 8825 } else { 8826 sbuf_printf(sb, "unknown mode %d", mode); 8827 } 8828 } 8829 rc = sbuf_finish(sb); 8830 sbuf_delete(sb); 8831 8832 return (rc); 8833 } 8834 8835 static int 8836 sysctl_cpus(SYSCTL_HANDLER_ARGS) 8837 { 8838 struct adapter *sc = arg1; 8839 enum cpu_sets op = arg2; 8840 cpuset_t cpuset; 8841 struct sbuf *sb; 8842 int i, rc; 8843 8844 MPASS(op == LOCAL_CPUS || op == INTR_CPUS); 8845 8846 CPU_ZERO(&cpuset); 8847 rc = bus_get_cpus(sc->dev, op, sizeof(cpuset), &cpuset); 8848 if (rc != 0) 8849 return (rc); 8850 8851 rc = sysctl_wire_old_buffer(req, 0); 8852 if (rc != 0) 8853 return (rc); 8854 8855 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 8856 if (sb == NULL) 8857 return (ENOMEM); 8858 8859 CPU_FOREACH(i) 8860 sbuf_printf(sb, "%d ", i); 8861 rc = sbuf_finish(sb); 8862 sbuf_delete(sb); 8863 8864 return (rc); 8865 } 8866 8867 #ifdef TCP_OFFLOAD 8868 static int 8869 sysctl_tls_rx_ports(SYSCTL_HANDLER_ARGS) 8870 { 8871 struct adapter *sc = arg1; 8872 int *old_ports, *new_ports; 8873 int i, new_count, rc; 8874 8875 if (req->newptr == NULL && req->oldptr == NULL) 8876 return (SYSCTL_OUT(req, NULL, imax(sc->tt.num_tls_rx_ports, 1) * 8877 sizeof(sc->tt.tls_rx_ports[0]))); 8878 8879 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4tlsrx"); 8880 if (rc) 8881 return (rc); 8882 8883 if (sc->tt.num_tls_rx_ports == 0) { 8884 i = -1; 8885 rc = SYSCTL_OUT(req, &i, sizeof(i)); 8886 } else 8887 rc = SYSCTL_OUT(req, sc->tt.tls_rx_ports, 8888 sc->tt.num_tls_rx_ports * sizeof(sc->tt.tls_rx_ports[0])); 8889 if (rc == 0 && req->newptr != NULL) { 8890 new_count = req->newlen / sizeof(new_ports[0]); 8891 new_ports = malloc(new_count * sizeof(new_ports[0]), M_CXGBE, 8892 M_WAITOK); 8893 rc = SYSCTL_IN(req, new_ports, new_count * 8894 sizeof(new_ports[0])); 8895 if (rc) 8896 goto err; 8897 8898 /* Allow setting to a single '-1' to clear the list. */ 8899 if (new_count == 1 && new_ports[0] == -1) { 8900 ADAPTER_LOCK(sc); 8901 old_ports = sc->tt.tls_rx_ports; 8902 sc->tt.tls_rx_ports = NULL; 8903 sc->tt.num_tls_rx_ports = 0; 8904 ADAPTER_UNLOCK(sc); 8905 free(old_ports, M_CXGBE); 8906 } else { 8907 for (i = 0; i < new_count; i++) { 8908 if (new_ports[i] < 1 || 8909 new_ports[i] > IPPORT_MAX) { 8910 rc = EINVAL; 8911 goto err; 8912 } 8913 } 8914 8915 ADAPTER_LOCK(sc); 8916 old_ports = sc->tt.tls_rx_ports; 8917 sc->tt.tls_rx_ports = new_ports; 8918 sc->tt.num_tls_rx_ports = new_count; 8919 ADAPTER_UNLOCK(sc); 8920 free(old_ports, M_CXGBE); 8921 new_ports = NULL; 8922 } 8923 err: 8924 free(new_ports, M_CXGBE); 8925 } 8926 end_synchronized_op(sc, 0); 8927 return (rc); 8928 } 8929 8930 static void 8931 unit_conv(char *buf, size_t len, u_int val, u_int factor) 8932 { 8933 u_int rem = val % factor; 8934 8935 if (rem == 0) 8936 snprintf(buf, len, "%u", val / factor); 8937 else { 8938 while (rem % 10 == 0) 8939 rem /= 10; 8940 snprintf(buf, len, "%u.%u", val / factor, rem); 8941 } 8942 } 8943 8944 static int 8945 sysctl_tp_tick(SYSCTL_HANDLER_ARGS) 8946 { 8947 struct adapter *sc = arg1; 8948 char buf[16]; 8949 u_int res, re; 8950 u_int cclk_ps = 1000000000 / sc->params.vpd.cclk; 8951 8952 res = t4_read_reg(sc, A_TP_TIMER_RESOLUTION); 8953 switch (arg2) { 8954 case 0: 8955 /* timer_tick */ 8956 re = G_TIMERRESOLUTION(res); 8957 break; 8958 case 1: 8959 /* TCP timestamp tick */ 8960 re = G_TIMESTAMPRESOLUTION(res); 8961 break; 8962 case 2: 8963 /* DACK tick */ 8964 re = G_DELAYEDACKRESOLUTION(res); 8965 break; 8966 default: 8967 return (EDOOFUS); 8968 } 8969 8970 unit_conv(buf, sizeof(buf), (cclk_ps << re), 1000000); 8971 8972 return (sysctl_handle_string(oidp, buf, sizeof(buf), req)); 8973 } 8974 8975 static int 8976 sysctl_tp_dack_timer(SYSCTL_HANDLER_ARGS) 8977 { 8978 struct adapter *sc = arg1; 8979 u_int res, dack_re, v; 8980 u_int cclk_ps = 1000000000 / sc->params.vpd.cclk; 8981 8982 res = t4_read_reg(sc, A_TP_TIMER_RESOLUTION); 8983 dack_re = G_DELAYEDACKRESOLUTION(res); 8984 v = ((cclk_ps << dack_re) / 1000000) * t4_read_reg(sc, A_TP_DACK_TIMER); 8985 8986 return (sysctl_handle_int(oidp, &v, 0, req)); 8987 } 8988 8989 static int 8990 sysctl_tp_timer(SYSCTL_HANDLER_ARGS) 8991 { 8992 struct adapter *sc = arg1; 8993 int reg = arg2; 8994 u_int tre; 8995 u_long tp_tick_us, v; 8996 u_int cclk_ps = 1000000000 / sc->params.vpd.cclk; 8997 8998 MPASS(reg == A_TP_RXT_MIN || reg == A_TP_RXT_MAX || 8999 reg == A_TP_PERS_MIN || reg == A_TP_PERS_MAX || 9000 reg == A_TP_KEEP_IDLE || reg == A_TP_KEEP_INTVL || 9001 reg == A_TP_INIT_SRTT || reg == A_TP_FINWAIT2_TIMER); 9002 9003 tre = G_TIMERRESOLUTION(t4_read_reg(sc, A_TP_TIMER_RESOLUTION)); 9004 tp_tick_us = (cclk_ps << tre) / 1000000; 9005 9006 if (reg == A_TP_INIT_SRTT) 9007 v = tp_tick_us * G_INITSRTT(t4_read_reg(sc, reg)); 9008 else 9009 v = tp_tick_us * t4_read_reg(sc, reg); 9010 9011 return (sysctl_handle_long(oidp, &v, 0, req)); 9012 } 9013 9014 /* 9015 * All fields in TP_SHIFT_CNT are 4b and the starting location of the field is 9016 * passed to this function. 9017 */ 9018 static int 9019 sysctl_tp_shift_cnt(SYSCTL_HANDLER_ARGS) 9020 { 9021 struct adapter *sc = arg1; 9022 int idx = arg2; 9023 u_int v; 9024 9025 MPASS(idx >= 0 && idx <= 24); 9026 9027 v = (t4_read_reg(sc, A_TP_SHIFT_CNT) >> idx) & 0xf; 9028 9029 return (sysctl_handle_int(oidp, &v, 0, req)); 9030 } 9031 9032 static int 9033 sysctl_tp_backoff(SYSCTL_HANDLER_ARGS) 9034 { 9035 struct adapter *sc = arg1; 9036 int idx = arg2; 9037 u_int shift, v, r; 9038 9039 MPASS(idx >= 0 && idx < 16); 9040 9041 r = A_TP_TCP_BACKOFF_REG0 + (idx & ~3); 9042 shift = (idx & 3) << 3; 9043 v = (t4_read_reg(sc, r) >> shift) & M_TIMERBACKOFFINDEX0; 9044 9045 return (sysctl_handle_int(oidp, &v, 0, req)); 9046 } 9047 9048 static int 9049 sysctl_holdoff_tmr_idx_ofld(SYSCTL_HANDLER_ARGS) 9050 { 9051 struct vi_info *vi = arg1; 9052 struct adapter *sc = vi->pi->adapter; 9053 int idx, rc, i; 9054 struct sge_ofld_rxq *ofld_rxq; 9055 uint8_t v; 9056 9057 idx = vi->ofld_tmr_idx; 9058 9059 rc = sysctl_handle_int(oidp, &idx, 0, req); 9060 if (rc != 0 || req->newptr == NULL) 9061 return (rc); 9062 9063 if (idx < 0 || idx >= SGE_NTIMERS) 9064 return (EINVAL); 9065 9066 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 9067 "t4otmr"); 9068 if (rc) 9069 return (rc); 9070 9071 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(vi->ofld_pktc_idx != -1); 9072 for_each_ofld_rxq(vi, i, ofld_rxq) { 9073 #ifdef atomic_store_rel_8 9074 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); 9075 #else 9076 ofld_rxq->iq.intr_params = v; 9077 #endif 9078 } 9079 vi->ofld_tmr_idx = idx; 9080 9081 end_synchronized_op(sc, LOCK_HELD); 9082 return (0); 9083 } 9084 9085 static int 9086 sysctl_holdoff_pktc_idx_ofld(SYSCTL_HANDLER_ARGS) 9087 { 9088 struct vi_info *vi = arg1; 9089 struct adapter *sc = vi->pi->adapter; 9090 int idx, rc; 9091 9092 idx = vi->ofld_pktc_idx; 9093 9094 rc = sysctl_handle_int(oidp, &idx, 0, req); 9095 if (rc != 0 || req->newptr == NULL) 9096 return (rc); 9097 9098 if (idx < -1 || idx >= SGE_NCOUNTERS) 9099 return (EINVAL); 9100 9101 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 9102 "t4opktc"); 9103 if (rc) 9104 return (rc); 9105 9106 if (vi->flags & VI_INIT_DONE) 9107 rc = EBUSY; /* cannot be changed once the queues are created */ 9108 else 9109 vi->ofld_pktc_idx = idx; 9110 9111 end_synchronized_op(sc, LOCK_HELD); 9112 return (rc); 9113 } 9114 #endif 9115 9116 static int 9117 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 9118 { 9119 int rc; 9120 9121 if (cntxt->cid > M_CTXTQID) 9122 return (EINVAL); 9123 9124 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 9125 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 9126 return (EINVAL); 9127 9128 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 9129 if (rc) 9130 return (rc); 9131 9132 if (sc->flags & FW_OK) { 9133 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 9134 &cntxt->data[0]); 9135 if (rc == 0) 9136 goto done; 9137 } 9138 9139 /* 9140 * Read via firmware failed or wasn't even attempted. Read directly via 9141 * the backdoor. 9142 */ 9143 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 9144 done: 9145 end_synchronized_op(sc, 0); 9146 return (rc); 9147 } 9148 9149 static int 9150 load_fw(struct adapter *sc, struct t4_data *fw) 9151 { 9152 int rc; 9153 uint8_t *fw_data; 9154 9155 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 9156 if (rc) 9157 return (rc); 9158 9159 /* 9160 * The firmware, with the sole exception of the memory parity error 9161 * handler, runs from memory and not flash. It is almost always safe to 9162 * install a new firmware on a running system. Just set bit 1 in 9163 * hw.cxgbe.dflags or dev.<nexus>.<n>.dflags first. 9164 */ 9165 if (sc->flags & FULL_INIT_DONE && 9166 (sc->debug_flags & DF_LOAD_FW_ANYTIME) == 0) { 9167 rc = EBUSY; 9168 goto done; 9169 } 9170 9171 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 9172 if (fw_data == NULL) { 9173 rc = ENOMEM; 9174 goto done; 9175 } 9176 9177 rc = copyin(fw->data, fw_data, fw->len); 9178 if (rc == 0) 9179 rc = -t4_load_fw(sc, fw_data, fw->len); 9180 9181 free(fw_data, M_CXGBE); 9182 done: 9183 end_synchronized_op(sc, 0); 9184 return (rc); 9185 } 9186 9187 static int 9188 load_cfg(struct adapter *sc, struct t4_data *cfg) 9189 { 9190 int rc; 9191 uint8_t *cfg_data = NULL; 9192 9193 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldcf"); 9194 if (rc) 9195 return (rc); 9196 9197 if (cfg->len == 0) { 9198 /* clear */ 9199 rc = -t4_load_cfg(sc, NULL, 0); 9200 goto done; 9201 } 9202 9203 cfg_data = malloc(cfg->len, M_CXGBE, M_WAITOK); 9204 if (cfg_data == NULL) { 9205 rc = ENOMEM; 9206 goto done; 9207 } 9208 9209 rc = copyin(cfg->data, cfg_data, cfg->len); 9210 if (rc == 0) 9211 rc = -t4_load_cfg(sc, cfg_data, cfg->len); 9212 9213 free(cfg_data, M_CXGBE); 9214 done: 9215 end_synchronized_op(sc, 0); 9216 return (rc); 9217 } 9218 9219 static int 9220 load_boot(struct adapter *sc, struct t4_bootrom *br) 9221 { 9222 int rc; 9223 uint8_t *br_data = NULL; 9224 u_int offset; 9225 9226 if (br->len > 1024 * 1024) 9227 return (EFBIG); 9228 9229 if (br->pf_offset == 0) { 9230 /* pfidx */ 9231 if (br->pfidx_addr > 7) 9232 return (EINVAL); 9233 offset = G_OFFSET(t4_read_reg(sc, PF_REG(br->pfidx_addr, 9234 A_PCIE_PF_EXPROM_OFST))); 9235 } else if (br->pf_offset == 1) { 9236 /* offset */ 9237 offset = G_OFFSET(br->pfidx_addr); 9238 } else { 9239 return (EINVAL); 9240 } 9241 9242 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldbr"); 9243 if (rc) 9244 return (rc); 9245 9246 if (br->len == 0) { 9247 /* clear */ 9248 rc = -t4_load_boot(sc, NULL, offset, 0); 9249 goto done; 9250 } 9251 9252 br_data = malloc(br->len, M_CXGBE, M_WAITOK); 9253 if (br_data == NULL) { 9254 rc = ENOMEM; 9255 goto done; 9256 } 9257 9258 rc = copyin(br->data, br_data, br->len); 9259 if (rc == 0) 9260 rc = -t4_load_boot(sc, br_data, offset, br->len); 9261 9262 free(br_data, M_CXGBE); 9263 done: 9264 end_synchronized_op(sc, 0); 9265 return (rc); 9266 } 9267 9268 static int 9269 load_bootcfg(struct adapter *sc, struct t4_data *bc) 9270 { 9271 int rc; 9272 uint8_t *bc_data = NULL; 9273 9274 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldcf"); 9275 if (rc) 9276 return (rc); 9277 9278 if (bc->len == 0) { 9279 /* clear */ 9280 rc = -t4_load_bootcfg(sc, NULL, 0); 9281 goto done; 9282 } 9283 9284 bc_data = malloc(bc->len, M_CXGBE, M_WAITOK); 9285 if (bc_data == NULL) { 9286 rc = ENOMEM; 9287 goto done; 9288 } 9289 9290 rc = copyin(bc->data, bc_data, bc->len); 9291 if (rc == 0) 9292 rc = -t4_load_bootcfg(sc, bc_data, bc->len); 9293 9294 free(bc_data, M_CXGBE); 9295 done: 9296 end_synchronized_op(sc, 0); 9297 return (rc); 9298 } 9299 9300 static int 9301 cudbg_dump(struct adapter *sc, struct t4_cudbg_dump *dump) 9302 { 9303 int rc; 9304 struct cudbg_init *cudbg; 9305 void *handle, *buf; 9306 9307 /* buf is large, don't block if no memory is available */ 9308 buf = malloc(dump->len, M_CXGBE, M_NOWAIT | M_ZERO); 9309 if (buf == NULL) 9310 return (ENOMEM); 9311 9312 handle = cudbg_alloc_handle(); 9313 if (handle == NULL) { 9314 rc = ENOMEM; 9315 goto done; 9316 } 9317 9318 cudbg = cudbg_get_init(handle); 9319 cudbg->adap = sc; 9320 cudbg->print = (cudbg_print_cb)printf; 9321 9322 #ifndef notyet 9323 device_printf(sc->dev, "%s: wr_flash %u, len %u, data %p.\n", 9324 __func__, dump->wr_flash, dump->len, dump->data); 9325 #endif 9326 9327 if (dump->wr_flash) 9328 cudbg->use_flash = 1; 9329 MPASS(sizeof(cudbg->dbg_bitmap) == sizeof(dump->bitmap)); 9330 memcpy(cudbg->dbg_bitmap, dump->bitmap, sizeof(cudbg->dbg_bitmap)); 9331 9332 rc = cudbg_collect(handle, buf, &dump->len); 9333 if (rc != 0) 9334 goto done; 9335 9336 rc = copyout(buf, dump->data, dump->len); 9337 done: 9338 cudbg_free_handle(handle); 9339 free(buf, M_CXGBE); 9340 return (rc); 9341 } 9342 9343 static void 9344 free_offload_policy(struct t4_offload_policy *op) 9345 { 9346 struct offload_rule *r; 9347 int i; 9348 9349 if (op == NULL) 9350 return; 9351 9352 r = &op->rule[0]; 9353 for (i = 0; i < op->nrules; i++, r++) { 9354 free(r->bpf_prog.bf_insns, M_CXGBE); 9355 } 9356 free(op->rule, M_CXGBE); 9357 free(op, M_CXGBE); 9358 } 9359 9360 static int 9361 set_offload_policy(struct adapter *sc, struct t4_offload_policy *uop) 9362 { 9363 int i, rc, len; 9364 struct t4_offload_policy *op, *old; 9365 struct bpf_program *bf; 9366 const struct offload_settings *s; 9367 struct offload_rule *r; 9368 void *u; 9369 9370 if (!is_offload(sc)) 9371 return (ENODEV); 9372 9373 if (uop->nrules == 0) { 9374 /* Delete installed policies. */ 9375 op = NULL; 9376 goto set_policy; 9377 } if (uop->nrules > 256) { /* arbitrary */ 9378 return (E2BIG); 9379 } 9380 9381 /* Copy userspace offload policy to kernel */ 9382 op = malloc(sizeof(*op), M_CXGBE, M_ZERO | M_WAITOK); 9383 op->nrules = uop->nrules; 9384 len = op->nrules * sizeof(struct offload_rule); 9385 op->rule = malloc(len, M_CXGBE, M_ZERO | M_WAITOK); 9386 rc = copyin(uop->rule, op->rule, len); 9387 if (rc) { 9388 free(op->rule, M_CXGBE); 9389 free(op, M_CXGBE); 9390 return (rc); 9391 } 9392 9393 r = &op->rule[0]; 9394 for (i = 0; i < op->nrules; i++, r++) { 9395 9396 /* Validate open_type */ 9397 if (r->open_type != OPEN_TYPE_LISTEN && 9398 r->open_type != OPEN_TYPE_ACTIVE && 9399 r->open_type != OPEN_TYPE_PASSIVE && 9400 r->open_type != OPEN_TYPE_DONTCARE) { 9401 error: 9402 /* 9403 * Rules 0 to i have malloc'd filters that need to be 9404 * freed. Rules i+1 to nrules have userspace pointers 9405 * and should be left alone. 9406 */ 9407 op->nrules = i; 9408 free_offload_policy(op); 9409 return (rc); 9410 } 9411 9412 /* Validate settings */ 9413 s = &r->settings; 9414 if ((s->offload != 0 && s->offload != 1) || 9415 s->cong_algo < -1 || s->cong_algo > CONG_ALG_HIGHSPEED || 9416 s->sched_class < -1 || 9417 s->sched_class >= sc->chip_params->nsched_cls) { 9418 rc = EINVAL; 9419 goto error; 9420 } 9421 9422 bf = &r->bpf_prog; 9423 u = bf->bf_insns; /* userspace ptr */ 9424 bf->bf_insns = NULL; 9425 if (bf->bf_len == 0) { 9426 /* legal, matches everything */ 9427 continue; 9428 } 9429 len = bf->bf_len * sizeof(*bf->bf_insns); 9430 bf->bf_insns = malloc(len, M_CXGBE, M_ZERO | M_WAITOK); 9431 rc = copyin(u, bf->bf_insns, len); 9432 if (rc != 0) 9433 goto error; 9434 9435 if (!bpf_validate(bf->bf_insns, bf->bf_len)) { 9436 rc = EINVAL; 9437 goto error; 9438 } 9439 } 9440 set_policy: 9441 rw_wlock(&sc->policy_lock); 9442 old = sc->policy; 9443 sc->policy = op; 9444 rw_wunlock(&sc->policy_lock); 9445 free_offload_policy(old); 9446 9447 return (0); 9448 } 9449 9450 #define MAX_READ_BUF_SIZE (128 * 1024) 9451 static int 9452 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 9453 { 9454 uint32_t addr, remaining, n; 9455 uint32_t *buf; 9456 int rc; 9457 uint8_t *dst; 9458 9459 rc = validate_mem_range(sc, mr->addr, mr->len); 9460 if (rc != 0) 9461 return (rc); 9462 9463 buf = malloc(min(mr->len, MAX_READ_BUF_SIZE), M_CXGBE, M_WAITOK); 9464 addr = mr->addr; 9465 remaining = mr->len; 9466 dst = (void *)mr->data; 9467 9468 while (remaining) { 9469 n = min(remaining, MAX_READ_BUF_SIZE); 9470 read_via_memwin(sc, 2, addr, buf, n); 9471 9472 rc = copyout(buf, dst, n); 9473 if (rc != 0) 9474 break; 9475 9476 dst += n; 9477 remaining -= n; 9478 addr += n; 9479 } 9480 9481 free(buf, M_CXGBE); 9482 return (rc); 9483 } 9484 #undef MAX_READ_BUF_SIZE 9485 9486 static int 9487 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 9488 { 9489 int rc; 9490 9491 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 9492 return (EINVAL); 9493 9494 if (i2cd->len > sizeof(i2cd->data)) 9495 return (EFBIG); 9496 9497 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 9498 if (rc) 9499 return (rc); 9500 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 9501 i2cd->offset, i2cd->len, &i2cd->data[0]); 9502 end_synchronized_op(sc, 0); 9503 9504 return (rc); 9505 } 9506 9507 int 9508 t4_os_find_pci_capability(struct adapter *sc, int cap) 9509 { 9510 int i; 9511 9512 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 9513 } 9514 9515 int 9516 t4_os_pci_save_state(struct adapter *sc) 9517 { 9518 device_t dev; 9519 struct pci_devinfo *dinfo; 9520 9521 dev = sc->dev; 9522 dinfo = device_get_ivars(dev); 9523 9524 pci_cfg_save(dev, dinfo, 0); 9525 return (0); 9526 } 9527 9528 int 9529 t4_os_pci_restore_state(struct adapter *sc) 9530 { 9531 device_t dev; 9532 struct pci_devinfo *dinfo; 9533 9534 dev = sc->dev; 9535 dinfo = device_get_ivars(dev); 9536 9537 pci_cfg_restore(dev, dinfo); 9538 return (0); 9539 } 9540 9541 void 9542 t4_os_portmod_changed(struct port_info *pi) 9543 { 9544 struct adapter *sc = pi->adapter; 9545 struct vi_info *vi; 9546 struct ifnet *ifp; 9547 static const char *mod_str[] = { 9548 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 9549 }; 9550 9551 KASSERT((pi->flags & FIXED_IFMEDIA) == 0, 9552 ("%s: port_type %u", __func__, pi->port_type)); 9553 9554 vi = &pi->vi[0]; 9555 if (begin_synchronized_op(sc, vi, HOLD_LOCK, "t4mod") == 0) { 9556 PORT_LOCK(pi); 9557 build_medialist(pi); 9558 if (pi->mod_type != FW_PORT_MOD_TYPE_NONE) { 9559 fixup_link_config(pi); 9560 apply_link_config(pi); 9561 } 9562 PORT_UNLOCK(pi); 9563 end_synchronized_op(sc, LOCK_HELD); 9564 } 9565 9566 ifp = vi->ifp; 9567 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 9568 if_printf(ifp, "transceiver unplugged.\n"); 9569 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 9570 if_printf(ifp, "unknown transceiver inserted.\n"); 9571 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 9572 if_printf(ifp, "unsupported transceiver inserted.\n"); 9573 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 9574 if_printf(ifp, "%dGbps %s transceiver inserted.\n", 9575 port_top_speed(pi), mod_str[pi->mod_type]); 9576 } else { 9577 if_printf(ifp, "transceiver (type %d) inserted.\n", 9578 pi->mod_type); 9579 } 9580 } 9581 9582 void 9583 t4_os_link_changed(struct port_info *pi) 9584 { 9585 struct vi_info *vi; 9586 struct ifnet *ifp; 9587 struct link_config *lc; 9588 int v; 9589 9590 PORT_LOCK_ASSERT_OWNED(pi); 9591 9592 for_each_vi(pi, v, vi) { 9593 ifp = vi->ifp; 9594 if (ifp == NULL) 9595 continue; 9596 9597 lc = &pi->link_cfg; 9598 if (lc->link_ok) { 9599 ifp->if_baudrate = IF_Mbps(lc->speed); 9600 if_link_state_change(ifp, LINK_STATE_UP); 9601 } else { 9602 if_link_state_change(ifp, LINK_STATE_DOWN); 9603 } 9604 } 9605 } 9606 9607 void 9608 t4_iterate(void (*func)(struct adapter *, void *), void *arg) 9609 { 9610 struct adapter *sc; 9611 9612 sx_slock(&t4_list_lock); 9613 SLIST_FOREACH(sc, &t4_list, link) { 9614 /* 9615 * func should not make any assumptions about what state sc is 9616 * in - the only guarantee is that sc->sc_lock is a valid lock. 9617 */ 9618 func(sc, arg); 9619 } 9620 sx_sunlock(&t4_list_lock); 9621 } 9622 9623 static int 9624 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 9625 struct thread *td) 9626 { 9627 int rc; 9628 struct adapter *sc = dev->si_drv1; 9629 9630 rc = priv_check(td, PRIV_DRIVER); 9631 if (rc != 0) 9632 return (rc); 9633 9634 switch (cmd) { 9635 case CHELSIO_T4_GETREG: { 9636 struct t4_reg *edata = (struct t4_reg *)data; 9637 9638 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 9639 return (EFAULT); 9640 9641 if (edata->size == 4) 9642 edata->val = t4_read_reg(sc, edata->addr); 9643 else if (edata->size == 8) 9644 edata->val = t4_read_reg64(sc, edata->addr); 9645 else 9646 return (EINVAL); 9647 9648 break; 9649 } 9650 case CHELSIO_T4_SETREG: { 9651 struct t4_reg *edata = (struct t4_reg *)data; 9652 9653 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 9654 return (EFAULT); 9655 9656 if (edata->size == 4) { 9657 if (edata->val & 0xffffffff00000000) 9658 return (EINVAL); 9659 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 9660 } else if (edata->size == 8) 9661 t4_write_reg64(sc, edata->addr, edata->val); 9662 else 9663 return (EINVAL); 9664 break; 9665 } 9666 case CHELSIO_T4_REGDUMP: { 9667 struct t4_regdump *regs = (struct t4_regdump *)data; 9668 int reglen = t4_get_regs_len(sc); 9669 uint8_t *buf; 9670 9671 if (regs->len < reglen) { 9672 regs->len = reglen; /* hint to the caller */ 9673 return (ENOBUFS); 9674 } 9675 9676 regs->len = reglen; 9677 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 9678 get_regs(sc, regs, buf); 9679 rc = copyout(buf, regs->data, reglen); 9680 free(buf, M_CXGBE); 9681 break; 9682 } 9683 case CHELSIO_T4_GET_FILTER_MODE: 9684 rc = get_filter_mode(sc, (uint32_t *)data); 9685 break; 9686 case CHELSIO_T4_SET_FILTER_MODE: 9687 rc = set_filter_mode(sc, *(uint32_t *)data); 9688 break; 9689 case CHELSIO_T4_GET_FILTER: 9690 rc = get_filter(sc, (struct t4_filter *)data); 9691 break; 9692 case CHELSIO_T4_SET_FILTER: 9693 rc = set_filter(sc, (struct t4_filter *)data); 9694 break; 9695 case CHELSIO_T4_DEL_FILTER: 9696 rc = del_filter(sc, (struct t4_filter *)data); 9697 break; 9698 case CHELSIO_T4_GET_SGE_CONTEXT: 9699 rc = get_sge_context(sc, (struct t4_sge_context *)data); 9700 break; 9701 case CHELSIO_T4_LOAD_FW: 9702 rc = load_fw(sc, (struct t4_data *)data); 9703 break; 9704 case CHELSIO_T4_GET_MEM: 9705 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 9706 break; 9707 case CHELSIO_T4_GET_I2C: 9708 rc = read_i2c(sc, (struct t4_i2c_data *)data); 9709 break; 9710 case CHELSIO_T4_CLEAR_STATS: { 9711 int i, v, bg_map; 9712 u_int port_id = *(uint32_t *)data; 9713 struct port_info *pi; 9714 struct vi_info *vi; 9715 9716 if (port_id >= sc->params.nports) 9717 return (EINVAL); 9718 pi = sc->port[port_id]; 9719 if (pi == NULL) 9720 return (EIO); 9721 9722 /* MAC stats */ 9723 t4_clr_port_stats(sc, pi->tx_chan); 9724 pi->tx_parse_error = 0; 9725 pi->tnl_cong_drops = 0; 9726 mtx_lock(&sc->reg_lock); 9727 for_each_vi(pi, v, vi) { 9728 if (vi->flags & VI_INIT_DONE) 9729 t4_clr_vi_stats(sc, vi->viid); 9730 } 9731 bg_map = pi->mps_bg_map; 9732 v = 0; /* reuse */ 9733 while (bg_map) { 9734 i = ffs(bg_map) - 1; 9735 t4_write_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 9736 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 9737 bg_map &= ~(1 << i); 9738 } 9739 mtx_unlock(&sc->reg_lock); 9740 9741 /* 9742 * Since this command accepts a port, clear stats for 9743 * all VIs on this port. 9744 */ 9745 for_each_vi(pi, v, vi) { 9746 if (vi->flags & VI_INIT_DONE) { 9747 struct sge_rxq *rxq; 9748 struct sge_txq *txq; 9749 struct sge_wrq *wrq; 9750 9751 for_each_rxq(vi, i, rxq) { 9752 #if defined(INET) || defined(INET6) 9753 rxq->lro.lro_queued = 0; 9754 rxq->lro.lro_flushed = 0; 9755 #endif 9756 rxq->rxcsum = 0; 9757 rxq->vlan_extraction = 0; 9758 } 9759 9760 for_each_txq(vi, i, txq) { 9761 txq->txcsum = 0; 9762 txq->tso_wrs = 0; 9763 txq->vlan_insertion = 0; 9764 txq->imm_wrs = 0; 9765 txq->sgl_wrs = 0; 9766 txq->txpkt_wrs = 0; 9767 txq->txpkts0_wrs = 0; 9768 txq->txpkts1_wrs = 0; 9769 txq->txpkts0_pkts = 0; 9770 txq->txpkts1_pkts = 0; 9771 mp_ring_reset_stats(txq->r); 9772 } 9773 9774 #ifdef TCP_OFFLOAD 9775 /* nothing to clear for each ofld_rxq */ 9776 9777 for_each_ofld_txq(vi, i, wrq) { 9778 wrq->tx_wrs_direct = 0; 9779 wrq->tx_wrs_copied = 0; 9780 } 9781 #endif 9782 9783 if (IS_MAIN_VI(vi)) { 9784 wrq = &sc->sge.ctrlq[pi->port_id]; 9785 wrq->tx_wrs_direct = 0; 9786 wrq->tx_wrs_copied = 0; 9787 } 9788 } 9789 } 9790 break; 9791 } 9792 case CHELSIO_T4_SCHED_CLASS: 9793 rc = t4_set_sched_class(sc, (struct t4_sched_params *)data); 9794 break; 9795 case CHELSIO_T4_SCHED_QUEUE: 9796 rc = t4_set_sched_queue(sc, (struct t4_sched_queue *)data); 9797 break; 9798 case CHELSIO_T4_GET_TRACER: 9799 rc = t4_get_tracer(sc, (struct t4_tracer *)data); 9800 break; 9801 case CHELSIO_T4_SET_TRACER: 9802 rc = t4_set_tracer(sc, (struct t4_tracer *)data); 9803 break; 9804 case CHELSIO_T4_LOAD_CFG: 9805 rc = load_cfg(sc, (struct t4_data *)data); 9806 break; 9807 case CHELSIO_T4_LOAD_BOOT: 9808 rc = load_boot(sc, (struct t4_bootrom *)data); 9809 break; 9810 case CHELSIO_T4_LOAD_BOOTCFG: 9811 rc = load_bootcfg(sc, (struct t4_data *)data); 9812 break; 9813 case CHELSIO_T4_CUDBG_DUMP: 9814 rc = cudbg_dump(sc, (struct t4_cudbg_dump *)data); 9815 break; 9816 case CHELSIO_T4_SET_OFLD_POLICY: 9817 rc = set_offload_policy(sc, (struct t4_offload_policy *)data); 9818 break; 9819 default: 9820 rc = ENOTTY; 9821 } 9822 9823 return (rc); 9824 } 9825 9826 void 9827 t4_db_full(struct adapter *sc) 9828 { 9829 9830 CXGBE_UNIMPLEMENTED(__func__); 9831 } 9832 9833 void 9834 t4_db_dropped(struct adapter *sc) 9835 { 9836 9837 CXGBE_UNIMPLEMENTED(__func__); 9838 } 9839 9840 #ifdef TCP_OFFLOAD 9841 static int 9842 toe_capability(struct vi_info *vi, int enable) 9843 { 9844 int rc; 9845 struct port_info *pi = vi->pi; 9846 struct adapter *sc = pi->adapter; 9847 9848 ASSERT_SYNCHRONIZED_OP(sc); 9849 9850 if (!is_offload(sc)) 9851 return (ENODEV); 9852 9853 if (enable) { 9854 if ((vi->ifp->if_capenable & IFCAP_TOE) != 0) { 9855 /* TOE is already enabled. */ 9856 return (0); 9857 } 9858 9859 /* 9860 * We need the port's queues around so that we're able to send 9861 * and receive CPLs to/from the TOE even if the ifnet for this 9862 * port has never been UP'd administratively. 9863 */ 9864 if (!(vi->flags & VI_INIT_DONE)) { 9865 rc = vi_full_init(vi); 9866 if (rc) 9867 return (rc); 9868 } 9869 if (!(pi->vi[0].flags & VI_INIT_DONE)) { 9870 rc = vi_full_init(&pi->vi[0]); 9871 if (rc) 9872 return (rc); 9873 } 9874 9875 if (isset(&sc->offload_map, pi->port_id)) { 9876 /* TOE is enabled on another VI of this port. */ 9877 pi->uld_vis++; 9878 return (0); 9879 } 9880 9881 if (!uld_active(sc, ULD_TOM)) { 9882 rc = t4_activate_uld(sc, ULD_TOM); 9883 if (rc == EAGAIN) { 9884 log(LOG_WARNING, 9885 "You must kldload t4_tom.ko before trying " 9886 "to enable TOE on a cxgbe interface.\n"); 9887 } 9888 if (rc != 0) 9889 return (rc); 9890 KASSERT(sc->tom_softc != NULL, 9891 ("%s: TOM activated but softc NULL", __func__)); 9892 KASSERT(uld_active(sc, ULD_TOM), 9893 ("%s: TOM activated but flag not set", __func__)); 9894 } 9895 9896 /* Activate iWARP and iSCSI too, if the modules are loaded. */ 9897 if (!uld_active(sc, ULD_IWARP)) 9898 (void) t4_activate_uld(sc, ULD_IWARP); 9899 if (!uld_active(sc, ULD_ISCSI)) 9900 (void) t4_activate_uld(sc, ULD_ISCSI); 9901 9902 pi->uld_vis++; 9903 setbit(&sc->offload_map, pi->port_id); 9904 } else { 9905 pi->uld_vis--; 9906 9907 if (!isset(&sc->offload_map, pi->port_id) || pi->uld_vis > 0) 9908 return (0); 9909 9910 KASSERT(uld_active(sc, ULD_TOM), 9911 ("%s: TOM never initialized?", __func__)); 9912 clrbit(&sc->offload_map, pi->port_id); 9913 } 9914 9915 return (0); 9916 } 9917 9918 /* 9919 * Add an upper layer driver to the global list. 9920 */ 9921 int 9922 t4_register_uld(struct uld_info *ui) 9923 { 9924 int rc = 0; 9925 struct uld_info *u; 9926 9927 sx_xlock(&t4_uld_list_lock); 9928 SLIST_FOREACH(u, &t4_uld_list, link) { 9929 if (u->uld_id == ui->uld_id) { 9930 rc = EEXIST; 9931 goto done; 9932 } 9933 } 9934 9935 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 9936 ui->refcount = 0; 9937 done: 9938 sx_xunlock(&t4_uld_list_lock); 9939 return (rc); 9940 } 9941 9942 int 9943 t4_unregister_uld(struct uld_info *ui) 9944 { 9945 int rc = EINVAL; 9946 struct uld_info *u; 9947 9948 sx_xlock(&t4_uld_list_lock); 9949 9950 SLIST_FOREACH(u, &t4_uld_list, link) { 9951 if (u == ui) { 9952 if (ui->refcount > 0) { 9953 rc = EBUSY; 9954 goto done; 9955 } 9956 9957 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 9958 rc = 0; 9959 goto done; 9960 } 9961 } 9962 done: 9963 sx_xunlock(&t4_uld_list_lock); 9964 return (rc); 9965 } 9966 9967 int 9968 t4_activate_uld(struct adapter *sc, int id) 9969 { 9970 int rc; 9971 struct uld_info *ui; 9972 9973 ASSERT_SYNCHRONIZED_OP(sc); 9974 9975 if (id < 0 || id > ULD_MAX) 9976 return (EINVAL); 9977 rc = EAGAIN; /* kldoad the module with this ULD and try again. */ 9978 9979 sx_slock(&t4_uld_list_lock); 9980 9981 SLIST_FOREACH(ui, &t4_uld_list, link) { 9982 if (ui->uld_id == id) { 9983 if (!(sc->flags & FULL_INIT_DONE)) { 9984 rc = adapter_full_init(sc); 9985 if (rc != 0) 9986 break; 9987 } 9988 9989 rc = ui->activate(sc); 9990 if (rc == 0) { 9991 setbit(&sc->active_ulds, id); 9992 ui->refcount++; 9993 } 9994 break; 9995 } 9996 } 9997 9998 sx_sunlock(&t4_uld_list_lock); 9999 10000 return (rc); 10001 } 10002 10003 int 10004 t4_deactivate_uld(struct adapter *sc, int id) 10005 { 10006 int rc; 10007 struct uld_info *ui; 10008 10009 ASSERT_SYNCHRONIZED_OP(sc); 10010 10011 if (id < 0 || id > ULD_MAX) 10012 return (EINVAL); 10013 rc = ENXIO; 10014 10015 sx_slock(&t4_uld_list_lock); 10016 10017 SLIST_FOREACH(ui, &t4_uld_list, link) { 10018 if (ui->uld_id == id) { 10019 rc = ui->deactivate(sc); 10020 if (rc == 0) { 10021 clrbit(&sc->active_ulds, id); 10022 ui->refcount--; 10023 } 10024 break; 10025 } 10026 } 10027 10028 sx_sunlock(&t4_uld_list_lock); 10029 10030 return (rc); 10031 } 10032 10033 int 10034 uld_active(struct adapter *sc, int uld_id) 10035 { 10036 10037 MPASS(uld_id >= 0 && uld_id <= ULD_MAX); 10038 10039 return (isset(&sc->active_ulds, uld_id)); 10040 } 10041 #endif 10042 10043 /* 10044 * t = ptr to tunable. 10045 * nc = number of CPUs. 10046 * c = compiled in default for that tunable. 10047 */ 10048 static void 10049 calculate_nqueues(int *t, int nc, const int c) 10050 { 10051 int nq; 10052 10053 if (*t > 0) 10054 return; 10055 nq = *t < 0 ? -*t : c; 10056 *t = min(nc, nq); 10057 } 10058 10059 /* 10060 * Come up with reasonable defaults for some of the tunables, provided they're 10061 * not set by the user (in which case we'll use the values as is). 10062 */ 10063 static void 10064 tweak_tunables(void) 10065 { 10066 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 10067 10068 if (t4_ntxq < 1) { 10069 #ifdef RSS 10070 t4_ntxq = rss_getnumbuckets(); 10071 #else 10072 calculate_nqueues(&t4_ntxq, nc, NTXQ); 10073 #endif 10074 } 10075 10076 calculate_nqueues(&t4_ntxq_vi, nc, NTXQ_VI); 10077 10078 if (t4_nrxq < 1) { 10079 #ifdef RSS 10080 t4_nrxq = rss_getnumbuckets(); 10081 #else 10082 calculate_nqueues(&t4_nrxq, nc, NRXQ); 10083 #endif 10084 } 10085 10086 calculate_nqueues(&t4_nrxq_vi, nc, NRXQ_VI); 10087 10088 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 10089 calculate_nqueues(&t4_nofldtxq, nc, NOFLDTXQ); 10090 calculate_nqueues(&t4_nofldtxq_vi, nc, NOFLDTXQ_VI); 10091 #endif 10092 #ifdef TCP_OFFLOAD 10093 calculate_nqueues(&t4_nofldrxq, nc, NOFLDRXQ); 10094 calculate_nqueues(&t4_nofldrxq_vi, nc, NOFLDRXQ_VI); 10095 10096 if (t4_toecaps_allowed == -1) 10097 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 10098 10099 if (t4_rdmacaps_allowed == -1) { 10100 t4_rdmacaps_allowed = FW_CAPS_CONFIG_RDMA_RDDP | 10101 FW_CAPS_CONFIG_RDMA_RDMAC; 10102 } 10103 10104 if (t4_iscsicaps_allowed == -1) { 10105 t4_iscsicaps_allowed = FW_CAPS_CONFIG_ISCSI_INITIATOR_PDU | 10106 FW_CAPS_CONFIG_ISCSI_TARGET_PDU | 10107 FW_CAPS_CONFIG_ISCSI_T10DIF; 10108 } 10109 10110 if (t4_tmr_idx_ofld < 0 || t4_tmr_idx_ofld >= SGE_NTIMERS) 10111 t4_tmr_idx_ofld = TMR_IDX_OFLD; 10112 10113 if (t4_pktc_idx_ofld < -1 || t4_pktc_idx_ofld >= SGE_NCOUNTERS) 10114 t4_pktc_idx_ofld = PKTC_IDX_OFLD; 10115 #else 10116 if (t4_toecaps_allowed == -1) 10117 t4_toecaps_allowed = 0; 10118 10119 if (t4_rdmacaps_allowed == -1) 10120 t4_rdmacaps_allowed = 0; 10121 10122 if (t4_iscsicaps_allowed == -1) 10123 t4_iscsicaps_allowed = 0; 10124 #endif 10125 10126 #ifdef DEV_NETMAP 10127 calculate_nqueues(&t4_nnmtxq_vi, nc, NNMTXQ_VI); 10128 calculate_nqueues(&t4_nnmrxq_vi, nc, NNMRXQ_VI); 10129 #endif 10130 10131 if (t4_tmr_idx < 0 || t4_tmr_idx >= SGE_NTIMERS) 10132 t4_tmr_idx = TMR_IDX; 10133 10134 if (t4_pktc_idx < -1 || t4_pktc_idx >= SGE_NCOUNTERS) 10135 t4_pktc_idx = PKTC_IDX; 10136 10137 if (t4_qsize_txq < 128) 10138 t4_qsize_txq = 128; 10139 10140 if (t4_qsize_rxq < 128) 10141 t4_qsize_rxq = 128; 10142 while (t4_qsize_rxq & 7) 10143 t4_qsize_rxq++; 10144 10145 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 10146 10147 /* 10148 * Number of VIs to create per-port. The first VI is the "main" regular 10149 * VI for the port. The rest are additional virtual interfaces on the 10150 * same physical port. Note that the main VI does not have native 10151 * netmap support but the extra VIs do. 10152 * 10153 * Limit the number of VIs per port to the number of available 10154 * MAC addresses per port. 10155 */ 10156 if (t4_num_vis < 1) 10157 t4_num_vis = 1; 10158 if (t4_num_vis > nitems(vi_mac_funcs)) { 10159 t4_num_vis = nitems(vi_mac_funcs); 10160 printf("cxgbe: number of VIs limited to %d\n", t4_num_vis); 10161 } 10162 10163 if (pcie_relaxed_ordering < 0 || pcie_relaxed_ordering > 2) { 10164 pcie_relaxed_ordering = 1; 10165 #if defined(__i386__) || defined(__amd64__) 10166 if (cpu_vendor_id == CPU_VENDOR_INTEL) 10167 pcie_relaxed_ordering = 0; 10168 #endif 10169 } 10170 } 10171 10172 #ifdef DDB 10173 static void 10174 t4_dump_tcb(struct adapter *sc, int tid) 10175 { 10176 uint32_t base, i, j, off, pf, reg, save, tcb_addr, win_pos; 10177 10178 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2); 10179 save = t4_read_reg(sc, reg); 10180 base = sc->memwin[2].mw_base; 10181 10182 /* Dump TCB for the tid */ 10183 tcb_addr = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 10184 tcb_addr += tid * TCB_SIZE; 10185 10186 if (is_t4(sc)) { 10187 pf = 0; 10188 win_pos = tcb_addr & ~0xf; /* start must be 16B aligned */ 10189 } else { 10190 pf = V_PFNUM(sc->pf); 10191 win_pos = tcb_addr & ~0x7f; /* start must be 128B aligned */ 10192 } 10193 t4_write_reg(sc, reg, win_pos | pf); 10194 t4_read_reg(sc, reg); 10195 10196 off = tcb_addr - win_pos; 10197 for (i = 0; i < 4; i++) { 10198 uint32_t buf[8]; 10199 for (j = 0; j < 8; j++, off += 4) 10200 buf[j] = htonl(t4_read_reg(sc, base + off)); 10201 10202 db_printf("%08x %08x %08x %08x %08x %08x %08x %08x\n", 10203 buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6], 10204 buf[7]); 10205 } 10206 10207 t4_write_reg(sc, reg, save); 10208 t4_read_reg(sc, reg); 10209 } 10210 10211 static void 10212 t4_dump_devlog(struct adapter *sc) 10213 { 10214 struct devlog_params *dparams = &sc->params.devlog; 10215 struct fw_devlog_e e; 10216 int i, first, j, m, nentries, rc; 10217 uint64_t ftstamp = UINT64_MAX; 10218 10219 if (dparams->start == 0) { 10220 db_printf("devlog params not valid\n"); 10221 return; 10222 } 10223 10224 nentries = dparams->size / sizeof(struct fw_devlog_e); 10225 m = fwmtype_to_hwmtype(dparams->memtype); 10226 10227 /* Find the first entry. */ 10228 first = -1; 10229 for (i = 0; i < nentries && !db_pager_quit; i++) { 10230 rc = -t4_mem_read(sc, m, dparams->start + i * sizeof(e), 10231 sizeof(e), (void *)&e); 10232 if (rc != 0) 10233 break; 10234 10235 if (e.timestamp == 0) 10236 break; 10237 10238 e.timestamp = be64toh(e.timestamp); 10239 if (e.timestamp < ftstamp) { 10240 ftstamp = e.timestamp; 10241 first = i; 10242 } 10243 } 10244 10245 if (first == -1) 10246 return; 10247 10248 i = first; 10249 do { 10250 rc = -t4_mem_read(sc, m, dparams->start + i * sizeof(e), 10251 sizeof(e), (void *)&e); 10252 if (rc != 0) 10253 return; 10254 10255 if (e.timestamp == 0) 10256 return; 10257 10258 e.timestamp = be64toh(e.timestamp); 10259 e.seqno = be32toh(e.seqno); 10260 for (j = 0; j < 8; j++) 10261 e.params[j] = be32toh(e.params[j]); 10262 10263 db_printf("%10d %15ju %8s %8s ", 10264 e.seqno, e.timestamp, 10265 (e.level < nitems(devlog_level_strings) ? 10266 devlog_level_strings[e.level] : "UNKNOWN"), 10267 (e.facility < nitems(devlog_facility_strings) ? 10268 devlog_facility_strings[e.facility] : "UNKNOWN")); 10269 db_printf(e.fmt, e.params[0], e.params[1], e.params[2], 10270 e.params[3], e.params[4], e.params[5], e.params[6], 10271 e.params[7]); 10272 10273 if (++i == nentries) 10274 i = 0; 10275 } while (i != first && !db_pager_quit); 10276 } 10277 10278 static struct command_table db_t4_table = LIST_HEAD_INITIALIZER(db_t4_table); 10279 _DB_SET(_show, t4, NULL, db_show_table, 0, &db_t4_table); 10280 10281 DB_FUNC(devlog, db_show_devlog, db_t4_table, CS_OWN, NULL) 10282 { 10283 device_t dev; 10284 int t; 10285 bool valid; 10286 10287 valid = false; 10288 t = db_read_token(); 10289 if (t == tIDENT) { 10290 dev = device_lookup_by_name(db_tok_string); 10291 valid = true; 10292 } 10293 db_skip_to_eol(); 10294 if (!valid) { 10295 db_printf("usage: show t4 devlog <nexus>\n"); 10296 return; 10297 } 10298 10299 if (dev == NULL) { 10300 db_printf("device not found\n"); 10301 return; 10302 } 10303 10304 t4_dump_devlog(device_get_softc(dev)); 10305 } 10306 10307 DB_FUNC(tcb, db_show_t4tcb, db_t4_table, CS_OWN, NULL) 10308 { 10309 device_t dev; 10310 int radix, tid, t; 10311 bool valid; 10312 10313 valid = false; 10314 radix = db_radix; 10315 db_radix = 10; 10316 t = db_read_token(); 10317 if (t == tIDENT) { 10318 dev = device_lookup_by_name(db_tok_string); 10319 t = db_read_token(); 10320 if (t == tNUMBER) { 10321 tid = db_tok_number; 10322 valid = true; 10323 } 10324 } 10325 db_radix = radix; 10326 db_skip_to_eol(); 10327 if (!valid) { 10328 db_printf("usage: show t4 tcb <nexus> <tid>\n"); 10329 return; 10330 } 10331 10332 if (dev == NULL) { 10333 db_printf("device not found\n"); 10334 return; 10335 } 10336 if (tid < 0) { 10337 db_printf("invalid tid\n"); 10338 return; 10339 } 10340 10341 t4_dump_tcb(device_get_softc(dev), tid); 10342 } 10343 #endif 10344 10345 /* 10346 * Borrowed from cesa_prep_aes_key(). 10347 * 10348 * NB: The crypto engine wants the words in the decryption key in reverse 10349 * order. 10350 */ 10351 void 10352 t4_aes_getdeckey(void *dec_key, const void *enc_key, unsigned int kbits) 10353 { 10354 uint32_t ek[4 * (RIJNDAEL_MAXNR + 1)]; 10355 uint32_t *dkey; 10356 int i; 10357 10358 rijndaelKeySetupEnc(ek, enc_key, kbits); 10359 dkey = dec_key; 10360 dkey += (kbits / 8) / 4; 10361 10362 switch (kbits) { 10363 case 128: 10364 for (i = 0; i < 4; i++) 10365 *--dkey = htobe32(ek[4 * 10 + i]); 10366 break; 10367 case 192: 10368 for (i = 0; i < 2; i++) 10369 *--dkey = htobe32(ek[4 * 11 + 2 + i]); 10370 for (i = 0; i < 4; i++) 10371 *--dkey = htobe32(ek[4 * 12 + i]); 10372 break; 10373 case 256: 10374 for (i = 0; i < 4; i++) 10375 *--dkey = htobe32(ek[4 * 13 + i]); 10376 for (i = 0; i < 4; i++) 10377 *--dkey = htobe32(ek[4 * 14 + i]); 10378 break; 10379 } 10380 MPASS(dkey == dec_key); 10381 } 10382 10383 static struct sx mlu; /* mod load unload */ 10384 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload"); 10385 10386 static int 10387 mod_event(module_t mod, int cmd, void *arg) 10388 { 10389 int rc = 0; 10390 static int loaded = 0; 10391 10392 switch (cmd) { 10393 case MOD_LOAD: 10394 sx_xlock(&mlu); 10395 if (loaded++ == 0) { 10396 t4_sge_modload(); 10397 t4_register_shared_cpl_handler(CPL_SET_TCB_RPL, 10398 t4_filter_rpl, CPL_COOKIE_FILTER); 10399 t4_register_shared_cpl_handler(CPL_L2T_WRITE_RPL, 10400 do_l2t_write_rpl, CPL_COOKIE_FILTER); 10401 t4_register_shared_cpl_handler(CPL_ACT_OPEN_RPL, 10402 t4_hashfilter_ao_rpl, CPL_COOKIE_HASHFILTER); 10403 t4_register_shared_cpl_handler(CPL_SET_TCB_RPL, 10404 t4_hashfilter_tcb_rpl, CPL_COOKIE_HASHFILTER); 10405 t4_register_shared_cpl_handler(CPL_ABORT_RPL_RSS, 10406 t4_del_hashfilter_rpl, CPL_COOKIE_HASHFILTER); 10407 t4_register_cpl_handler(CPL_TRACE_PKT, t4_trace_pkt); 10408 t4_register_cpl_handler(CPL_T5_TRACE_PKT, t5_trace_pkt); 10409 t4_register_cpl_handler(CPL_SMT_WRITE_RPL, 10410 do_smt_write_rpl); 10411 sx_init(&t4_list_lock, "T4/T5 adapters"); 10412 SLIST_INIT(&t4_list); 10413 #ifdef TCP_OFFLOAD 10414 sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); 10415 SLIST_INIT(&t4_uld_list); 10416 #endif 10417 t4_tracer_modload(); 10418 tweak_tunables(); 10419 } 10420 sx_xunlock(&mlu); 10421 break; 10422 10423 case MOD_UNLOAD: 10424 sx_xlock(&mlu); 10425 if (--loaded == 0) { 10426 int tries; 10427 10428 sx_slock(&t4_list_lock); 10429 if (!SLIST_EMPTY(&t4_list)) { 10430 rc = EBUSY; 10431 sx_sunlock(&t4_list_lock); 10432 goto done_unload; 10433 } 10434 #ifdef TCP_OFFLOAD 10435 sx_slock(&t4_uld_list_lock); 10436 if (!SLIST_EMPTY(&t4_uld_list)) { 10437 rc = EBUSY; 10438 sx_sunlock(&t4_uld_list_lock); 10439 sx_sunlock(&t4_list_lock); 10440 goto done_unload; 10441 } 10442 #endif 10443 tries = 0; 10444 while (tries++ < 5 && t4_sge_extfree_refs() != 0) { 10445 uprintf("%ju clusters with custom free routine " 10446 "still is use.\n", t4_sge_extfree_refs()); 10447 pause("t4unload", 2 * hz); 10448 } 10449 #ifdef TCP_OFFLOAD 10450 sx_sunlock(&t4_uld_list_lock); 10451 #endif 10452 sx_sunlock(&t4_list_lock); 10453 10454 if (t4_sge_extfree_refs() == 0) { 10455 t4_tracer_modunload(); 10456 #ifdef TCP_OFFLOAD 10457 sx_destroy(&t4_uld_list_lock); 10458 #endif 10459 sx_destroy(&t4_list_lock); 10460 t4_sge_modunload(); 10461 loaded = 0; 10462 } else { 10463 rc = EBUSY; 10464 loaded++; /* undo earlier decrement */ 10465 } 10466 } 10467 done_unload: 10468 sx_xunlock(&mlu); 10469 break; 10470 } 10471 10472 return (rc); 10473 } 10474 10475 static devclass_t t4_devclass, t5_devclass, t6_devclass; 10476 static devclass_t cxgbe_devclass, cxl_devclass, cc_devclass; 10477 static devclass_t vcxgbe_devclass, vcxl_devclass, vcc_devclass; 10478 10479 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 10480 MODULE_VERSION(t4nex, 1); 10481 MODULE_DEPEND(t4nex, firmware, 1, 1, 1); 10482 #ifdef DEV_NETMAP 10483 MODULE_DEPEND(t4nex, netmap, 1, 1, 1); 10484 #endif /* DEV_NETMAP */ 10485 10486 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 10487 MODULE_VERSION(t5nex, 1); 10488 MODULE_DEPEND(t5nex, firmware, 1, 1, 1); 10489 #ifdef DEV_NETMAP 10490 MODULE_DEPEND(t5nex, netmap, 1, 1, 1); 10491 #endif /* DEV_NETMAP */ 10492 10493 DRIVER_MODULE(t6nex, pci, t6_driver, t6_devclass, mod_event, 0); 10494 MODULE_VERSION(t6nex, 1); 10495 MODULE_DEPEND(t6nex, firmware, 1, 1, 1); 10496 #ifdef DEV_NETMAP 10497 MODULE_DEPEND(t6nex, netmap, 1, 1, 1); 10498 #endif /* DEV_NETMAP */ 10499 10500 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 10501 MODULE_VERSION(cxgbe, 1); 10502 10503 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 10504 MODULE_VERSION(cxl, 1); 10505 10506 DRIVER_MODULE(cc, t6nex, cc_driver, cc_devclass, 0, 0); 10507 MODULE_VERSION(cc, 1); 10508 10509 DRIVER_MODULE(vcxgbe, cxgbe, vcxgbe_driver, vcxgbe_devclass, 0, 0); 10510 MODULE_VERSION(vcxgbe, 1); 10511 10512 DRIVER_MODULE(vcxl, cxl, vcxl_driver, vcxl_devclass, 0, 0); 10513 MODULE_VERSION(vcxl, 1); 10514 10515 DRIVER_MODULE(vcc, cc, vcc_driver, vcc_devclass, 0, 0); 10516 MODULE_VERSION(vcc, 1); 10517