1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2011, 2025 Chelsio Communications. 5 * Written by: Navdeep Parhar <np@FreeBSD.org> 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 #include "opt_ddb.h" 31 #include "opt_inet.h" 32 #include "opt_inet6.h" 33 #include "opt_kern_tls.h" 34 #include "opt_ratelimit.h" 35 #include "opt_rss.h" 36 37 #include <sys/param.h> 38 #include <sys/conf.h> 39 #include <sys/priv.h> 40 #include <sys/kernel.h> 41 #include <sys/bus.h> 42 #include <sys/eventhandler.h> 43 #include <sys/module.h> 44 #include <sys/malloc.h> 45 #include <sys/queue.h> 46 #include <sys/taskqueue.h> 47 #include <dev/pci/pcireg.h> 48 #include <dev/pci/pcivar.h> 49 #include <sys/firmware.h> 50 #include <sys/sbuf.h> 51 #include <sys/smp.h> 52 #include <sys/socket.h> 53 #include <sys/sockio.h> 54 #include <sys/sysctl.h> 55 #include <net/ethernet.h> 56 #include <net/if.h> 57 #include <net/if_types.h> 58 #include <net/if_dl.h> 59 #include <net/if_vlan_var.h> 60 #include <net/rss_config.h> 61 #include <netinet/in.h> 62 #include <netinet/ip.h> 63 #ifdef KERN_TLS 64 #include <netinet/tcp_seq.h> 65 #endif 66 #if defined(__i386__) || defined(__amd64__) 67 #include <machine/md_var.h> 68 #include <machine/cputypes.h> 69 #include <vm/vm.h> 70 #include <vm/pmap.h> 71 #endif 72 #ifdef DDB 73 #include <ddb/ddb.h> 74 #include <ddb/db_lex.h> 75 #endif 76 77 #include "common/common.h" 78 #include "common/t4_msg.h" 79 #include "common/t4_regs.h" 80 #include "common/t4_regs_values.h" 81 #include "cudbg/cudbg.h" 82 #include "t4_clip.h" 83 #include "t4_ioctl.h" 84 #include "t4_l2t.h" 85 #include "t4_mp_ring.h" 86 #include "t4_if.h" 87 #include "t4_smt.h" 88 89 /* T4 bus driver interface */ 90 static int t4_probe(device_t); 91 static int t4_attach(device_t); 92 static int t4_detach(device_t); 93 static int t4_child_location(device_t, device_t, struct sbuf *); 94 static int t4_ready(device_t); 95 static int t4_read_port_device(device_t, int, device_t *); 96 static int t4_suspend(device_t); 97 static int t4_resume(device_t); 98 static int t4_reset_prepare(device_t, device_t); 99 static int t4_reset_post(device_t, device_t); 100 static device_method_t t4_methods[] = { 101 DEVMETHOD(device_probe, t4_probe), 102 DEVMETHOD(device_attach, t4_attach), 103 DEVMETHOD(device_detach, t4_detach), 104 DEVMETHOD(device_suspend, t4_suspend), 105 DEVMETHOD(device_resume, t4_resume), 106 107 DEVMETHOD(bus_child_location, t4_child_location), 108 DEVMETHOD(bus_reset_prepare, t4_reset_prepare), 109 DEVMETHOD(bus_reset_post, t4_reset_post), 110 111 DEVMETHOD(t4_is_main_ready, t4_ready), 112 DEVMETHOD(t4_read_port_device, t4_read_port_device), 113 114 DEVMETHOD_END 115 }; 116 static driver_t t4_driver = { 117 "t4nex", 118 t4_methods, 119 sizeof(struct adapter) 120 }; 121 122 123 /* T4 port (cxgbe) interface */ 124 static int cxgbe_probe(device_t); 125 static int cxgbe_attach(device_t); 126 static int cxgbe_detach(device_t); 127 device_method_t cxgbe_methods[] = { 128 DEVMETHOD(device_probe, cxgbe_probe), 129 DEVMETHOD(device_attach, cxgbe_attach), 130 DEVMETHOD(device_detach, cxgbe_detach), 131 DEVMETHOD_END 132 }; 133 static driver_t cxgbe_driver = { 134 "cxgbe", 135 cxgbe_methods, 136 sizeof(struct port_info) 137 }; 138 139 /* T4 VI (vcxgbe) interface */ 140 static int vcxgbe_probe(device_t); 141 static int vcxgbe_attach(device_t); 142 static int vcxgbe_detach(device_t); 143 static device_method_t vcxgbe_methods[] = { 144 DEVMETHOD(device_probe, vcxgbe_probe), 145 DEVMETHOD(device_attach, vcxgbe_attach), 146 DEVMETHOD(device_detach, vcxgbe_detach), 147 DEVMETHOD_END 148 }; 149 static driver_t vcxgbe_driver = { 150 "vcxgbe", 151 vcxgbe_methods, 152 sizeof(struct vi_info) 153 }; 154 155 static d_ioctl_t t4_ioctl; 156 157 static struct cdevsw t4_cdevsw = { 158 .d_version = D_VERSION, 159 .d_ioctl = t4_ioctl, 160 .d_name = "t4nex", 161 }; 162 163 /* T5 bus driver interface */ 164 static int t5_probe(device_t); 165 static device_method_t t5_methods[] = { 166 DEVMETHOD(device_probe, t5_probe), 167 DEVMETHOD(device_attach, t4_attach), 168 DEVMETHOD(device_detach, t4_detach), 169 DEVMETHOD(device_suspend, t4_suspend), 170 DEVMETHOD(device_resume, t4_resume), 171 172 DEVMETHOD(bus_child_location, t4_child_location), 173 DEVMETHOD(bus_reset_prepare, t4_reset_prepare), 174 DEVMETHOD(bus_reset_post, t4_reset_post), 175 176 DEVMETHOD(t4_is_main_ready, t4_ready), 177 DEVMETHOD(t4_read_port_device, t4_read_port_device), 178 179 DEVMETHOD_END 180 }; 181 static driver_t t5_driver = { 182 "t5nex", 183 t5_methods, 184 sizeof(struct adapter) 185 }; 186 187 188 /* T5 port (cxl) interface */ 189 static driver_t cxl_driver = { 190 "cxl", 191 cxgbe_methods, 192 sizeof(struct port_info) 193 }; 194 195 /* T5 VI (vcxl) interface */ 196 static driver_t vcxl_driver = { 197 "vcxl", 198 vcxgbe_methods, 199 sizeof(struct vi_info) 200 }; 201 202 /* T6 bus driver interface */ 203 static int t6_probe(device_t); 204 static device_method_t t6_methods[] = { 205 DEVMETHOD(device_probe, t6_probe), 206 DEVMETHOD(device_attach, t4_attach), 207 DEVMETHOD(device_detach, t4_detach), 208 DEVMETHOD(device_suspend, t4_suspend), 209 DEVMETHOD(device_resume, t4_resume), 210 211 DEVMETHOD(bus_child_location, t4_child_location), 212 DEVMETHOD(bus_reset_prepare, t4_reset_prepare), 213 DEVMETHOD(bus_reset_post, t4_reset_post), 214 215 DEVMETHOD(t4_is_main_ready, t4_ready), 216 DEVMETHOD(t4_read_port_device, t4_read_port_device), 217 218 DEVMETHOD_END 219 }; 220 static driver_t t6_driver = { 221 "t6nex", 222 t6_methods, 223 sizeof(struct adapter) 224 }; 225 226 227 /* T6 port (cc) interface */ 228 static driver_t cc_driver = { 229 "cc", 230 cxgbe_methods, 231 sizeof(struct port_info) 232 }; 233 234 /* T6 VI (vcc) interface */ 235 static driver_t vcc_driver = { 236 "vcc", 237 vcxgbe_methods, 238 sizeof(struct vi_info) 239 }; 240 241 /* T7+ bus driver interface */ 242 static int ch_probe(device_t); 243 static device_method_t ch_methods[] = { 244 DEVMETHOD(device_probe, ch_probe), 245 DEVMETHOD(device_attach, t4_attach), 246 DEVMETHOD(device_detach, t4_detach), 247 DEVMETHOD(device_suspend, t4_suspend), 248 DEVMETHOD(device_resume, t4_resume), 249 250 DEVMETHOD(bus_child_location, t4_child_location), 251 DEVMETHOD(bus_reset_prepare, t4_reset_prepare), 252 DEVMETHOD(bus_reset_post, t4_reset_post), 253 254 DEVMETHOD(t4_is_main_ready, t4_ready), 255 DEVMETHOD(t4_read_port_device, t4_read_port_device), 256 257 DEVMETHOD_END 258 }; 259 static driver_t ch_driver = { 260 "chnex", 261 ch_methods, 262 sizeof(struct adapter) 263 }; 264 265 266 /* T7+ port (che) interface */ 267 static driver_t che_driver = { 268 "che", 269 cxgbe_methods, 270 sizeof(struct port_info) 271 }; 272 273 /* T7+ VI (vche) interface */ 274 static driver_t vche_driver = { 275 "vche", 276 vcxgbe_methods, 277 sizeof(struct vi_info) 278 }; 279 280 /* ifnet interface */ 281 static void cxgbe_init(void *); 282 static int cxgbe_ioctl(if_t, unsigned long, caddr_t); 283 static int cxgbe_transmit(if_t, struct mbuf *); 284 static void cxgbe_qflush(if_t); 285 #if defined(KERN_TLS) || defined(RATELIMIT) 286 static int cxgbe_snd_tag_alloc(if_t, union if_snd_tag_alloc_params *, 287 struct m_snd_tag **); 288 #endif 289 290 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 291 292 /* 293 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 294 * then ADAPTER_LOCK, then t4_uld_list_lock. 295 */ 296 static struct sx t4_list_lock; 297 SLIST_HEAD(, adapter) t4_list; 298 #ifdef TCP_OFFLOAD 299 static struct sx t4_uld_list_lock; 300 struct uld_info *t4_uld_list[ULD_MAX + 1]; 301 #endif 302 303 /* 304 * Tunables. See tweak_tunables() too. 305 * 306 * Each tunable is set to a default value here if it's known at compile-time. 307 * Otherwise it is set to -n as an indication to tweak_tunables() that it should 308 * provide a reasonable default (upto n) when the driver is loaded. 309 * 310 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 311 * T5 are under hw.cxl. 312 */ 313 SYSCTL_NODE(_hw, OID_AUTO, cxgbe, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 314 "cxgbe(4) parameters"); 315 SYSCTL_NODE(_hw, OID_AUTO, cxl, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 316 "cxgbe(4) T5+ parameters"); 317 SYSCTL_NODE(_hw_cxgbe, OID_AUTO, toe, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 318 "cxgbe(4) TOE parameters"); 319 320 /* 321 * Number of queues for tx and rx, NIC and offload. 322 */ 323 #define NTXQ 16 324 int t4_ntxq = -NTXQ; 325 SYSCTL_INT(_hw_cxgbe, OID_AUTO, ntxq, CTLFLAG_RDTUN, &t4_ntxq, 0, 326 "Number of TX queues per port"); 327 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq); /* Old name, undocumented */ 328 329 #define NRXQ 8 330 int t4_nrxq = -NRXQ; 331 SYSCTL_INT(_hw_cxgbe, OID_AUTO, nrxq, CTLFLAG_RDTUN, &t4_nrxq, 0, 332 "Number of RX queues per port"); 333 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq); /* Old name, undocumented */ 334 335 #define NTXQ_VI 1 336 static int t4_ntxq_vi = -NTXQ_VI; 337 SYSCTL_INT(_hw_cxgbe, OID_AUTO, ntxq_vi, CTLFLAG_RDTUN, &t4_ntxq_vi, 0, 338 "Number of TX queues per VI"); 339 340 #define NRXQ_VI 1 341 static int t4_nrxq_vi = -NRXQ_VI; 342 SYSCTL_INT(_hw_cxgbe, OID_AUTO, nrxq_vi, CTLFLAG_RDTUN, &t4_nrxq_vi, 0, 343 "Number of RX queues per VI"); 344 345 static int t4_rsrv_noflowq = 0; 346 SYSCTL_INT(_hw_cxgbe, OID_AUTO, rsrv_noflowq, CTLFLAG_RDTUN, &t4_rsrv_noflowq, 347 0, "Reserve TX queue 0 of each VI for non-flowid packets"); 348 349 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 350 #define NOFLDTXQ 8 351 static int t4_nofldtxq = -NOFLDTXQ; 352 SYSCTL_INT(_hw_cxgbe, OID_AUTO, nofldtxq, CTLFLAG_RDTUN, &t4_nofldtxq, 0, 353 "Number of offload TX queues per port"); 354 355 #define NOFLDTXQ_VI 1 356 static int t4_nofldtxq_vi = -NOFLDTXQ_VI; 357 SYSCTL_INT(_hw_cxgbe, OID_AUTO, nofldtxq_vi, CTLFLAG_RDTUN, &t4_nofldtxq_vi, 0, 358 "Number of offload TX queues per VI"); 359 #endif 360 361 #if defined(TCP_OFFLOAD) 362 #define NOFLDRXQ 2 363 static int t4_nofldrxq = -NOFLDRXQ; 364 SYSCTL_INT(_hw_cxgbe, OID_AUTO, nofldrxq, CTLFLAG_RDTUN, &t4_nofldrxq, 0, 365 "Number of offload RX queues per port"); 366 367 #define NOFLDRXQ_VI 1 368 static int t4_nofldrxq_vi = -NOFLDRXQ_VI; 369 SYSCTL_INT(_hw_cxgbe, OID_AUTO, nofldrxq_vi, CTLFLAG_RDTUN, &t4_nofldrxq_vi, 0, 370 "Number of offload RX queues per VI"); 371 372 #define TMR_IDX_OFLD 1 373 static int t4_tmr_idx_ofld = TMR_IDX_OFLD; 374 SYSCTL_INT(_hw_cxgbe, OID_AUTO, holdoff_timer_idx_ofld, CTLFLAG_RDTUN, 375 &t4_tmr_idx_ofld, 0, "Holdoff timer index for offload queues"); 376 377 #define PKTC_IDX_OFLD (-1) 378 static int t4_pktc_idx_ofld = PKTC_IDX_OFLD; 379 SYSCTL_INT(_hw_cxgbe, OID_AUTO, holdoff_pktc_idx_ofld, CTLFLAG_RDTUN, 380 &t4_pktc_idx_ofld, 0, "holdoff packet counter index for offload queues"); 381 382 /* 0 means chip/fw default, non-zero number is value in microseconds */ 383 static u_long t4_toe_keepalive_idle = 0; 384 SYSCTL_ULONG(_hw_cxgbe_toe, OID_AUTO, keepalive_idle, CTLFLAG_RDTUN, 385 &t4_toe_keepalive_idle, 0, "TOE keepalive idle timer (us)"); 386 387 /* 0 means chip/fw default, non-zero number is value in microseconds */ 388 static u_long t4_toe_keepalive_interval = 0; 389 SYSCTL_ULONG(_hw_cxgbe_toe, OID_AUTO, keepalive_interval, CTLFLAG_RDTUN, 390 &t4_toe_keepalive_interval, 0, "TOE keepalive interval timer (us)"); 391 392 /* 0 means chip/fw default, non-zero number is # of keepalives before abort */ 393 static int t4_toe_keepalive_count = 0; 394 SYSCTL_INT(_hw_cxgbe_toe, OID_AUTO, keepalive_count, CTLFLAG_RDTUN, 395 &t4_toe_keepalive_count, 0, "Number of TOE keepalive probes before abort"); 396 397 /* 0 means chip/fw default, non-zero number is value in microseconds */ 398 static u_long t4_toe_rexmt_min = 0; 399 SYSCTL_ULONG(_hw_cxgbe_toe, OID_AUTO, rexmt_min, CTLFLAG_RDTUN, 400 &t4_toe_rexmt_min, 0, "Minimum TOE retransmit interval (us)"); 401 402 /* 0 means chip/fw default, non-zero number is value in microseconds */ 403 static u_long t4_toe_rexmt_max = 0; 404 SYSCTL_ULONG(_hw_cxgbe_toe, OID_AUTO, rexmt_max, CTLFLAG_RDTUN, 405 &t4_toe_rexmt_max, 0, "Maximum TOE retransmit interval (us)"); 406 407 /* 0 means chip/fw default, non-zero number is # of rexmt before abort */ 408 static int t4_toe_rexmt_count = 0; 409 SYSCTL_INT(_hw_cxgbe_toe, OID_AUTO, rexmt_count, CTLFLAG_RDTUN, 410 &t4_toe_rexmt_count, 0, "Number of TOE retransmissions before abort"); 411 412 /* -1 means chip/fw default, other values are raw backoff values to use */ 413 static int t4_toe_rexmt_backoff[16] = { 414 -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 415 }; 416 SYSCTL_NODE(_hw_cxgbe_toe, OID_AUTO, rexmt_backoff, 417 CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 418 "cxgbe(4) TOE retransmit backoff values"); 419 SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 0, CTLFLAG_RDTUN, 420 &t4_toe_rexmt_backoff[0], 0, ""); 421 SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 1, CTLFLAG_RDTUN, 422 &t4_toe_rexmt_backoff[1], 0, ""); 423 SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 2, CTLFLAG_RDTUN, 424 &t4_toe_rexmt_backoff[2], 0, ""); 425 SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 3, CTLFLAG_RDTUN, 426 &t4_toe_rexmt_backoff[3], 0, ""); 427 SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 4, CTLFLAG_RDTUN, 428 &t4_toe_rexmt_backoff[4], 0, ""); 429 SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 5, CTLFLAG_RDTUN, 430 &t4_toe_rexmt_backoff[5], 0, ""); 431 SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 6, CTLFLAG_RDTUN, 432 &t4_toe_rexmt_backoff[6], 0, ""); 433 SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 7, CTLFLAG_RDTUN, 434 &t4_toe_rexmt_backoff[7], 0, ""); 435 SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 8, CTLFLAG_RDTUN, 436 &t4_toe_rexmt_backoff[8], 0, ""); 437 SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 9, CTLFLAG_RDTUN, 438 &t4_toe_rexmt_backoff[9], 0, ""); 439 SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 10, CTLFLAG_RDTUN, 440 &t4_toe_rexmt_backoff[10], 0, ""); 441 SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 11, CTLFLAG_RDTUN, 442 &t4_toe_rexmt_backoff[11], 0, ""); 443 SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 12, CTLFLAG_RDTUN, 444 &t4_toe_rexmt_backoff[12], 0, ""); 445 SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 13, CTLFLAG_RDTUN, 446 &t4_toe_rexmt_backoff[13], 0, ""); 447 SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 14, CTLFLAG_RDTUN, 448 &t4_toe_rexmt_backoff[14], 0, ""); 449 SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 15, CTLFLAG_RDTUN, 450 &t4_toe_rexmt_backoff[15], 0, ""); 451 452 int t4_ddp_rcvbuf_len = 256 * 1024; 453 SYSCTL_INT(_hw_cxgbe_toe, OID_AUTO, ddp_rcvbuf_len, CTLFLAG_RWTUN, 454 &t4_ddp_rcvbuf_len, 0, "length of each DDP RX buffer"); 455 456 unsigned int t4_ddp_rcvbuf_cache = 4; 457 SYSCTL_UINT(_hw_cxgbe_toe, OID_AUTO, ddp_rcvbuf_cache, CTLFLAG_RWTUN, 458 &t4_ddp_rcvbuf_cache, 0, 459 "maximum number of free DDP RX buffers to cache per connection"); 460 #endif 461 462 #ifdef DEV_NETMAP 463 #define NN_MAIN_VI (1 << 0) /* Native netmap on the main VI */ 464 #define NN_EXTRA_VI (1 << 1) /* Native netmap on the extra VI(s) */ 465 static int t4_native_netmap = NN_EXTRA_VI; 466 SYSCTL_INT(_hw_cxgbe, OID_AUTO, native_netmap, CTLFLAG_RDTUN, &t4_native_netmap, 467 0, "Native netmap support. bit 0 = main VI, bit 1 = extra VIs"); 468 469 #define NNMTXQ 8 470 static int t4_nnmtxq = -NNMTXQ; 471 SYSCTL_INT(_hw_cxgbe, OID_AUTO, nnmtxq, CTLFLAG_RDTUN, &t4_nnmtxq, 0, 472 "Number of netmap TX queues"); 473 474 #define NNMRXQ 8 475 static int t4_nnmrxq = -NNMRXQ; 476 SYSCTL_INT(_hw_cxgbe, OID_AUTO, nnmrxq, CTLFLAG_RDTUN, &t4_nnmrxq, 0, 477 "Number of netmap RX queues"); 478 479 #define NNMTXQ_VI 2 480 static int t4_nnmtxq_vi = -NNMTXQ_VI; 481 SYSCTL_INT(_hw_cxgbe, OID_AUTO, nnmtxq_vi, CTLFLAG_RDTUN, &t4_nnmtxq_vi, 0, 482 "Number of netmap TX queues per VI"); 483 484 #define NNMRXQ_VI 2 485 static int t4_nnmrxq_vi = -NNMRXQ_VI; 486 SYSCTL_INT(_hw_cxgbe, OID_AUTO, nnmrxq_vi, CTLFLAG_RDTUN, &t4_nnmrxq_vi, 0, 487 "Number of netmap RX queues per VI"); 488 #endif 489 490 /* 491 * Holdoff parameters for ports. 492 */ 493 #define TMR_IDX 1 494 int t4_tmr_idx = TMR_IDX; 495 SYSCTL_INT(_hw_cxgbe, OID_AUTO, holdoff_timer_idx, CTLFLAG_RDTUN, &t4_tmr_idx, 496 0, "Holdoff timer index"); 497 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx); /* Old name */ 498 499 #define PKTC_IDX (-1) 500 int t4_pktc_idx = PKTC_IDX; 501 SYSCTL_INT(_hw_cxgbe, OID_AUTO, holdoff_pktc_idx, CTLFLAG_RDTUN, &t4_pktc_idx, 502 0, "Holdoff packet counter index"); 503 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx); /* Old name */ 504 505 /* 506 * Size (# of entries) of each tx and rx queue. 507 */ 508 unsigned int t4_qsize_txq = TX_EQ_QSIZE; 509 SYSCTL_INT(_hw_cxgbe, OID_AUTO, qsize_txq, CTLFLAG_RDTUN, &t4_qsize_txq, 0, 510 "Number of descriptors in each TX queue"); 511 512 unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 513 SYSCTL_INT(_hw_cxgbe, OID_AUTO, qsize_rxq, CTLFLAG_RDTUN, &t4_qsize_rxq, 0, 514 "Number of descriptors in each RX queue"); 515 516 /* 517 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 518 */ 519 int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 520 SYSCTL_INT(_hw_cxgbe, OID_AUTO, interrupt_types, CTLFLAG_RDTUN, &t4_intr_types, 521 0, "Interrupt types allowed (bit 0 = INTx, 1 = MSI, 2 = MSI-X)"); 522 523 /* 524 * Configuration file. All the _CF names here are special. 525 */ 526 #define DEFAULT_CF "default" 527 #define BUILTIN_CF "built-in" 528 #define FLASH_CF "flash" 529 #define UWIRE_CF "uwire" 530 #define FPGA_CF "fpga" 531 static char t4_cfg_file[32] = DEFAULT_CF; 532 SYSCTL_STRING(_hw_cxgbe, OID_AUTO, config_file, CTLFLAG_RDTUN, t4_cfg_file, 533 sizeof(t4_cfg_file), "Firmware configuration file"); 534 535 /* 536 * PAUSE settings (bit 0, 1, 2 = rx_pause, tx_pause, pause_autoneg respectively). 537 * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them. 538 * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water 539 * mark or when signalled to do so, 0 to never emit PAUSE. 540 * pause_autoneg = 1 means PAUSE will be negotiated if possible and the 541 * negotiated settings will override rx_pause/tx_pause. 542 * Otherwise rx_pause/tx_pause are applied forcibly. 543 */ 544 static int t4_pause_settings = PAUSE_RX | PAUSE_TX | PAUSE_AUTONEG; 545 SYSCTL_INT(_hw_cxgbe, OID_AUTO, pause_settings, CTLFLAG_RDTUN, 546 &t4_pause_settings, 0, 547 "PAUSE settings (bit 0 = rx_pause, 1 = tx_pause, 2 = pause_autoneg)"); 548 549 /* 550 * Forward Error Correction settings (bit 0, 1 = RS, BASER respectively). 551 * -1 to run with the firmware default. Same as FEC_AUTO (bit 5) 552 * 0 to disable FEC. 553 */ 554 static int t4_fec = -1; 555 SYSCTL_INT(_hw_cxgbe, OID_AUTO, fec, CTLFLAG_RDTUN, &t4_fec, 0, 556 "Forward Error Correction (bit 0 = RS, bit 1 = BASER_RS)"); 557 558 static const char * 559 t4_fec_bits = "\20\1RS-FEC\2FC-FEC\3NO-FEC\4RSVD1\5RSVD2\6auto\7module"; 560 561 /* 562 * Controls when the driver sets the FORCE_FEC bit in the L1_CFG32 that it 563 * issues to the firmware. If the firmware doesn't support FORCE_FEC then the 564 * driver runs as if this is set to 0. 565 * -1 to set FORCE_FEC iff requested_fec != AUTO. Multiple FEC bits are okay. 566 * 0 to never set FORCE_FEC. requested_fec = AUTO means use the hint from the 567 * transceiver. Multiple FEC bits may not be okay but will be passed on to 568 * the firmware anyway (may result in l1cfg errors with old firmwares). 569 * 1 to always set FORCE_FEC. Multiple FEC bits are okay. requested_fec = AUTO 570 * means set all FEC bits that are valid for the speed. 571 */ 572 static int t4_force_fec = -1; 573 SYSCTL_INT(_hw_cxgbe, OID_AUTO, force_fec, CTLFLAG_RDTUN, &t4_force_fec, 0, 574 "Controls the use of FORCE_FEC bit in L1 configuration."); 575 576 /* 577 * Link autonegotiation. 578 * -1 to run with the firmware default. 579 * 0 to disable. 580 * 1 to enable. 581 */ 582 static int t4_autoneg = -1; 583 SYSCTL_INT(_hw_cxgbe, OID_AUTO, autoneg, CTLFLAG_RDTUN, &t4_autoneg, 0, 584 "Link autonegotiation"); 585 586 /* 587 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 588 * encouraged respectively). '-n' is the same as 'n' except the firmware 589 * version used in the checks is read from the firmware bundled with the driver. 590 */ 591 static int t4_fw_install = 1; 592 SYSCTL_INT(_hw_cxgbe, OID_AUTO, fw_install, CTLFLAG_RDTUN, &t4_fw_install, 0, 593 "Firmware auto-install (0 = prohibited, 1 = allowed, 2 = encouraged)"); 594 595 /* 596 * ASIC features that will be used. Disable the ones you don't want so that the 597 * chip resources aren't wasted on features that will not be used. 598 */ 599 static int t4_nbmcaps_allowed = 0; 600 SYSCTL_INT(_hw_cxgbe, OID_AUTO, nbmcaps_allowed, CTLFLAG_RDTUN, 601 &t4_nbmcaps_allowed, 0, "Default NBM capabilities"); 602 603 static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 604 SYSCTL_INT(_hw_cxgbe, OID_AUTO, linkcaps_allowed, CTLFLAG_RDTUN, 605 &t4_linkcaps_allowed, 0, "Default link capabilities"); 606 607 static int t4_switchcaps_allowed = FW_CAPS_CONFIG_SWITCH_INGRESS | 608 FW_CAPS_CONFIG_SWITCH_EGRESS; 609 SYSCTL_INT(_hw_cxgbe, OID_AUTO, switchcaps_allowed, CTLFLAG_RDTUN, 610 &t4_switchcaps_allowed, 0, "Default switch capabilities"); 611 612 static int t4_nvmecaps_allowed = -1; 613 SYSCTL_INT(_hw_cxgbe, OID_AUTO, nvmecaps_allowed, CTLFLAG_RDTUN, 614 &t4_nvmecaps_allowed, 0, "Default NVMe capabilities"); 615 616 #ifdef RATELIMIT 617 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC | 618 FW_CAPS_CONFIG_NIC_HASHFILTER | FW_CAPS_CONFIG_NIC_ETHOFLD; 619 #else 620 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC | 621 FW_CAPS_CONFIG_NIC_HASHFILTER; 622 #endif 623 SYSCTL_INT(_hw_cxgbe, OID_AUTO, niccaps_allowed, CTLFLAG_RDTUN, 624 &t4_niccaps_allowed, 0, "Default NIC capabilities"); 625 626 static int t4_toecaps_allowed = -1; 627 SYSCTL_INT(_hw_cxgbe, OID_AUTO, toecaps_allowed, CTLFLAG_RDTUN, 628 &t4_toecaps_allowed, 0, "Default TCP offload capabilities"); 629 630 static int t4_rdmacaps_allowed = -1; 631 SYSCTL_INT(_hw_cxgbe, OID_AUTO, rdmacaps_allowed, CTLFLAG_RDTUN, 632 &t4_rdmacaps_allowed, 0, "Default RDMA capabilities"); 633 634 static int t4_cryptocaps_allowed = -1; 635 SYSCTL_INT(_hw_cxgbe, OID_AUTO, cryptocaps_allowed, CTLFLAG_RDTUN, 636 &t4_cryptocaps_allowed, 0, "Default crypto capabilities"); 637 638 static int t4_iscsicaps_allowed = -1; 639 SYSCTL_INT(_hw_cxgbe, OID_AUTO, iscsicaps_allowed, CTLFLAG_RDTUN, 640 &t4_iscsicaps_allowed, 0, "Default iSCSI capabilities"); 641 642 static int t4_fcoecaps_allowed = 0; 643 SYSCTL_INT(_hw_cxgbe, OID_AUTO, fcoecaps_allowed, CTLFLAG_RDTUN, 644 &t4_fcoecaps_allowed, 0, "Default FCoE capabilities"); 645 646 static int t5_write_combine = 0; 647 SYSCTL_INT(_hw_cxl, OID_AUTO, write_combine, CTLFLAG_RDTUN, &t5_write_combine, 648 0, "Use WC instead of UC for BAR2"); 649 650 /* From t4_sysctls: doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"} */ 651 static int t4_doorbells_allowed = 0xf; 652 SYSCTL_INT(_hw_cxgbe, OID_AUTO, doorbells_allowed, CTLFLAG_RDTUN, 653 &t4_doorbells_allowed, 0, "Limit tx queues to these doorbells"); 654 655 static int t4_num_vis = 1; 656 SYSCTL_INT(_hw_cxgbe, OID_AUTO, num_vis, CTLFLAG_RDTUN, &t4_num_vis, 0, 657 "Number of VIs per port"); 658 659 /* 660 * PCIe Relaxed Ordering. 661 * -1: driver should figure out a good value. 662 * 0: disable RO. 663 * 1: enable RO. 664 * 2: leave RO alone. 665 */ 666 static int pcie_relaxed_ordering = -1; 667 SYSCTL_INT(_hw_cxgbe, OID_AUTO, pcie_relaxed_ordering, CTLFLAG_RDTUN, 668 &pcie_relaxed_ordering, 0, 669 "PCIe Relaxed Ordering: 0 = disable, 1 = enable, 2 = leave alone"); 670 671 static int t4_panic_on_fatal_err = 0; 672 SYSCTL_INT(_hw_cxgbe, OID_AUTO, panic_on_fatal_err, CTLFLAG_RWTUN, 673 &t4_panic_on_fatal_err, 0, "panic on fatal errors"); 674 675 static int t4_reset_on_fatal_err = 0; 676 SYSCTL_INT(_hw_cxgbe, OID_AUTO, reset_on_fatal_err, CTLFLAG_RWTUN, 677 &t4_reset_on_fatal_err, 0, "reset adapter on fatal errors"); 678 679 static int t4_reset_method = 1; 680 SYSCTL_INT(_hw_cxgbe, OID_AUTO, reset_method, CTLFLAG_RWTUN, &t4_reset_method, 681 0, "reset method: 0 = PL_RST, 1 = PCIe secondary bus reset, 2 = PCIe link bounce"); 682 683 static int t4_clock_gate_on_suspend = 0; 684 SYSCTL_INT(_hw_cxgbe, OID_AUTO, clock_gate_on_suspend, CTLFLAG_RWTUN, 685 &t4_clock_gate_on_suspend, 0, "gate the clock on suspend"); 686 687 static int t4_tx_vm_wr = 0; 688 SYSCTL_INT(_hw_cxgbe, OID_AUTO, tx_vm_wr, CTLFLAG_RWTUN, &t4_tx_vm_wr, 0, 689 "Use VM work requests to transmit packets."); 690 691 /* 692 * Set to non-zero to enable the attack filter. A packet that matches any of 693 * these conditions will get dropped on ingress: 694 * 1) IP && source address == destination address. 695 * 2) TCP/IP && source address is not a unicast address. 696 * 3) TCP/IP && destination address is not a unicast address. 697 * 4) IP && source address is loopback (127.x.y.z). 698 * 5) IP && destination address is loopback (127.x.y.z). 699 * 6) IPv6 && source address == destination address. 700 * 7) IPv6 && source address is not a unicast address. 701 * 8) IPv6 && source address is loopback (::1/128). 702 * 9) IPv6 && destination address is loopback (::1/128). 703 * 10) IPv6 && source address is unspecified (::/128). 704 * 11) IPv6 && destination address is unspecified (::/128). 705 * 12) TCP/IPv6 && source address is multicast (ff00::/8). 706 * 13) TCP/IPv6 && destination address is multicast (ff00::/8). 707 */ 708 static int t4_attack_filter = 0; 709 SYSCTL_INT(_hw_cxgbe, OID_AUTO, attack_filter, CTLFLAG_RDTUN, 710 &t4_attack_filter, 0, "Drop suspicious traffic"); 711 712 static int t4_drop_ip_fragments = 0; 713 SYSCTL_INT(_hw_cxgbe, OID_AUTO, drop_ip_fragments, CTLFLAG_RDTUN, 714 &t4_drop_ip_fragments, 0, "Drop IP fragments"); 715 716 static int t4_drop_pkts_with_l2_errors = 1; 717 SYSCTL_INT(_hw_cxgbe, OID_AUTO, drop_pkts_with_l2_errors, CTLFLAG_RDTUN, 718 &t4_drop_pkts_with_l2_errors, 0, 719 "Drop all frames with Layer 2 length or checksum errors"); 720 721 static int t4_drop_pkts_with_l3_errors = 0; 722 SYSCTL_INT(_hw_cxgbe, OID_AUTO, drop_pkts_with_l3_errors, CTLFLAG_RDTUN, 723 &t4_drop_pkts_with_l3_errors, 0, 724 "Drop all frames with IP version, length, or checksum errors"); 725 726 static int t4_drop_pkts_with_l4_errors = 0; 727 SYSCTL_INT(_hw_cxgbe, OID_AUTO, drop_pkts_with_l4_errors, CTLFLAG_RDTUN, 728 &t4_drop_pkts_with_l4_errors, 0, 729 "Drop all frames with Layer 4 length, checksum, or other errors"); 730 731 #ifdef TCP_OFFLOAD 732 /* 733 * TOE tunables. 734 */ 735 static int t4_cop_managed_offloading = 0; 736 SYSCTL_INT(_hw_cxgbe_toe, OID_AUTO, cop_managed_offloading, CTLFLAG_RDTUN, 737 &t4_cop_managed_offloading, 0, 738 "COP (Connection Offload Policy) controls all TOE offload"); 739 TUNABLE_INT("hw.cxgbe.cop_managed_offloading", &t4_cop_managed_offloading); 740 #endif 741 742 #ifdef KERN_TLS 743 /* 744 * This enables KERN_TLS for all adapters if set. 745 */ 746 static int t4_kern_tls = 0; 747 SYSCTL_INT(_hw_cxgbe, OID_AUTO, kern_tls, CTLFLAG_RDTUN, &t4_kern_tls, 0, 748 "Enable KERN_TLS mode for T6 adapters"); 749 750 SYSCTL_NODE(_hw_cxgbe, OID_AUTO, tls, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 751 "cxgbe(4) KERN_TLS parameters"); 752 753 static int t4_tls_inline_keys = 0; 754 SYSCTL_INT(_hw_cxgbe_tls, OID_AUTO, inline_keys, CTLFLAG_RDTUN, 755 &t4_tls_inline_keys, 0, 756 "Always pass TLS keys in work requests (1) or attempt to store TLS keys " 757 "in card memory."); 758 759 static int t4_tls_combo_wrs = 0; 760 SYSCTL_INT(_hw_cxgbe_tls, OID_AUTO, combo_wrs, CTLFLAG_RDTUN, &t4_tls_combo_wrs, 761 0, "Attempt to combine TCB field updates with TLS record work requests."); 762 763 static int t4_tls_short_records = 1; 764 SYSCTL_INT(_hw_cxgbe_tls, OID_AUTO, short_records, CTLFLAG_RDTUN, 765 &t4_tls_short_records, 0, "Use cipher-only mode for short records."); 766 767 static int t4_tls_partial_ghash = 1; 768 SYSCTL_INT(_hw_cxgbe_tls, OID_AUTO, partial_ghash, CTLFLAG_RDTUN, 769 &t4_tls_partial_ghash, 0, "Use partial GHASH for AES-GCM records."); 770 #endif 771 772 /* Functions used by VIs to obtain unique MAC addresses for each VI. */ 773 static int vi_mac_funcs[] = { 774 FW_VI_FUNC_ETH, 775 FW_VI_FUNC_OFLD, 776 FW_VI_FUNC_IWARP, 777 FW_VI_FUNC_OPENISCSI, 778 FW_VI_FUNC_OPENFCOE, 779 FW_VI_FUNC_FOISCSI, 780 FW_VI_FUNC_FOFCOE, 781 }; 782 783 struct intrs_and_queues { 784 uint16_t intr_type; /* INTx, MSI, or MSI-X */ 785 uint16_t num_vis; /* number of VIs for each port */ 786 uint16_t nirq; /* Total # of vectors */ 787 uint16_t ntxq; /* # of NIC txq's for each port */ 788 uint16_t nrxq; /* # of NIC rxq's for each port */ 789 uint16_t nofldtxq; /* # of TOE/ETHOFLD txq's for each port */ 790 uint16_t nofldrxq; /* # of TOE rxq's for each port */ 791 uint16_t nnmtxq; /* # of netmap txq's */ 792 uint16_t nnmrxq; /* # of netmap rxq's */ 793 794 /* The vcxgbe/vcxl interfaces use these and not the ones above. */ 795 uint16_t ntxq_vi; /* # of NIC txq's */ 796 uint16_t nrxq_vi; /* # of NIC rxq's */ 797 uint16_t nofldtxq_vi; /* # of TOE txq's */ 798 uint16_t nofldrxq_vi; /* # of TOE rxq's */ 799 uint16_t nnmtxq_vi; /* # of netmap txq's */ 800 uint16_t nnmrxq_vi; /* # of netmap rxq's */ 801 }; 802 803 static void setup_memwin(struct adapter *); 804 static void position_memwin(struct adapter *, int, uint32_t); 805 static int validate_mem_range(struct adapter *, uint32_t, uint32_t); 806 static int fwmtype_to_hwmtype(int); 807 static int validate_mt_off_len(struct adapter *, int, uint32_t, uint32_t, 808 uint32_t *); 809 static int fixup_devlog_ncores_params(struct adapter *); 810 static int cfg_itype_and_nqueues(struct adapter *, struct intrs_and_queues *); 811 static int contact_firmware(struct adapter *); 812 static int partition_resources(struct adapter *); 813 static int get_params__pre_init(struct adapter *); 814 static int set_params__pre_init(struct adapter *); 815 static int get_params__post_init(struct adapter *); 816 static int set_params__post_init(struct adapter *); 817 static void t4_set_desc(struct adapter *); 818 static bool fixed_ifmedia(struct port_info *); 819 static void build_medialist(struct port_info *); 820 static void init_link_config(struct port_info *); 821 static int fixup_link_config(struct port_info *); 822 static int apply_link_config(struct port_info *); 823 static int cxgbe_init_synchronized(struct vi_info *); 824 static int cxgbe_uninit_synchronized(struct vi_info *); 825 static int adapter_full_init(struct adapter *); 826 static void adapter_full_uninit(struct adapter *); 827 static int vi_full_init(struct vi_info *); 828 static void vi_full_uninit(struct vi_info *); 829 static int alloc_extra_vi(struct adapter *, struct port_info *, struct vi_info *); 830 static void quiesce_txq(struct sge_txq *); 831 static void quiesce_wrq(struct sge_wrq *); 832 static void quiesce_iq_fl(struct adapter *, struct sge_iq *, struct sge_fl *); 833 static void quiesce_vi(struct vi_info *); 834 static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 835 driver_intr_t *, void *, char *); 836 static int t4_free_irq(struct adapter *, struct irq *); 837 static void t4_init_atid_table(struct adapter *); 838 static void t4_free_atid_table(struct adapter *); 839 static void stop_atid_allocator(struct adapter *); 840 static void restart_atid_allocator(struct adapter *); 841 static void get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 842 static void vi_refresh_stats(struct vi_info *); 843 static void cxgbe_refresh_stats(struct vi_info *); 844 static void cxgbe_tick(void *); 845 static void vi_tick(void *); 846 static void cxgbe_sysctls(struct port_info *); 847 static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 848 static int sysctl_bitfield_8b(SYSCTL_HANDLER_ARGS); 849 static int sysctl_bitfield_16b(SYSCTL_HANDLER_ARGS); 850 static int sysctl_btphy(SYSCTL_HANDLER_ARGS); 851 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); 852 static int sysctl_tx_vm_wr(SYSCTL_HANDLER_ARGS); 853 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 854 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 855 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 856 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 857 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS); 858 static int sysctl_link_fec(SYSCTL_HANDLER_ARGS); 859 static int sysctl_requested_fec(SYSCTL_HANDLER_ARGS); 860 static int sysctl_module_fec(SYSCTL_HANDLER_ARGS); 861 static int sysctl_autoneg(SYSCTL_HANDLER_ARGS); 862 static int sysctl_force_fec(SYSCTL_HANDLER_ARGS); 863 static int sysctl_handle_t4_portstat64(SYSCTL_HANDLER_ARGS); 864 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 865 static int sysctl_temperature(SYSCTL_HANDLER_ARGS); 866 static int sysctl_vdd(SYSCTL_HANDLER_ARGS); 867 static int sysctl_reset_sensor(SYSCTL_HANDLER_ARGS); 868 static int sysctl_loadavg(SYSCTL_HANDLER_ARGS); 869 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 870 static int sysctl_cim_ibq(SYSCTL_HANDLER_ARGS); 871 static int sysctl_cim_obq(SYSCTL_HANDLER_ARGS); 872 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 873 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); 874 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); 875 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 876 static int sysctl_cim_qcfg_t7(SYSCTL_HANDLER_ARGS); 877 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 878 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 879 static int sysctl_tid_stats(SYSCTL_HANDLER_ARGS); 880 static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 881 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 882 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 883 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 884 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); 885 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 886 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); 887 static int sysctl_mps_tcam_t6(SYSCTL_HANDLER_ARGS); 888 static int sysctl_mps_tcam_t7(SYSCTL_HANDLER_ARGS); 889 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 890 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 891 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 892 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 893 static int sysctl_tids(SYSCTL_HANDLER_ARGS); 894 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 895 static int sysctl_tnl_stats(SYSCTL_HANDLER_ARGS); 896 static int sysctl_tp_la_mask(SYSCTL_HANDLER_ARGS); 897 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 898 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 899 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 900 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 901 static int sysctl_cpus(SYSCTL_HANDLER_ARGS); 902 static int sysctl_reset(SYSCTL_HANDLER_ARGS); 903 static int sysctl_tcb_cache(SYSCTL_HANDLER_ARGS); 904 #ifdef TCP_OFFLOAD 905 static int sysctl_tls(SYSCTL_HANDLER_ARGS); 906 static int sysctl_tp_tick(SYSCTL_HANDLER_ARGS); 907 static int sysctl_tp_dack_timer(SYSCTL_HANDLER_ARGS); 908 static int sysctl_tp_timer(SYSCTL_HANDLER_ARGS); 909 static int sysctl_tp_shift_cnt(SYSCTL_HANDLER_ARGS); 910 static int sysctl_tp_backoff(SYSCTL_HANDLER_ARGS); 911 static int sysctl_holdoff_tmr_idx_ofld(SYSCTL_HANDLER_ARGS); 912 static int sysctl_holdoff_pktc_idx_ofld(SYSCTL_HANDLER_ARGS); 913 #endif 914 static int get_sge_context(struct adapter *, int, uint32_t, int, uint32_t *); 915 static int load_fw(struct adapter *, struct t4_data *); 916 static int load_cfg(struct adapter *, struct t4_data *); 917 static int load_boot(struct adapter *, struct t4_bootrom *); 918 static int load_bootcfg(struct adapter *, struct t4_data *); 919 static int cudbg_dump(struct adapter *, struct t4_cudbg_dump *); 920 static void free_offload_policy(struct t4_offload_policy *); 921 static int set_offload_policy(struct adapter *, struct t4_offload_policy *); 922 static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 923 static int read_i2c(struct adapter *, struct t4_i2c_data *); 924 static int clear_stats(struct adapter *, u_int); 925 static int hold_clip_addr(struct adapter *, struct t4_clip_addr *); 926 static int release_clip_addr(struct adapter *, struct t4_clip_addr *); 927 static inline int stop_adapter(struct adapter *); 928 static inline void set_adapter_hwstatus(struct adapter *, const bool); 929 static int stop_lld(struct adapter *); 930 static inline int restart_adapter(struct adapter *); 931 static int restart_lld(struct adapter *); 932 #ifdef TCP_OFFLOAD 933 static int deactivate_all_uld(struct adapter *); 934 static void stop_all_uld(struct adapter *); 935 static void restart_all_uld(struct adapter *); 936 #endif 937 #ifdef KERN_TLS 938 static int ktls_capability(struct adapter *, bool); 939 #endif 940 static int mod_event(module_t, int, void *); 941 static int notify_siblings(device_t, int); 942 static uint64_t vi_get_counter(if_t, ift_counter); 943 static uint64_t cxgbe_get_counter(if_t, ift_counter); 944 static void enable_vxlan_rx(struct adapter *); 945 static void reset_adapter_task(void *, int); 946 static void fatal_error_task(void *, int); 947 static void dump_devlog(struct adapter *); 948 static void dump_cim_regs(struct adapter *); 949 static void dump_cimla(struct adapter *); 950 951 struct { 952 uint16_t device; 953 char *desc; 954 } t4_pciids[] = { 955 {0xa000, "Chelsio Terminator 4 FPGA"}, 956 {0x4400, "Chelsio T440-dbg"}, 957 {0x4401, "Chelsio T420-CR"}, 958 {0x4402, "Chelsio T422-CR"}, 959 {0x4403, "Chelsio T440-CR"}, 960 {0x4404, "Chelsio T420-BCH"}, 961 {0x4405, "Chelsio T440-BCH"}, 962 {0x4406, "Chelsio T440-CH"}, 963 {0x4407, "Chelsio T420-SO"}, 964 {0x4408, "Chelsio T420-CX"}, 965 {0x4409, "Chelsio T420-BT"}, 966 {0x440a, "Chelsio T404-BT"}, 967 {0x440e, "Chelsio T440-LP-CR"}, 968 }, t5_pciids[] = { 969 {0xb000, "Chelsio Terminator 5 FPGA"}, 970 {0x5400, "Chelsio T580-dbg"}, 971 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ 972 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ 973 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ 974 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ 975 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ 976 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ 977 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ 978 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ 979 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 980 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ 981 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ 982 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ 983 {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */ 984 {0x5418, "Chelsio T540-BT"}, /* 4 x 10GBaseT */ 985 {0x5419, "Chelsio T540-LP-BT"}, /* 4 x 10GBaseT */ 986 {0x541a, "Chelsio T540-SO-BT"}, /* 4 x 10GBaseT, nomem */ 987 {0x541b, "Chelsio T540-SO-CR"}, /* 4 x 10G, nomem */ 988 989 /* Custom */ 990 {0x5483, "Custom T540-CR"}, 991 {0x5484, "Custom T540-BT"}, 992 }, t6_pciids[] = { 993 {0xc006, "Chelsio Terminator 6 FPGA"}, /* T6 PE10K6 FPGA (PF0) */ 994 {0x6400, "Chelsio T6-DBG-25"}, /* 2 x 10/25G, debug */ 995 {0x6401, "Chelsio T6225-CR"}, /* 2 x 10/25G */ 996 {0x6402, "Chelsio T6225-SO-CR"}, /* 2 x 10/25G, nomem */ 997 {0x6403, "Chelsio T6425-CR"}, /* 4 x 10/25G */ 998 {0x6404, "Chelsio T6425-SO-CR"}, /* 4 x 10/25G, nomem */ 999 {0x6405, "Chelsio T6225-SO-OCP3"}, /* 2 x 10/25G, nomem */ 1000 {0x6406, "Chelsio T6225-OCP3"}, /* 2 x 10/25G */ 1001 {0x6407, "Chelsio T62100-LP-CR"}, /* 2 x 40/50/100G */ 1002 {0x6408, "Chelsio T62100-SO-CR"}, /* 2 x 40/50/100G, nomem */ 1003 {0x6409, "Chelsio T6210-BT"}, /* 2 x 10GBASE-T */ 1004 {0x640d, "Chelsio T62100-CR"}, /* 2 x 40/50/100G */ 1005 {0x6410, "Chelsio T6-DBG-100"}, /* 2 x 40/50/100G, debug */ 1006 {0x6411, "Chelsio T6225-LL-CR"}, /* 2 x 10/25G */ 1007 {0x6414, "Chelsio T62100-SO-OCP3"}, /* 2 x 40/50/100G, nomem */ 1008 {0x6415, "Chelsio T6201-BT"}, /* 2 x 1000BASE-T */ 1009 1010 /* Custom */ 1011 {0x6480, "Custom T6225-CR"}, 1012 {0x6481, "Custom T62100-CR"}, 1013 {0x6482, "Custom T6225-CR"}, 1014 {0x6483, "Custom T62100-CR"}, 1015 {0x6484, "Custom T64100-CR"}, 1016 {0x6485, "Custom T6240-SO"}, 1017 {0x6486, "Custom T6225-SO-CR"}, 1018 {0x6487, "Custom T6225-CR"}, 1019 }, t7_pciids[] = { 1020 {0xd000, "Chelsio Terminator 7 FPGA"}, /* T7 PE12K FPGA */ 1021 {0x7400, "Chelsio T72200-DBG"}, /* 2 x 200G, debug */ 1022 {0x7401, "Chelsio T7250"}, /* 2 x 10/25/50G, 1 mem */ 1023 {0x7402, "Chelsio S7250"}, /* 2 x 10/25/50G, nomem */ 1024 {0x7403, "Chelsio T7450"}, /* 4 x 10/25/50G, 1 mem */ 1025 {0x7404, "Chelsio S7450"}, /* 4 x 10/25/50G, nomem */ 1026 {0x7405, "Chelsio T72200"}, /* 2 x 40/100/200G, 1 mem */ 1027 {0x7406, "Chelsio S72200"}, /* 2 x 40/100/200G, nomem */ 1028 {0x7407, "Chelsio T72200-FH"}, /* 2 x 40/100/200G, 2 mem */ 1029 {0x7408, "Chelsio S71400"}, /* 1 x 400G, nomem */ 1030 {0x7409, "Chelsio S7210-BT"}, /* 2 x 10GBASE-T, nomem */ 1031 {0x740a, "Chelsio T7450-RC"}, /* 4 x 10/25/50G, 1 mem, RC */ 1032 {0x740b, "Chelsio T72200-RC"}, /* 2 x 40/100/200G, 1 mem, RC */ 1033 {0x740c, "Chelsio T72200-FH-RC"}, /* 2 x 40/100/200G, 2 mem, RC */ 1034 {0x740d, "Chelsio S72200-OCP3"}, /* 2 x 40/100/200G OCP3 */ 1035 {0x740e, "Chelsio S7450-OCP3"}, /* 4 x 1/20/25/50G OCP3 */ 1036 {0x740f, "Chelsio S7410-BT-OCP3"}, /* 4 x 10GBASE-T OCP3 */ 1037 {0x7410, "Chelsio S7210-BT-A"}, /* 2 x 10GBASE-T */ 1038 {0x7411, "Chelsio T7_MAYRA_7"}, /* Motherboard */ 1039 1040 /* Custom */ 1041 {0x7480, "Custom T7"}, 1042 }; 1043 1044 #ifdef TCP_OFFLOAD 1045 /* 1046 * service_iq_fl() has an iq and needs the fl. Offset of fl from the iq should 1047 * be exactly the same for both rxq and ofld_rxq. 1048 */ 1049 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 1050 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 1051 #endif 1052 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); 1053 1054 static int 1055 t4_probe(device_t dev) 1056 { 1057 int i; 1058 uint16_t v = pci_get_vendor(dev); 1059 uint16_t d = pci_get_device(dev); 1060 uint8_t f = pci_get_function(dev); 1061 1062 if (v != PCI_VENDOR_ID_CHELSIO) 1063 return (ENXIO); 1064 1065 /* Attach only to PF0 of the FPGA */ 1066 if (d == 0xa000 && f != 0) 1067 return (ENXIO); 1068 1069 for (i = 0; i < nitems(t4_pciids); i++) { 1070 if (d == t4_pciids[i].device) { 1071 device_set_desc(dev, t4_pciids[i].desc); 1072 return (BUS_PROBE_DEFAULT); 1073 } 1074 } 1075 1076 return (ENXIO); 1077 } 1078 1079 static int 1080 t5_probe(device_t dev) 1081 { 1082 int i; 1083 uint16_t v = pci_get_vendor(dev); 1084 uint16_t d = pci_get_device(dev); 1085 uint8_t f = pci_get_function(dev); 1086 1087 if (v != PCI_VENDOR_ID_CHELSIO) 1088 return (ENXIO); 1089 1090 /* Attach only to PF0 of the FPGA */ 1091 if (d == 0xb000 && f != 0) 1092 return (ENXIO); 1093 1094 for (i = 0; i < nitems(t5_pciids); i++) { 1095 if (d == t5_pciids[i].device) { 1096 device_set_desc(dev, t5_pciids[i].desc); 1097 return (BUS_PROBE_DEFAULT); 1098 } 1099 } 1100 1101 return (ENXIO); 1102 } 1103 1104 static int 1105 t6_probe(device_t dev) 1106 { 1107 int i; 1108 uint16_t v = pci_get_vendor(dev); 1109 uint16_t d = pci_get_device(dev); 1110 1111 if (v != PCI_VENDOR_ID_CHELSIO) 1112 return (ENXIO); 1113 1114 for (i = 0; i < nitems(t6_pciids); i++) { 1115 if (d == t6_pciids[i].device) { 1116 device_set_desc(dev, t6_pciids[i].desc); 1117 return (BUS_PROBE_DEFAULT); 1118 } 1119 } 1120 1121 return (ENXIO); 1122 } 1123 1124 static int 1125 ch_probe(device_t dev) 1126 { 1127 int i; 1128 uint16_t v = pci_get_vendor(dev); 1129 uint16_t d = pci_get_device(dev); 1130 uint8_t f = pci_get_function(dev); 1131 1132 if (v != PCI_VENDOR_ID_CHELSIO) 1133 return (ENXIO); 1134 1135 /* Attach only to PF0 of the FPGA */ 1136 if (d == 0xd000 && f != 0) 1137 return (ENXIO); 1138 1139 for (i = 0; i < nitems(t7_pciids); i++) { 1140 if (d == t7_pciids[i].device) { 1141 device_set_desc(dev, t7_pciids[i].desc); 1142 return (BUS_PROBE_DEFAULT); 1143 } 1144 } 1145 1146 return (ENXIO); 1147 } 1148 1149 static void 1150 t5_attribute_workaround(device_t dev) 1151 { 1152 device_t root_port; 1153 uint32_t v; 1154 1155 /* 1156 * The T5 chips do not properly echo the No Snoop and Relaxed 1157 * Ordering attributes when replying to a TLP from a Root 1158 * Port. As a workaround, find the parent Root Port and 1159 * disable No Snoop and Relaxed Ordering. Note that this 1160 * affects all devices under this root port. 1161 */ 1162 root_port = pci_find_pcie_root_port(dev); 1163 if (root_port == NULL) { 1164 device_printf(dev, "Unable to find parent root port\n"); 1165 return; 1166 } 1167 1168 v = pcie_adjust_config(root_port, PCIER_DEVICE_CTL, 1169 PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE, 0, 2); 1170 if ((v & (PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE)) != 1171 0) 1172 device_printf(dev, "Disabled No Snoop/Relaxed Ordering on %s\n", 1173 device_get_nameunit(root_port)); 1174 } 1175 1176 static const struct devnames devnames[] = { 1177 { 1178 .nexus_name = "t4nex", 1179 .ifnet_name = "cxgbe", 1180 .vi_ifnet_name = "vcxgbe", 1181 .pf03_drv_name = "t4iov", 1182 .vf_nexus_name = "t4vf", 1183 .vf_ifnet_name = "cxgbev" 1184 }, { 1185 .nexus_name = "t5nex", 1186 .ifnet_name = "cxl", 1187 .vi_ifnet_name = "vcxl", 1188 .pf03_drv_name = "t5iov", 1189 .vf_nexus_name = "t5vf", 1190 .vf_ifnet_name = "cxlv" 1191 }, { 1192 .nexus_name = "t6nex", 1193 .ifnet_name = "cc", 1194 .vi_ifnet_name = "vcc", 1195 .pf03_drv_name = "t6iov", 1196 .vf_nexus_name = "t6vf", 1197 .vf_ifnet_name = "ccv" 1198 }, { 1199 .nexus_name = "chnex", 1200 .ifnet_name = "che", 1201 .vi_ifnet_name = "vche", 1202 .pf03_drv_name = "chiov", 1203 .vf_nexus_name = "chvf", 1204 .vf_ifnet_name = "chev" 1205 } 1206 }; 1207 1208 void 1209 t4_init_devnames(struct adapter *sc) 1210 { 1211 int id; 1212 1213 id = chip_id(sc); 1214 if (id < CHELSIO_T4) { 1215 device_printf(sc->dev, "chip id %d is not supported.\n", id); 1216 sc->names = NULL; 1217 } else if (id - CHELSIO_T4 < nitems(devnames)) 1218 sc->names = &devnames[id - CHELSIO_T4]; 1219 else 1220 sc->names = &devnames[nitems(devnames) - 1]; 1221 } 1222 1223 static int 1224 t4_ifnet_unit(struct adapter *sc, struct port_info *pi) 1225 { 1226 const char *parent, *name; 1227 long value; 1228 int line, unit; 1229 1230 line = 0; 1231 parent = device_get_nameunit(sc->dev); 1232 name = sc->names->ifnet_name; 1233 while (resource_find_dev(&line, name, &unit, "at", parent) == 0) { 1234 if (resource_long_value(name, unit, "port", &value) == 0 && 1235 value == pi->port_id) 1236 return (unit); 1237 } 1238 return (-1); 1239 } 1240 1241 static void 1242 t4_calibration(void *arg) 1243 { 1244 struct adapter *sc; 1245 struct clock_sync *cur, *nex; 1246 uint64_t hw; 1247 sbintime_t sbt; 1248 int next_up; 1249 1250 sc = (struct adapter *)arg; 1251 1252 KASSERT(!hw_off_limits(sc), ("hw_off_limits at t4_calibration")); 1253 hw = t4_read_reg64(sc, A_SGE_TIMESTAMP_LO); 1254 sbt = sbinuptime(); 1255 1256 cur = &sc->cal_info[sc->cal_current]; 1257 next_up = (sc->cal_current + 1) % CNT_CAL_INFO; 1258 nex = &sc->cal_info[next_up]; 1259 if (__predict_false(sc->cal_count == 0)) { 1260 /* First time in, just get the values in */ 1261 cur->hw_cur = hw; 1262 cur->sbt_cur = sbt; 1263 sc->cal_count++; 1264 goto done; 1265 } 1266 1267 if (cur->hw_cur == hw) { 1268 /* The clock is not advancing? */ 1269 sc->cal_count = 0; 1270 atomic_store_rel_int(&cur->gen, 0); 1271 goto done; 1272 } 1273 1274 seqc_write_begin(&nex->gen); 1275 nex->hw_prev = cur->hw_cur; 1276 nex->sbt_prev = cur->sbt_cur; 1277 nex->hw_cur = hw; 1278 nex->sbt_cur = sbt; 1279 seqc_write_end(&nex->gen); 1280 sc->cal_current = next_up; 1281 done: 1282 callout_reset_sbt_curcpu(&sc->cal_callout, SBT_1S, 0, t4_calibration, 1283 sc, C_DIRECT_EXEC); 1284 } 1285 1286 static void 1287 t4_calibration_start(struct adapter *sc) 1288 { 1289 /* 1290 * Here if we have not done a calibration 1291 * then do so otherwise start the appropriate 1292 * timer. 1293 */ 1294 int i; 1295 1296 for (i = 0; i < CNT_CAL_INFO; i++) { 1297 sc->cal_info[i].gen = 0; 1298 } 1299 sc->cal_current = 0; 1300 sc->cal_count = 0; 1301 sc->cal_gen = 0; 1302 t4_calibration(sc); 1303 } 1304 1305 static int 1306 t4_attach(device_t dev) 1307 { 1308 struct adapter *sc; 1309 int rc = 0, i, j, rqidx, tqidx, nports; 1310 struct make_dev_args mda; 1311 struct intrs_and_queues iaq; 1312 struct sge *s; 1313 uint32_t *buf; 1314 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 1315 int ofld_tqidx; 1316 #endif 1317 #ifdef TCP_OFFLOAD 1318 int ofld_rqidx; 1319 #endif 1320 #ifdef DEV_NETMAP 1321 int nm_rqidx, nm_tqidx; 1322 #endif 1323 int num_vis; 1324 1325 sc = device_get_softc(dev); 1326 sc->dev = dev; 1327 sysctl_ctx_init(&sc->ctx); 1328 TUNABLE_INT_FETCH("hw.cxgbe.dflags", &sc->debug_flags); 1329 if (TUNABLE_INT_FETCH("hw.cxgbe.iflags", &sc->intr_flags) == 0) 1330 sc->intr_flags = IHF_INTR_CLEAR_ON_INIT | IHF_CLR_ALL_UNIGNORED; 1331 1332 if ((pci_get_device(dev) & 0xff00) == 0x5400) 1333 t5_attribute_workaround(dev); 1334 pci_enable_busmaster(dev); 1335 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 1336 uint32_t v; 1337 1338 pci_set_max_read_req(dev, 4096); 1339 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 1340 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5); 1341 if (pcie_relaxed_ordering == 0 && 1342 (v & PCIEM_CTL_RELAXED_ORD_ENABLE) != 0) { 1343 v &= ~PCIEM_CTL_RELAXED_ORD_ENABLE; 1344 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 1345 } else if (pcie_relaxed_ordering == 1 && 1346 (v & PCIEM_CTL_RELAXED_ORD_ENABLE) == 0) { 1347 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 1348 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 1349 } 1350 } 1351 1352 sc->sge_gts_reg = MYPF_REG(A_SGE_PF_GTS); 1353 sc->sge_kdoorbell_reg = MYPF_REG(A_SGE_PF_KDOORBELL); 1354 sc->traceq = -1; 1355 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); 1356 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", 1357 device_get_nameunit(dev)); 1358 1359 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 1360 device_get_nameunit(dev)); 1361 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 1362 t4_add_adapter(sc); 1363 1364 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 1365 TAILQ_INIT(&sc->sfl); 1366 callout_init_mtx(&sc->sfl_callout, &sc->sfl_lock, 0); 1367 1368 mtx_init(&sc->reg_lock, "indirect register access", 0, MTX_DEF); 1369 1370 sc->policy = NULL; 1371 rw_init(&sc->policy_lock, "connection offload policy"); 1372 1373 callout_init(&sc->ktls_tick, 1); 1374 1375 callout_init(&sc->cal_callout, 1); 1376 1377 refcount_init(&sc->vxlan_refcount, 0); 1378 1379 TASK_INIT(&sc->reset_task, 0, reset_adapter_task, sc); 1380 TASK_INIT(&sc->fatal_error_task, 0, fatal_error_task, sc); 1381 1382 sc->ctrlq_oid = SYSCTL_ADD_NODE(&sc->ctx, 1383 SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)), OID_AUTO, "ctrlq", 1384 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "control queues"); 1385 sc->fwq_oid = SYSCTL_ADD_NODE(&sc->ctx, 1386 SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)), OID_AUTO, "fwq", 1387 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "firmware event queue"); 1388 1389 rc = t4_map_bars_0_and_4(sc); 1390 if (rc != 0) 1391 goto done; /* error message displayed already */ 1392 1393 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 1394 memset(sc->port_map, 0xff, sizeof(sc->port_map)); 1395 1396 /* Prepare the adapter for operation. */ 1397 buf = malloc(PAGE_SIZE, M_CXGBE, M_ZERO | M_WAITOK); 1398 rc = -t4_prep_adapter(sc, buf); 1399 free(buf, M_CXGBE); 1400 if (rc != 0) { 1401 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 1402 goto done; 1403 } 1404 1405 /* 1406 * This is the real PF# to which we're attaching. Works from within PCI 1407 * passthrough environments too, where pci_get_function() could return a 1408 * different PF# depending on the passthrough configuration. We need to 1409 * use the real PF# in all our communication with the firmware. 1410 */ 1411 j = t4_read_reg(sc, A_PL_WHOAMI); 1412 sc->pf = chip_id(sc) <= CHELSIO_T5 ? G_SOURCEPF(j) : G_T6_SOURCEPF(j); 1413 sc->mbox = sc->pf; 1414 1415 t4_init_devnames(sc); 1416 if (sc->names == NULL) { 1417 rc = ENOTSUP; 1418 goto done; /* error message displayed already */ 1419 } 1420 1421 /* 1422 * Do this really early, with the memory windows set up even before the 1423 * character device. The userland tool's register i/o and mem read 1424 * will work even in "recovery mode". 1425 */ 1426 setup_memwin(sc); 1427 if (t4_init_devlog_ncores_params(sc, 0) == 0) 1428 fixup_devlog_ncores_params(sc); 1429 make_dev_args_init(&mda); 1430 mda.mda_devsw = &t4_cdevsw; 1431 mda.mda_uid = UID_ROOT; 1432 mda.mda_gid = GID_WHEEL; 1433 mda.mda_mode = 0600; 1434 mda.mda_si_drv1 = sc; 1435 rc = make_dev_s(&mda, &sc->cdev, "%s", device_get_nameunit(dev)); 1436 if (rc != 0) 1437 device_printf(dev, "failed to create nexus char device: %d.\n", 1438 rc); 1439 1440 /* Go no further if recovery mode has been requested. */ 1441 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 1442 device_printf(dev, "recovery mode.\n"); 1443 goto done; 1444 } 1445 1446 #if defined(__i386__) 1447 if ((cpu_feature & CPUID_CX8) == 0) { 1448 device_printf(dev, "64 bit atomics not available.\n"); 1449 rc = ENOTSUP; 1450 goto done; 1451 } 1452 #endif 1453 1454 /* Contact the firmware and try to become the master driver. */ 1455 rc = contact_firmware(sc); 1456 if (rc != 0) 1457 goto done; /* error message displayed already */ 1458 MPASS(sc->flags & FW_OK); 1459 1460 rc = get_params__pre_init(sc); 1461 if (rc != 0) 1462 goto done; /* error message displayed already */ 1463 1464 if (sc->flags & MASTER_PF) { 1465 rc = partition_resources(sc); 1466 if (rc != 0) 1467 goto done; /* error message displayed already */ 1468 } 1469 1470 rc = get_params__post_init(sc); 1471 if (rc != 0) 1472 goto done; /* error message displayed already */ 1473 1474 rc = set_params__post_init(sc); 1475 if (rc != 0) 1476 goto done; /* error message displayed already */ 1477 1478 rc = t4_map_bar_2(sc); 1479 if (rc != 0) 1480 goto done; /* error message displayed already */ 1481 1482 rc = t4_adj_doorbells(sc); 1483 if (rc != 0) 1484 goto done; /* error message displayed already */ 1485 1486 rc = t4_create_dma_tag(sc); 1487 if (rc != 0) 1488 goto done; /* error message displayed already */ 1489 1490 /* 1491 * First pass over all the ports - allocate VIs and initialize some 1492 * basic parameters like mac address, port type, etc. 1493 */ 1494 for_each_port(sc, i) { 1495 struct port_info *pi; 1496 1497 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 1498 sc->port[i] = pi; 1499 1500 /* These must be set before t4_port_init */ 1501 pi->adapter = sc; 1502 pi->port_id = i; 1503 /* 1504 * XXX: vi[0] is special so we can't delay this allocation until 1505 * pi->nvi's final value is known. 1506 */ 1507 pi->vi = malloc(sizeof(struct vi_info) * t4_num_vis, M_CXGBE, 1508 M_ZERO | M_WAITOK); 1509 1510 /* 1511 * Allocate the "main" VI and initialize parameters 1512 * like mac addr. 1513 */ 1514 rc = -t4_port_init(sc, sc->mbox, sc->pf, 0, i); 1515 if (rc != 0) { 1516 device_printf(dev, "unable to initialize port %d: %d\n", 1517 i, rc); 1518 free(pi->vi, M_CXGBE); 1519 free(pi, M_CXGBE); 1520 sc->port[i] = NULL; 1521 goto done; 1522 } 1523 1524 if (is_bt(pi->port_type)) 1525 setbit(&sc->bt_map, pi->hw_port); 1526 else 1527 MPASS(!isset(&sc->bt_map, pi->hw_port)); 1528 1529 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 1530 device_get_nameunit(dev), i); 1531 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 1532 for (j = 0; j < sc->params.tp.lb_nchan; j++) 1533 sc->chan_map[pi->tx_chan + j] = i; 1534 sc->port_map[pi->hw_port] = i; 1535 1536 /* 1537 * The MPS counter for FCS errors doesn't work correctly on the 1538 * T6 so we use the MAC counter here. Which MAC is in use 1539 * depends on the link settings which will be known when the 1540 * link comes up. 1541 */ 1542 if (is_t6(sc)) 1543 pi->fcs_reg = -1; 1544 else 1545 pi->fcs_reg = A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L; 1546 pi->fcs_base = 0; 1547 1548 /* All VIs on this port share this media. */ 1549 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 1550 cxgbe_media_status); 1551 1552 PORT_LOCK(pi); 1553 init_link_config(pi); 1554 fixup_link_config(pi); 1555 build_medialist(pi); 1556 if (fixed_ifmedia(pi)) 1557 pi->flags |= FIXED_IFMEDIA; 1558 PORT_UNLOCK(pi); 1559 1560 pi->dev = device_add_child(dev, sc->names->ifnet_name, 1561 t4_ifnet_unit(sc, pi)); 1562 if (pi->dev == NULL) { 1563 device_printf(dev, 1564 "failed to add device for port %d.\n", i); 1565 rc = ENXIO; 1566 goto done; 1567 } 1568 pi->vi[0].dev = pi->dev; 1569 device_set_softc(pi->dev, pi); 1570 } 1571 1572 /* 1573 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 1574 */ 1575 nports = sc->params.nports; 1576 rc = cfg_itype_and_nqueues(sc, &iaq); 1577 if (rc != 0) 1578 goto done; /* error message displayed already */ 1579 1580 num_vis = iaq.num_vis; 1581 sc->intr_type = iaq.intr_type; 1582 sc->intr_count = iaq.nirq; 1583 1584 s = &sc->sge; 1585 s->nctrlq = max(sc->params.nports, sc->params.ncores); 1586 s->nrxq = nports * iaq.nrxq; 1587 s->ntxq = nports * iaq.ntxq; 1588 if (num_vis > 1) { 1589 s->nrxq += nports * (num_vis - 1) * iaq.nrxq_vi; 1590 s->ntxq += nports * (num_vis - 1) * iaq.ntxq_vi; 1591 } 1592 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 1593 s->neq += nports; /* ctrl queues: 1 per port */ 1594 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 1595 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 1596 if (is_offload(sc) || is_ethoffload(sc)) { 1597 s->nofldtxq = nports * iaq.nofldtxq; 1598 if (num_vis > 1) 1599 s->nofldtxq += nports * (num_vis - 1) * iaq.nofldtxq_vi; 1600 s->neq += s->nofldtxq; 1601 1602 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_ofld_txq), 1603 M_CXGBE, M_ZERO | M_WAITOK); 1604 } 1605 #endif 1606 #ifdef TCP_OFFLOAD 1607 if (is_offload(sc)) { 1608 s->nofldrxq = nports * iaq.nofldrxq; 1609 if (num_vis > 1) 1610 s->nofldrxq += nports * (num_vis - 1) * iaq.nofldrxq_vi; 1611 s->neq += s->nofldrxq; /* free list */ 1612 s->niq += s->nofldrxq; 1613 1614 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 1615 M_CXGBE, M_ZERO | M_WAITOK); 1616 } 1617 #endif 1618 #ifdef DEV_NETMAP 1619 s->nnmrxq = 0; 1620 s->nnmtxq = 0; 1621 if (t4_native_netmap & NN_MAIN_VI) { 1622 s->nnmrxq += nports * iaq.nnmrxq; 1623 s->nnmtxq += nports * iaq.nnmtxq; 1624 } 1625 if (num_vis > 1 && t4_native_netmap & NN_EXTRA_VI) { 1626 s->nnmrxq += nports * (num_vis - 1) * iaq.nnmrxq_vi; 1627 s->nnmtxq += nports * (num_vis - 1) * iaq.nnmtxq_vi; 1628 } 1629 s->neq += s->nnmtxq + s->nnmrxq; 1630 s->niq += s->nnmrxq; 1631 1632 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq), 1633 M_CXGBE, M_ZERO | M_WAITOK); 1634 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq), 1635 M_CXGBE, M_ZERO | M_WAITOK); 1636 #endif 1637 MPASS(s->niq <= s->iqmap_sz); 1638 MPASS(s->neq <= s->eqmap_sz); 1639 1640 s->ctrlq = malloc(s->nctrlq * sizeof(struct sge_wrq), M_CXGBE, 1641 M_ZERO | M_WAITOK); 1642 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 1643 M_ZERO | M_WAITOK); 1644 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 1645 M_ZERO | M_WAITOK); 1646 s->iqmap = malloc(s->iqmap_sz * sizeof(struct sge_iq *), M_CXGBE, 1647 M_ZERO | M_WAITOK); 1648 s->eqmap = malloc(s->eqmap_sz * sizeof(struct sge_eq *), M_CXGBE, 1649 M_ZERO | M_WAITOK); 1650 1651 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 1652 M_ZERO | M_WAITOK); 1653 1654 t4_init_l2t(sc, M_WAITOK); 1655 t4_init_smt(sc, M_WAITOK); 1656 t4_init_tx_sched(sc); 1657 t4_init_atid_table(sc); 1658 #ifdef RATELIMIT 1659 t4_init_etid_table(sc); 1660 #endif 1661 #ifdef INET6 1662 t4_init_clip_table(sc); 1663 #endif 1664 if (sc->vres.key.size != 0) 1665 sc->key_map = vmem_create("T4TLS key map", sc->vres.key.start, 1666 sc->vres.key.size, 32, 0, M_FIRSTFIT | M_WAITOK); 1667 t4_init_tpt(sc); 1668 1669 /* 1670 * Second pass over the ports. This time we know the number of rx and 1671 * tx queues that each port should get. 1672 */ 1673 rqidx = tqidx = 0; 1674 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 1675 ofld_tqidx = 0; 1676 #endif 1677 #ifdef TCP_OFFLOAD 1678 ofld_rqidx = 0; 1679 #endif 1680 #ifdef DEV_NETMAP 1681 nm_rqidx = nm_tqidx = 0; 1682 #endif 1683 for_each_port(sc, i) { 1684 struct port_info *pi = sc->port[i]; 1685 struct vi_info *vi; 1686 1687 if (pi == NULL) 1688 continue; 1689 1690 pi->nvi = num_vis; 1691 for_each_vi(pi, j, vi) { 1692 vi->pi = pi; 1693 vi->adapter = sc; 1694 vi->first_intr = -1; 1695 vi->qsize_rxq = t4_qsize_rxq; 1696 vi->qsize_txq = t4_qsize_txq; 1697 1698 vi->first_rxq = rqidx; 1699 vi->first_txq = tqidx; 1700 vi->tmr_idx = t4_tmr_idx; 1701 vi->pktc_idx = t4_pktc_idx; 1702 vi->nrxq = j == 0 ? iaq.nrxq : iaq.nrxq_vi; 1703 vi->ntxq = j == 0 ? iaq.ntxq : iaq.ntxq_vi; 1704 1705 rqidx += vi->nrxq; 1706 tqidx += vi->ntxq; 1707 1708 if (j == 0 && vi->ntxq > 1) 1709 vi->rsrv_noflowq = t4_rsrv_noflowq ? 1 : 0; 1710 else 1711 vi->rsrv_noflowq = 0; 1712 1713 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 1714 vi->first_ofld_txq = ofld_tqidx; 1715 vi->nofldtxq = j == 0 ? iaq.nofldtxq : iaq.nofldtxq_vi; 1716 ofld_tqidx += vi->nofldtxq; 1717 #endif 1718 #ifdef TCP_OFFLOAD 1719 vi->ofld_tmr_idx = t4_tmr_idx_ofld; 1720 vi->ofld_pktc_idx = t4_pktc_idx_ofld; 1721 vi->first_ofld_rxq = ofld_rqidx; 1722 vi->nofldrxq = j == 0 ? iaq.nofldrxq : iaq.nofldrxq_vi; 1723 1724 ofld_rqidx += vi->nofldrxq; 1725 #endif 1726 #ifdef DEV_NETMAP 1727 vi->first_nm_rxq = nm_rqidx; 1728 vi->first_nm_txq = nm_tqidx; 1729 if (j == 0) { 1730 vi->nnmrxq = iaq.nnmrxq; 1731 vi->nnmtxq = iaq.nnmtxq; 1732 } else { 1733 vi->nnmrxq = iaq.nnmrxq_vi; 1734 vi->nnmtxq = iaq.nnmtxq_vi; 1735 } 1736 nm_rqidx += vi->nnmrxq; 1737 nm_tqidx += vi->nnmtxq; 1738 #endif 1739 } 1740 } 1741 1742 rc = t4_setup_intr_handlers(sc); 1743 if (rc != 0) { 1744 device_printf(dev, 1745 "failed to setup interrupt handlers: %d\n", rc); 1746 goto done; 1747 } 1748 1749 bus_identify_children(dev); 1750 1751 /* 1752 * Ensure thread-safe mailbox access (in debug builds). 1753 * 1754 * So far this was the only thread accessing the mailbox but various 1755 * ifnets and sysctls are about to be created and their handlers/ioctls 1756 * will access the mailbox from different threads. 1757 */ 1758 sc->flags |= CHK_MBOX_ACCESS; 1759 1760 bus_attach_children(dev); 1761 t4_calibration_start(sc); 1762 1763 device_printf(dev, 1764 "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 1765 sc->params.pci.speed, sc->params.pci.width, sc->params.nports, 1766 sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" : 1767 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 1768 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 1769 1770 t4_set_desc(sc); 1771 1772 notify_siblings(dev, 0); 1773 1774 done: 1775 if (rc != 0 && sc->cdev) { 1776 /* cdev was created and so cxgbetool works; recover that way. */ 1777 device_printf(dev, 1778 "error during attach, adapter is now in recovery mode.\n"); 1779 rc = 0; 1780 } 1781 1782 if (rc != 0) 1783 t4_detach_common(dev); 1784 else 1785 t4_sysctls(sc); 1786 1787 return (rc); 1788 } 1789 1790 static int 1791 t4_child_location(device_t bus, device_t dev, struct sbuf *sb) 1792 { 1793 struct adapter *sc; 1794 struct port_info *pi; 1795 int i; 1796 1797 sc = device_get_softc(bus); 1798 for_each_port(sc, i) { 1799 pi = sc->port[i]; 1800 if (pi != NULL && pi->dev == dev) { 1801 sbuf_printf(sb, "port=%d", pi->port_id); 1802 break; 1803 } 1804 } 1805 return (0); 1806 } 1807 1808 static int 1809 t4_ready(device_t dev) 1810 { 1811 struct adapter *sc; 1812 1813 sc = device_get_softc(dev); 1814 if (sc->flags & FW_OK) 1815 return (0); 1816 return (ENXIO); 1817 } 1818 1819 static int 1820 t4_read_port_device(device_t dev, int port, device_t *child) 1821 { 1822 struct adapter *sc; 1823 struct port_info *pi; 1824 1825 sc = device_get_softc(dev); 1826 if (port < 0 || port >= MAX_NPORTS) 1827 return (EINVAL); 1828 pi = sc->port[port]; 1829 if (pi == NULL || pi->dev == NULL) 1830 return (ENXIO); 1831 *child = pi->dev; 1832 return (0); 1833 } 1834 1835 static int 1836 notify_siblings(device_t dev, int detaching) 1837 { 1838 device_t sibling; 1839 int error, i; 1840 1841 error = 0; 1842 for (i = 0; i < PCI_FUNCMAX; i++) { 1843 if (i == pci_get_function(dev)) 1844 continue; 1845 sibling = pci_find_dbsf(pci_get_domain(dev), pci_get_bus(dev), 1846 pci_get_slot(dev), i); 1847 if (sibling == NULL || !device_is_attached(sibling)) 1848 continue; 1849 if (detaching) 1850 error = T4_DETACH_CHILD(sibling); 1851 else 1852 (void)T4_ATTACH_CHILD(sibling); 1853 if (error) 1854 break; 1855 } 1856 return (error); 1857 } 1858 1859 /* 1860 * Idempotent 1861 */ 1862 static int 1863 t4_detach(device_t dev) 1864 { 1865 int rc; 1866 1867 rc = notify_siblings(dev, 1); 1868 if (rc) { 1869 device_printf(dev, 1870 "failed to detach sibling devices: %d\n", rc); 1871 return (rc); 1872 } 1873 1874 return (t4_detach_common(dev)); 1875 } 1876 1877 int 1878 t4_detach_common(device_t dev) 1879 { 1880 struct adapter *sc; 1881 struct port_info *pi; 1882 int i, rc; 1883 1884 sc = device_get_softc(dev); 1885 1886 #ifdef TCP_OFFLOAD 1887 rc = deactivate_all_uld(sc); 1888 if (rc) { 1889 device_printf(dev, 1890 "failed to detach upper layer drivers: %d\n", rc); 1891 return (rc); 1892 } 1893 #endif 1894 1895 if (sc->cdev) { 1896 destroy_dev(sc->cdev); 1897 sc->cdev = NULL; 1898 } 1899 1900 sx_xlock(&t4_list_lock); 1901 SLIST_REMOVE(&t4_list, sc, adapter, link); 1902 sx_xunlock(&t4_list_lock); 1903 1904 sc->flags &= ~CHK_MBOX_ACCESS; 1905 if (sc->flags & FULL_INIT_DONE) { 1906 if (!(sc->flags & IS_VF)) 1907 t4_intr_disable(sc); 1908 } 1909 1910 if (device_is_attached(dev)) { 1911 rc = bus_detach_children(dev); 1912 if (rc) { 1913 device_printf(dev, 1914 "failed to detach child devices: %d\n", rc); 1915 return (rc); 1916 } 1917 } 1918 1919 for (i = 0; i < sc->intr_count; i++) 1920 t4_free_irq(sc, &sc->irq[i]); 1921 1922 if ((sc->flags & (IS_VF | FW_OK)) == FW_OK) 1923 t4_free_tx_sched(sc); 1924 1925 for (i = 0; i < MAX_NPORTS; i++) { 1926 pi = sc->port[i]; 1927 if (pi) { 1928 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->vi[0].viid); 1929 1930 mtx_destroy(&pi->pi_lock); 1931 free(pi->vi, M_CXGBE); 1932 free(pi, M_CXGBE); 1933 } 1934 } 1935 callout_stop(&sc->cal_callout); 1936 callout_drain(&sc->cal_callout); 1937 device_delete_children(dev); 1938 sysctl_ctx_free(&sc->ctx); 1939 adapter_full_uninit(sc); 1940 1941 if ((sc->flags & (IS_VF | FW_OK)) == FW_OK) 1942 t4_fw_bye(sc, sc->mbox); 1943 1944 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 1945 pci_release_msi(dev); 1946 1947 if (sc->regs_res) 1948 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 1949 sc->regs_res); 1950 1951 if (sc->udbs_res) 1952 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 1953 sc->udbs_res); 1954 1955 if (sc->msix_res) 1956 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 1957 sc->msix_res); 1958 1959 if (sc->l2t) 1960 t4_free_l2t(sc); 1961 if (sc->smt) 1962 t4_free_smt(sc->smt); 1963 t4_free_atid_table(sc); 1964 #ifdef RATELIMIT 1965 t4_free_etid_table(sc); 1966 #endif 1967 if (sc->key_map) 1968 vmem_destroy(sc->key_map); 1969 t4_free_tpt(sc); 1970 #ifdef INET6 1971 t4_destroy_clip_table(sc); 1972 #endif 1973 1974 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 1975 free(sc->sge.ofld_txq, M_CXGBE); 1976 #endif 1977 #ifdef TCP_OFFLOAD 1978 free(sc->sge.ofld_rxq, M_CXGBE); 1979 #endif 1980 #ifdef DEV_NETMAP 1981 free(sc->sge.nm_rxq, M_CXGBE); 1982 free(sc->sge.nm_txq, M_CXGBE); 1983 #endif 1984 free(sc->irq, M_CXGBE); 1985 free(sc->sge.rxq, M_CXGBE); 1986 free(sc->sge.txq, M_CXGBE); 1987 free(sc->sge.ctrlq, M_CXGBE); 1988 free(sc->sge.iqmap, M_CXGBE); 1989 free(sc->sge.eqmap, M_CXGBE); 1990 free(sc->tids.ftid_tab, M_CXGBE); 1991 free(sc->tids.hpftid_tab, M_CXGBE); 1992 free_hftid_hash(&sc->tids); 1993 free(sc->tids.tid_tab, M_CXGBE); 1994 t4_destroy_dma_tag(sc); 1995 1996 callout_drain(&sc->ktls_tick); 1997 callout_drain(&sc->sfl_callout); 1998 if (mtx_initialized(&sc->tids.ftid_lock)) { 1999 mtx_destroy(&sc->tids.ftid_lock); 2000 cv_destroy(&sc->tids.ftid_cv); 2001 } 2002 if (mtx_initialized(&sc->tids.atid_lock)) 2003 mtx_destroy(&sc->tids.atid_lock); 2004 if (mtx_initialized(&sc->ifp_lock)) 2005 mtx_destroy(&sc->ifp_lock); 2006 2007 if (rw_initialized(&sc->policy_lock)) { 2008 rw_destroy(&sc->policy_lock); 2009 #ifdef TCP_OFFLOAD 2010 if (sc->policy != NULL) 2011 free_offload_policy(sc->policy); 2012 #endif 2013 } 2014 2015 for (i = 0; i < NUM_MEMWIN; i++) { 2016 struct memwin *mw = &sc->memwin[i]; 2017 2018 if (rw_initialized(&mw->mw_lock)) 2019 rw_destroy(&mw->mw_lock); 2020 } 2021 2022 mtx_destroy(&sc->sfl_lock); 2023 mtx_destroy(&sc->reg_lock); 2024 mtx_destroy(&sc->sc_lock); 2025 2026 bzero(sc, sizeof(*sc)); 2027 2028 return (0); 2029 } 2030 2031 static inline int 2032 stop_adapter(struct adapter *sc) 2033 { 2034 struct port_info *pi; 2035 int i; 2036 2037 if (atomic_testandset_int(&sc->error_flags, ilog2(ADAP_STOPPED))) { 2038 CH_ALERT(sc, "%s from %p, flags 0x%08x,0x%08x, EALREADY\n", 2039 __func__, curthread, sc->flags, sc->error_flags); 2040 return (EALREADY); 2041 } 2042 CH_ALERT(sc, "%s from %p, flags 0x%08x,0x%08x\n", __func__, curthread, 2043 sc->flags, sc->error_flags); 2044 t4_shutdown_adapter(sc); 2045 for_each_port(sc, i) { 2046 pi = sc->port[i]; 2047 if (pi == NULL) 2048 continue; 2049 PORT_LOCK(pi); 2050 if (pi->up_vis > 0 && pi->link_cfg.link_ok) { 2051 /* 2052 * t4_shutdown_adapter has already shut down all the 2053 * PHYs but it also disables interrupts and DMA so there 2054 * won't be a link interrupt. Update the state manually 2055 * if the link was up previously and inform the kernel. 2056 */ 2057 pi->link_cfg.link_ok = false; 2058 t4_os_link_changed(pi); 2059 } 2060 PORT_UNLOCK(pi); 2061 } 2062 2063 return (0); 2064 } 2065 2066 static inline int 2067 restart_adapter(struct adapter *sc) 2068 { 2069 uint32_t val; 2070 2071 if (!atomic_testandclear_int(&sc->error_flags, ilog2(ADAP_STOPPED))) { 2072 CH_ALERT(sc, "%s from %p, flags 0x%08x,0x%08x, EALREADY\n", 2073 __func__, curthread, sc->flags, sc->error_flags); 2074 return (EALREADY); 2075 } 2076 CH_ALERT(sc, "%s from %p, flags 0x%08x,0x%08x\n", __func__, curthread, 2077 sc->flags, sc->error_flags); 2078 2079 MPASS(hw_off_limits(sc)); 2080 MPASS((sc->flags & FW_OK) == 0); 2081 MPASS((sc->flags & MASTER_PF) == 0); 2082 MPASS(sc->reset_thread == NULL); 2083 2084 /* 2085 * The adapter is supposed to be back on PCIE with its config space and 2086 * BARs restored to their state before reset. Register access via 2087 * t4_read_reg BAR0 should just work. 2088 */ 2089 sc->reset_thread = curthread; 2090 val = t4_read_reg(sc, A_PL_WHOAMI); 2091 if (val == 0xffffffff || val == 0xeeeeeeee) { 2092 CH_ERR(sc, "%s: device registers not readable.\n", __func__); 2093 sc->reset_thread = NULL; 2094 atomic_set_int(&sc->error_flags, ADAP_STOPPED); 2095 return (ENXIO); 2096 } 2097 atomic_clear_int(&sc->error_flags, ADAP_FATAL_ERR); 2098 atomic_add_int(&sc->incarnation, 1); 2099 atomic_add_int(&sc->num_resets, 1); 2100 2101 return (0); 2102 } 2103 2104 static inline void 2105 set_adapter_hwstatus(struct adapter *sc, const bool usable) 2106 { 2107 if (usable) { 2108 /* Must be marked reusable by the designated thread. */ 2109 ASSERT_SYNCHRONIZED_OP(sc); 2110 MPASS(sc->reset_thread == curthread); 2111 mtx_lock(&sc->reg_lock); 2112 atomic_clear_int(&sc->error_flags, HW_OFF_LIMITS); 2113 mtx_unlock(&sc->reg_lock); 2114 } else { 2115 /* Mark the adapter totally off limits. */ 2116 begin_synchronized_op(sc, NULL, SLEEP_OK, "t4hwsts"); 2117 mtx_lock(&sc->reg_lock); 2118 atomic_set_int(&sc->error_flags, HW_OFF_LIMITS); 2119 mtx_unlock(&sc->reg_lock); 2120 sc->flags &= ~(FW_OK | MASTER_PF); 2121 sc->reset_thread = NULL; 2122 end_synchronized_op(sc, 0); 2123 } 2124 } 2125 2126 static int 2127 stop_lld(struct adapter *sc) 2128 { 2129 struct port_info *pi; 2130 struct vi_info *vi; 2131 if_t ifp; 2132 struct sge_rxq *rxq; 2133 struct sge_txq *txq; 2134 struct sge_wrq *wrq; 2135 #ifdef TCP_OFFLOAD 2136 struct sge_ofld_rxq *ofld_rxq; 2137 #endif 2138 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 2139 struct sge_ofld_txq *ofld_txq; 2140 #endif 2141 int rc, i, j, k; 2142 2143 /* 2144 * XXX: Can there be a synch_op in progress that will hang because 2145 * hardware has been stopped? We'll hang too and the solution will be 2146 * to use a version of begin_synch_op that wakes up existing synch_op 2147 * with errors. Maybe stop_adapter should do this wakeup? 2148 * 2149 * I don't think any synch_op could get stranded waiting for DMA or 2150 * interrupt so I think we're okay here. Remove this comment block 2151 * after testing. 2152 */ 2153 rc = begin_synchronized_op(sc, NULL, SLEEP_OK, "t4slld"); 2154 if (rc != 0) 2155 return (ENXIO); 2156 2157 /* Quiesce all activity. */ 2158 for_each_port(sc, i) { 2159 pi = sc->port[i]; 2160 if (pi == NULL) 2161 continue; 2162 pi->vxlan_tcam_entry = false; 2163 for_each_vi(pi, j, vi) { 2164 vi->xact_addr_filt = -1; 2165 mtx_lock(&vi->tick_mtx); 2166 vi->flags |= VI_SKIP_STATS; 2167 mtx_unlock(&vi->tick_mtx); 2168 if (!(vi->flags & VI_INIT_DONE)) 2169 continue; 2170 2171 ifp = vi->ifp; 2172 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 2173 mtx_lock(&vi->tick_mtx); 2174 callout_stop(&vi->tick); 2175 mtx_unlock(&vi->tick_mtx); 2176 callout_drain(&vi->tick); 2177 } 2178 2179 /* 2180 * Note that the HW is not available. 2181 */ 2182 for_each_txq(vi, k, txq) { 2183 TXQ_LOCK(txq); 2184 txq->eq.flags &= ~(EQ_ENABLED | EQ_HW_ALLOCATED); 2185 TXQ_UNLOCK(txq); 2186 } 2187 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 2188 for_each_ofld_txq(vi, k, ofld_txq) { 2189 TXQ_LOCK(&ofld_txq->wrq); 2190 ofld_txq->wrq.eq.flags &= ~EQ_HW_ALLOCATED; 2191 TXQ_UNLOCK(&ofld_txq->wrq); 2192 } 2193 #endif 2194 for_each_rxq(vi, k, rxq) { 2195 rxq->iq.flags &= ~IQ_HW_ALLOCATED; 2196 } 2197 #if defined(TCP_OFFLOAD) 2198 for_each_ofld_rxq(vi, k, ofld_rxq) { 2199 ofld_rxq->iq.flags &= ~IQ_HW_ALLOCATED; 2200 } 2201 #endif 2202 2203 quiesce_vi(vi); 2204 } 2205 2206 if (sc->flags & FULL_INIT_DONE) { 2207 /* Control queue */ 2208 wrq = &sc->sge.ctrlq[i]; 2209 TXQ_LOCK(wrq); 2210 wrq->eq.flags &= ~EQ_HW_ALLOCATED; 2211 TXQ_UNLOCK(wrq); 2212 quiesce_wrq(wrq); 2213 } 2214 2215 if (pi->flags & HAS_TRACEQ) { 2216 pi->flags &= ~HAS_TRACEQ; 2217 sc->traceq = -1; 2218 sc->tracer_valid = 0; 2219 sc->tracer_enabled = 0; 2220 } 2221 } 2222 if (sc->flags & FULL_INIT_DONE) { 2223 /* Firmware event queue */ 2224 sc->sge.fwq.flags &= ~IQ_HW_ALLOCATED; 2225 quiesce_iq_fl(sc, &sc->sge.fwq, NULL); 2226 } 2227 2228 /* Stop calibration */ 2229 callout_stop(&sc->cal_callout); 2230 callout_drain(&sc->cal_callout); 2231 2232 if (t4_clock_gate_on_suspend) { 2233 t4_set_reg_field(sc, A_PMU_PART_CG_PWRMODE, F_MA_PART_CGEN | 2234 F_LE_PART_CGEN | F_EDC1_PART_CGEN | F_EDC0_PART_CGEN | 2235 F_TP_PART_CGEN | F_PDP_PART_CGEN | F_SGE_PART_CGEN, 0); 2236 } 2237 2238 end_synchronized_op(sc, 0); 2239 2240 stop_atid_allocator(sc); 2241 t4_stop_l2t(sc); 2242 2243 return (rc); 2244 } 2245 2246 int 2247 suspend_adapter(struct adapter *sc) 2248 { 2249 stop_adapter(sc); 2250 stop_lld(sc); 2251 #ifdef TCP_OFFLOAD 2252 stop_all_uld(sc); 2253 #endif 2254 set_adapter_hwstatus(sc, false); 2255 2256 return (0); 2257 } 2258 2259 static int 2260 t4_suspend(device_t dev) 2261 { 2262 struct adapter *sc = device_get_softc(dev); 2263 int rc; 2264 2265 CH_ALERT(sc, "%s from thread %p.\n", __func__, curthread); 2266 rc = suspend_adapter(sc); 2267 CH_ALERT(sc, "%s end (thread %p).\n", __func__, curthread); 2268 2269 return (rc); 2270 } 2271 2272 struct adapter_pre_reset_state { 2273 u_int flags; 2274 uint16_t nbmcaps; 2275 uint16_t linkcaps; 2276 uint16_t switchcaps; 2277 uint16_t nvmecaps; 2278 uint16_t niccaps; 2279 uint16_t toecaps; 2280 uint16_t rdmacaps; 2281 uint16_t cryptocaps; 2282 uint16_t iscsicaps; 2283 uint16_t fcoecaps; 2284 2285 u_int cfcsum; 2286 char cfg_file[32]; 2287 2288 struct adapter_params params; 2289 struct t4_virt_res vres; 2290 struct tid_info tids; 2291 struct sge sge; 2292 2293 int rawf_base; 2294 int nrawf; 2295 2296 }; 2297 2298 static void 2299 save_caps_and_params(struct adapter *sc, struct adapter_pre_reset_state *o) 2300 { 2301 2302 ASSERT_SYNCHRONIZED_OP(sc); 2303 2304 o->flags = sc->flags; 2305 2306 o->nbmcaps = sc->nbmcaps; 2307 o->linkcaps = sc->linkcaps; 2308 o->switchcaps = sc->switchcaps; 2309 o->nvmecaps = sc->nvmecaps; 2310 o->niccaps = sc->niccaps; 2311 o->toecaps = sc->toecaps; 2312 o->rdmacaps = sc->rdmacaps; 2313 o->cryptocaps = sc->cryptocaps; 2314 o->iscsicaps = sc->iscsicaps; 2315 o->fcoecaps = sc->fcoecaps; 2316 2317 o->cfcsum = sc->cfcsum; 2318 MPASS(sizeof(o->cfg_file) == sizeof(sc->cfg_file)); 2319 memcpy(o->cfg_file, sc->cfg_file, sizeof(o->cfg_file)); 2320 2321 o->params = sc->params; 2322 o->vres = sc->vres; 2323 o->tids = sc->tids; 2324 o->sge = sc->sge; 2325 2326 o->rawf_base = sc->rawf_base; 2327 o->nrawf = sc->nrawf; 2328 } 2329 2330 static int 2331 compare_caps_and_params(struct adapter *sc, struct adapter_pre_reset_state *o) 2332 { 2333 int rc = 0; 2334 2335 ASSERT_SYNCHRONIZED_OP(sc); 2336 2337 /* Capabilities */ 2338 #define COMPARE_CAPS(c) do { \ 2339 if (o->c##caps != sc->c##caps) { \ 2340 CH_ERR(sc, "%scaps 0x%04x -> 0x%04x.\n", #c, o->c##caps, \ 2341 sc->c##caps); \ 2342 rc = EINVAL; \ 2343 } \ 2344 } while (0) 2345 COMPARE_CAPS(nbm); 2346 COMPARE_CAPS(link); 2347 COMPARE_CAPS(switch); 2348 COMPARE_CAPS(nvme); 2349 COMPARE_CAPS(nic); 2350 COMPARE_CAPS(toe); 2351 COMPARE_CAPS(rdma); 2352 COMPARE_CAPS(crypto); 2353 COMPARE_CAPS(iscsi); 2354 COMPARE_CAPS(fcoe); 2355 #undef COMPARE_CAPS 2356 2357 /* Firmware config file */ 2358 if (o->cfcsum != sc->cfcsum) { 2359 CH_ERR(sc, "config file %s (0x%x) -> %s (0x%x)\n", o->cfg_file, 2360 o->cfcsum, sc->cfg_file, sc->cfcsum); 2361 rc = EINVAL; 2362 } 2363 2364 #define COMPARE_PARAM(p, name) do { \ 2365 if (o->p != sc->p) { \ 2366 CH_ERR(sc, #name " %d -> %d\n", o->p, sc->p); \ 2367 rc = EINVAL; \ 2368 } \ 2369 } while (0) 2370 COMPARE_PARAM(sge.iq_start, iq_start); 2371 COMPARE_PARAM(sge.eq_start, eq_start); 2372 COMPARE_PARAM(tids.ftid_base, ftid_base); 2373 COMPARE_PARAM(tids.ftid_end, ftid_end); 2374 COMPARE_PARAM(tids.nftids, nftids); 2375 COMPARE_PARAM(vres.l2t.start, l2t_start); 2376 COMPARE_PARAM(vres.l2t.size, l2t_size); 2377 COMPARE_PARAM(sge.iqmap_sz, iqmap_sz); 2378 COMPARE_PARAM(sge.eqmap_sz, eqmap_sz); 2379 COMPARE_PARAM(tids.tid_base, tid_base); 2380 COMPARE_PARAM(tids.hpftid_base, hpftid_base); 2381 COMPARE_PARAM(tids.hpftid_end, hpftid_end); 2382 COMPARE_PARAM(tids.nhpftids, nhpftids); 2383 COMPARE_PARAM(rawf_base, rawf_base); 2384 COMPARE_PARAM(nrawf, nrawf); 2385 COMPARE_PARAM(params.mps_bg_map, mps_bg_map); 2386 COMPARE_PARAM(params.filter2_wr_support, filter2_wr_support); 2387 COMPARE_PARAM(params.ulptx_memwrite_dsgl, ulptx_memwrite_dsgl); 2388 COMPARE_PARAM(params.fr_nsmr_tpte_wr_support, fr_nsmr_tpte_wr_support); 2389 COMPARE_PARAM(params.max_pkts_per_eth_tx_pkts_wr, max_pkts_per_eth_tx_pkts_wr); 2390 COMPARE_PARAM(tids.ntids, ntids); 2391 COMPARE_PARAM(tids.etid_base, etid_base); 2392 COMPARE_PARAM(tids.etid_end, etid_end); 2393 COMPARE_PARAM(tids.netids, netids); 2394 COMPARE_PARAM(params.eo_wr_cred, eo_wr_cred); 2395 COMPARE_PARAM(params.ethoffload, ethoffload); 2396 COMPARE_PARAM(tids.natids, natids); 2397 COMPARE_PARAM(tids.stid_base, stid_base); 2398 COMPARE_PARAM(vres.ddp.start, ddp_start); 2399 COMPARE_PARAM(vres.ddp.size, ddp_size); 2400 COMPARE_PARAM(params.ofldq_wr_cred, ofldq_wr_cred); 2401 COMPARE_PARAM(vres.stag.start, stag_start); 2402 COMPARE_PARAM(vres.stag.size, stag_size); 2403 COMPARE_PARAM(vres.rq.start, rq_start); 2404 COMPARE_PARAM(vres.rq.size, rq_size); 2405 COMPARE_PARAM(vres.pbl.start, pbl_start); 2406 COMPARE_PARAM(vres.pbl.size, pbl_size); 2407 COMPARE_PARAM(vres.qp.start, qp_start); 2408 COMPARE_PARAM(vres.qp.size, qp_size); 2409 COMPARE_PARAM(vres.cq.start, cq_start); 2410 COMPARE_PARAM(vres.cq.size, cq_size); 2411 COMPARE_PARAM(vres.ocq.start, ocq_start); 2412 COMPARE_PARAM(vres.ocq.size, ocq_size); 2413 COMPARE_PARAM(vres.srq.start, srq_start); 2414 COMPARE_PARAM(vres.srq.size, srq_size); 2415 COMPARE_PARAM(params.max_ordird_qp, max_ordird_qp); 2416 COMPARE_PARAM(params.max_ird_adapter, max_ird_adapter); 2417 COMPARE_PARAM(vres.iscsi.start, iscsi_start); 2418 COMPARE_PARAM(vres.iscsi.size, iscsi_size); 2419 COMPARE_PARAM(vres.key.start, key_start); 2420 COMPARE_PARAM(vres.key.size, key_size); 2421 #undef COMPARE_PARAM 2422 2423 return (rc); 2424 } 2425 2426 static int 2427 restart_lld(struct adapter *sc) 2428 { 2429 struct adapter_pre_reset_state *old_state = NULL; 2430 struct port_info *pi; 2431 struct vi_info *vi; 2432 if_t ifp; 2433 struct sge_txq *txq; 2434 int rc, i, j, k; 2435 2436 rc = begin_synchronized_op(sc, NULL, SLEEP_OK, "t4rlld"); 2437 if (rc != 0) 2438 return (ENXIO); 2439 2440 /* Restore memory window. */ 2441 setup_memwin(sc); 2442 2443 /* Go no further if recovery mode has been requested. */ 2444 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 2445 CH_ALERT(sc, "%s: recovery mode during restart.\n", __func__); 2446 rc = 0; 2447 set_adapter_hwstatus(sc, true); 2448 goto done; 2449 } 2450 2451 old_state = malloc(sizeof(*old_state), M_CXGBE, M_ZERO | M_WAITOK); 2452 save_caps_and_params(sc, old_state); 2453 2454 /* Reestablish contact with firmware and become the primary PF. */ 2455 rc = contact_firmware(sc); 2456 if (rc != 0) 2457 goto done; /* error message displayed already */ 2458 MPASS(sc->flags & FW_OK); 2459 2460 if (sc->flags & MASTER_PF) { 2461 rc = partition_resources(sc); 2462 if (rc != 0) 2463 goto done; /* error message displayed already */ 2464 } 2465 2466 rc = get_params__post_init(sc); 2467 if (rc != 0) 2468 goto done; /* error message displayed already */ 2469 2470 rc = set_params__post_init(sc); 2471 if (rc != 0) 2472 goto done; /* error message displayed already */ 2473 2474 rc = compare_caps_and_params(sc, old_state); 2475 if (rc != 0) 2476 goto done; /* error message displayed already */ 2477 2478 for_each_port(sc, i) { 2479 pi = sc->port[i]; 2480 MPASS(pi != NULL); 2481 MPASS(pi->vi != NULL); 2482 MPASS(pi->vi[0].dev == pi->dev); 2483 2484 rc = -t4_port_init(sc, sc->mbox, sc->pf, 0, i); 2485 if (rc != 0) { 2486 CH_ERR(sc, 2487 "failed to re-initialize port %d: %d\n", i, rc); 2488 goto done; 2489 } 2490 MPASS(sc->chan_map[pi->tx_chan] == i); 2491 2492 PORT_LOCK(pi); 2493 fixup_link_config(pi); 2494 build_medialist(pi); 2495 PORT_UNLOCK(pi); 2496 for_each_vi(pi, j, vi) { 2497 if (IS_MAIN_VI(vi)) 2498 continue; 2499 rc = alloc_extra_vi(sc, pi, vi); 2500 if (rc != 0) { 2501 CH_ERR(vi, 2502 "failed to re-allocate extra VI: %d\n", rc); 2503 goto done; 2504 } 2505 } 2506 } 2507 2508 /* 2509 * Interrupts and queues are about to be enabled and other threads will 2510 * want to access the hardware too. It is safe to do so. Note that 2511 * this thread is still in the middle of a synchronized_op. 2512 */ 2513 set_adapter_hwstatus(sc, true); 2514 2515 if (sc->flags & FULL_INIT_DONE) { 2516 rc = adapter_full_init(sc); 2517 if (rc != 0) { 2518 CH_ERR(sc, "failed to re-initialize adapter: %d\n", rc); 2519 goto done; 2520 } 2521 2522 if (sc->vxlan_refcount > 0) 2523 enable_vxlan_rx(sc); 2524 2525 for_each_port(sc, i) { 2526 pi = sc->port[i]; 2527 for_each_vi(pi, j, vi) { 2528 mtx_lock(&vi->tick_mtx); 2529 vi->flags &= ~VI_SKIP_STATS; 2530 mtx_unlock(&vi->tick_mtx); 2531 if (!(vi->flags & VI_INIT_DONE)) 2532 continue; 2533 rc = vi_full_init(vi); 2534 if (rc != 0) { 2535 CH_ERR(vi, "failed to re-initialize " 2536 "interface: %d\n", rc); 2537 goto done; 2538 } 2539 if (sc->traceq < 0 && IS_MAIN_VI(vi)) { 2540 sc->traceq = sc->sge.rxq[vi->first_rxq].iq.abs_id; 2541 t4_set_trace_rss_control(sc, pi->tx_chan, sc->traceq); 2542 pi->flags |= HAS_TRACEQ; 2543 } 2544 2545 ifp = vi->ifp; 2546 if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) 2547 continue; 2548 /* 2549 * Note that we do not setup multicast addresses 2550 * in the first pass. This ensures that the 2551 * unicast DMACs for all VIs on all ports get an 2552 * MPS TCAM entry. 2553 */ 2554 rc = update_mac_settings(ifp, XGMAC_ALL & 2555 ~XGMAC_MCADDRS); 2556 if (rc != 0) { 2557 CH_ERR(vi, "failed to re-configure MAC: %d\n", rc); 2558 goto done; 2559 } 2560 rc = -t4_enable_vi(sc, sc->mbox, vi->viid, true, 2561 true); 2562 if (rc != 0) { 2563 CH_ERR(vi, "failed to re-enable VI: %d\n", rc); 2564 goto done; 2565 } 2566 for_each_txq(vi, k, txq) { 2567 TXQ_LOCK(txq); 2568 txq->eq.flags |= EQ_ENABLED; 2569 TXQ_UNLOCK(txq); 2570 } 2571 mtx_lock(&vi->tick_mtx); 2572 callout_schedule(&vi->tick, hz); 2573 mtx_unlock(&vi->tick_mtx); 2574 } 2575 PORT_LOCK(pi); 2576 if (pi->up_vis > 0) { 2577 t4_update_port_info(pi); 2578 fixup_link_config(pi); 2579 build_medialist(pi); 2580 apply_link_config(pi); 2581 if (pi->link_cfg.link_ok) 2582 t4_os_link_changed(pi); 2583 } 2584 PORT_UNLOCK(pi); 2585 } 2586 2587 /* Now reprogram the L2 multicast addresses. */ 2588 for_each_port(sc, i) { 2589 pi = sc->port[i]; 2590 for_each_vi(pi, j, vi) { 2591 if (!(vi->flags & VI_INIT_DONE)) 2592 continue; 2593 ifp = vi->ifp; 2594 if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) 2595 continue; 2596 rc = update_mac_settings(ifp, XGMAC_MCADDRS); 2597 if (rc != 0) { 2598 CH_ERR(vi, "failed to re-configure MCAST MACs: %d\n", rc); 2599 rc = 0; /* carry on */ 2600 } 2601 } 2602 } 2603 } 2604 2605 /* Reset all calibration */ 2606 t4_calibration_start(sc); 2607 done: 2608 end_synchronized_op(sc, 0); 2609 free(old_state, M_CXGBE); 2610 2611 restart_atid_allocator(sc); 2612 t4_restart_l2t(sc); 2613 2614 return (rc); 2615 } 2616 2617 int 2618 resume_adapter(struct adapter *sc) 2619 { 2620 restart_adapter(sc); 2621 restart_lld(sc); 2622 #ifdef TCP_OFFLOAD 2623 restart_all_uld(sc); 2624 #endif 2625 return (0); 2626 } 2627 2628 static int 2629 t4_resume(device_t dev) 2630 { 2631 struct adapter *sc = device_get_softc(dev); 2632 int rc; 2633 2634 CH_ALERT(sc, "%s from thread %p.\n", __func__, curthread); 2635 rc = resume_adapter(sc); 2636 CH_ALERT(sc, "%s end (thread %p).\n", __func__, curthread); 2637 2638 return (rc); 2639 } 2640 2641 static int 2642 t4_reset_prepare(device_t dev, device_t child) 2643 { 2644 struct adapter *sc = device_get_softc(dev); 2645 2646 CH_ALERT(sc, "%s from thread %p.\n", __func__, curthread); 2647 return (0); 2648 } 2649 2650 static int 2651 t4_reset_post(device_t dev, device_t child) 2652 { 2653 struct adapter *sc = device_get_softc(dev); 2654 2655 CH_ALERT(sc, "%s from thread %p.\n", __func__, curthread); 2656 return (0); 2657 } 2658 2659 static int 2660 reset_adapter_with_pl_rst(struct adapter *sc) 2661 { 2662 /* This is a t4_write_reg without the hw_off_limits check. */ 2663 MPASS(sc->error_flags & HW_OFF_LIMITS); 2664 bus_write_4(sc->regs_res, A_PL_RST, 2665 F_PIORSTMODE | F_PIORST | F_AUTOPCIEPAUSE); 2666 pause("pl_rst", 1 * hz); /* Wait 1s for reset */ 2667 return (0); 2668 } 2669 2670 static int 2671 reset_adapter_with_pcie_sbr(struct adapter *sc) 2672 { 2673 device_t pdev = device_get_parent(sc->dev); 2674 device_t gpdev = device_get_parent(pdev); 2675 device_t *children; 2676 int rc, i, lcap, lsta, nchildren; 2677 uint32_t v; 2678 2679 rc = pci_find_cap(gpdev, PCIY_EXPRESS, &v); 2680 if (rc != 0) { 2681 CH_ERR(sc, "%s: pci_find_cap(%s, pcie) failed: %d\n", __func__, 2682 device_get_nameunit(gpdev), rc); 2683 return (ENOTSUP); 2684 } 2685 lcap = v + PCIER_LINK_CAP; 2686 lsta = v + PCIER_LINK_STA; 2687 2688 nchildren = 0; 2689 device_get_children(pdev, &children, &nchildren); 2690 for (i = 0; i < nchildren; i++) 2691 pci_save_state(children[i]); 2692 v = pci_read_config(gpdev, PCIR_BRIDGECTL_1, 2); 2693 pci_write_config(gpdev, PCIR_BRIDGECTL_1, v | PCIB_BCR_SECBUS_RESET, 2); 2694 pause("pcie_sbr1", hz / 10); /* 100ms */ 2695 pci_write_config(gpdev, PCIR_BRIDGECTL_1, v, 2); 2696 pause("pcie_sbr2", hz); /* Wait 1s before restore_state. */ 2697 v = pci_read_config(gpdev, lsta, 2); 2698 if (pci_read_config(gpdev, lcap, 2) & PCIEM_LINK_CAP_DL_ACTIVE) 2699 rc = v & PCIEM_LINK_STA_DL_ACTIVE ? 0 : ETIMEDOUT; 2700 else if (v & (PCIEM_LINK_STA_TRAINING_ERROR | PCIEM_LINK_STA_TRAINING)) 2701 rc = ETIMEDOUT; 2702 else 2703 rc = 0; 2704 if (rc != 0) 2705 CH_ERR(sc, "%s: PCIe link is down after reset, LINK_STA 0x%x\n", 2706 __func__, v); 2707 else { 2708 for (i = 0; i < nchildren; i++) 2709 pci_restore_state(children[i]); 2710 } 2711 free(children, M_TEMP); 2712 2713 return (rc); 2714 } 2715 2716 static int 2717 reset_adapter_with_pcie_link_bounce(struct adapter *sc) 2718 { 2719 device_t pdev = device_get_parent(sc->dev); 2720 device_t gpdev = device_get_parent(pdev); 2721 device_t *children; 2722 int rc, i, lcap, lctl, lsta, nchildren; 2723 uint32_t v; 2724 2725 rc = pci_find_cap(gpdev, PCIY_EXPRESS, &v); 2726 if (rc != 0) { 2727 CH_ERR(sc, "%s: pci_find_cap(%s, pcie) failed: %d\n", __func__, 2728 device_get_nameunit(gpdev), rc); 2729 return (ENOTSUP); 2730 } 2731 lcap = v + PCIER_LINK_CAP; 2732 lctl = v + PCIER_LINK_CTL; 2733 lsta = v + PCIER_LINK_STA; 2734 2735 nchildren = 0; 2736 device_get_children(pdev, &children, &nchildren); 2737 for (i = 0; i < nchildren; i++) 2738 pci_save_state(children[i]); 2739 v = pci_read_config(gpdev, lctl, 2); 2740 pci_write_config(gpdev, lctl, v | PCIEM_LINK_CTL_LINK_DIS, 2); 2741 pause("pcie_lnk1", 100 * hz / 1000); /* 100ms */ 2742 pci_write_config(gpdev, lctl, v | PCIEM_LINK_CTL_RETRAIN_LINK, 2); 2743 pause("pcie_lnk2", hz); /* Wait 1s before restore_state. */ 2744 v = pci_read_config(gpdev, lsta, 2); 2745 if (pci_read_config(gpdev, lcap, 2) & PCIEM_LINK_CAP_DL_ACTIVE) 2746 rc = v & PCIEM_LINK_STA_DL_ACTIVE ? 0 : ETIMEDOUT; 2747 else if (v & (PCIEM_LINK_STA_TRAINING_ERROR | PCIEM_LINK_STA_TRAINING)) 2748 rc = ETIMEDOUT; 2749 else 2750 rc = 0; 2751 if (rc != 0) 2752 CH_ERR(sc, "%s: PCIe link is down after reset, LINK_STA 0x%x\n", 2753 __func__, v); 2754 else { 2755 for (i = 0; i < nchildren; i++) 2756 pci_restore_state(children[i]); 2757 } 2758 free(children, M_TEMP); 2759 2760 return (rc); 2761 } 2762 2763 static inline int 2764 reset_adapter(struct adapter *sc) 2765 { 2766 int rc; 2767 const int reset_method = vm_guest == VM_GUEST_NO ? t4_reset_method : 0; 2768 2769 rc = suspend_adapter(sc); 2770 if (rc != 0) 2771 return (rc); 2772 2773 switch (reset_method) { 2774 case 1: 2775 rc = reset_adapter_with_pcie_sbr(sc); 2776 break; 2777 case 2: 2778 rc = reset_adapter_with_pcie_link_bounce(sc); 2779 break; 2780 case 0: 2781 default: 2782 rc = reset_adapter_with_pl_rst(sc); 2783 break; 2784 } 2785 if (rc == 0) 2786 rc = resume_adapter(sc); 2787 return (rc); 2788 } 2789 2790 static void 2791 reset_adapter_task(void *arg, int pending) 2792 { 2793 struct adapter *sc = arg; 2794 const int flags = sc->flags; 2795 const int eflags = sc->error_flags; 2796 int rc; 2797 2798 if (pending > 1) 2799 CH_ALERT(sc, "%s: pending %d\n", __func__, pending); 2800 rc = reset_adapter(sc); 2801 if (rc != 0) { 2802 CH_ERR(sc, "adapter did not reset properly, rc = %d, " 2803 "flags 0x%08x -> 0x%08x, err_flags 0x%08x -> 0x%08x.\n", 2804 rc, flags, sc->flags, eflags, sc->error_flags); 2805 } 2806 } 2807 2808 static int 2809 cxgbe_probe(device_t dev) 2810 { 2811 struct port_info *pi = device_get_softc(dev); 2812 2813 device_set_descf(dev, "port %d", pi->port_id); 2814 2815 return (BUS_PROBE_DEFAULT); 2816 } 2817 2818 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 2819 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 2820 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS | \ 2821 IFCAP_HWRXTSTMP | IFCAP_MEXTPG | IFCAP_NV) 2822 #define T4_CAP_ENABLE (T4_CAP) 2823 2824 static void 2825 cxgbe_vi_attach(device_t dev, struct vi_info *vi) 2826 { 2827 if_t ifp; 2828 struct sbuf *sb; 2829 struct sysctl_ctx_list *ctx = &vi->ctx; 2830 struct sysctl_oid_list *children; 2831 struct pfil_head_args pa; 2832 struct adapter *sc = vi->adapter; 2833 2834 sysctl_ctx_init(ctx); 2835 children = SYSCTL_CHILDREN(device_get_sysctl_tree(vi->dev)); 2836 vi->rxq_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "rxq", 2837 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "NIC rx queues"); 2838 vi->txq_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "txq", 2839 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "NIC tx queues"); 2840 #ifdef DEV_NETMAP 2841 vi->nm_rxq_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "nm_rxq", 2842 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "netmap rx queues"); 2843 vi->nm_txq_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "nm_txq", 2844 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "netmap tx queues"); 2845 #endif 2846 #ifdef TCP_OFFLOAD 2847 vi->ofld_rxq_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "ofld_rxq", 2848 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "TOE rx queues"); 2849 #endif 2850 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 2851 vi->ofld_txq_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "ofld_txq", 2852 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "TOE/ETHOFLD tx queues"); 2853 #endif 2854 2855 vi->xact_addr_filt = -1; 2856 mtx_init(&vi->tick_mtx, "vi tick", NULL, MTX_DEF); 2857 callout_init_mtx(&vi->tick, &vi->tick_mtx, 0); 2858 if (sc->flags & IS_VF || t4_tx_vm_wr != 0) 2859 vi->flags |= TX_USES_VM_WR; 2860 2861 /* Allocate an ifnet and set it up */ 2862 ifp = if_alloc_dev(IFT_ETHER, dev); 2863 vi->ifp = ifp; 2864 if_setsoftc(ifp, vi); 2865 2866 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 2867 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); 2868 2869 if_setinitfn(ifp, cxgbe_init); 2870 if_setioctlfn(ifp, cxgbe_ioctl); 2871 if_settransmitfn(ifp, cxgbe_transmit); 2872 if_setqflushfn(ifp, cxgbe_qflush); 2873 if (vi->pi->nvi > 1 || sc->flags & IS_VF) 2874 if_setgetcounterfn(ifp, vi_get_counter); 2875 else 2876 if_setgetcounterfn(ifp, cxgbe_get_counter); 2877 #if defined(KERN_TLS) || defined(RATELIMIT) 2878 if_setsndtagallocfn(ifp, cxgbe_snd_tag_alloc); 2879 #endif 2880 #ifdef RATELIMIT 2881 if_setratelimitqueryfn(ifp, cxgbe_ratelimit_query); 2882 #endif 2883 2884 if_setcapabilities(ifp, T4_CAP); 2885 if_setcapenable(ifp, T4_CAP_ENABLE); 2886 if_sethwassist(ifp, CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 2887 CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 2888 if (chip_id(sc) >= CHELSIO_T6) { 2889 if_setcapabilitiesbit(ifp, IFCAP_VXLAN_HWCSUM | IFCAP_VXLAN_HWTSO, 0); 2890 if_setcapenablebit(ifp, IFCAP_VXLAN_HWCSUM | IFCAP_VXLAN_HWTSO, 0); 2891 if_sethwassistbits(ifp, CSUM_INNER_IP6_UDP | CSUM_INNER_IP6_TCP | 2892 CSUM_INNER_IP6_TSO | CSUM_INNER_IP | CSUM_INNER_IP_UDP | 2893 CSUM_INNER_IP_TCP | CSUM_INNER_IP_TSO | CSUM_ENCAP_VXLAN, 0); 2894 } 2895 2896 #ifdef TCP_OFFLOAD 2897 if (vi->nofldrxq != 0) 2898 if_setcapabilitiesbit(ifp, IFCAP_TOE, 0); 2899 #endif 2900 #ifdef RATELIMIT 2901 if (is_ethoffload(sc) && vi->nofldtxq != 0) { 2902 if_setcapabilitiesbit(ifp, IFCAP_TXRTLMT, 0); 2903 if_setcapenablebit(ifp, IFCAP_TXRTLMT, 0); 2904 } 2905 #endif 2906 2907 if_sethwtsomax(ifp, IP_MAXPACKET); 2908 if (vi->flags & TX_USES_VM_WR) 2909 if_sethwtsomaxsegcount(ifp, TX_SGL_SEGS_VM_TSO); 2910 else 2911 if_sethwtsomaxsegcount(ifp, TX_SGL_SEGS_TSO); 2912 #ifdef RATELIMIT 2913 if (is_ethoffload(sc) && vi->nofldtxq != 0) 2914 if_sethwtsomaxsegcount(ifp, TX_SGL_SEGS_EO_TSO); 2915 #endif 2916 if_sethwtsomaxsegsize(ifp, 65536); 2917 #ifdef KERN_TLS 2918 if (is_ktls(sc)) { 2919 if_setcapabilitiesbit(ifp, IFCAP_TXTLS, 0); 2920 if (sc->flags & KERN_TLS_ON || !is_t6(sc)) 2921 if_setcapenablebit(ifp, IFCAP_TXTLS, 0); 2922 } 2923 #endif 2924 2925 ether_ifattach(ifp, vi->hw_addr); 2926 #ifdef DEV_NETMAP 2927 if (vi->nnmrxq != 0) 2928 cxgbe_nm_attach(vi); 2929 #endif 2930 sb = sbuf_new_auto(); 2931 sbuf_printf(sb, "%d txq, %d rxq (NIC)", vi->ntxq, vi->nrxq); 2932 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 2933 switch (if_getcapabilities(ifp) & (IFCAP_TOE | IFCAP_TXRTLMT)) { 2934 case IFCAP_TOE: 2935 sbuf_printf(sb, "; %d txq (TOE)", vi->nofldtxq); 2936 break; 2937 case IFCAP_TOE | IFCAP_TXRTLMT: 2938 sbuf_printf(sb, "; %d txq (TOE/ETHOFLD)", vi->nofldtxq); 2939 break; 2940 case IFCAP_TXRTLMT: 2941 sbuf_printf(sb, "; %d txq (ETHOFLD)", vi->nofldtxq); 2942 break; 2943 } 2944 #endif 2945 #ifdef TCP_OFFLOAD 2946 if (if_getcapabilities(ifp) & IFCAP_TOE) 2947 sbuf_printf(sb, ", %d rxq (TOE)", vi->nofldrxq); 2948 #endif 2949 #ifdef DEV_NETMAP 2950 if (if_getcapabilities(ifp) & IFCAP_NETMAP) 2951 sbuf_printf(sb, "; %d txq, %d rxq (netmap)", 2952 vi->nnmtxq, vi->nnmrxq); 2953 #endif 2954 sbuf_finish(sb); 2955 device_printf(dev, "%s\n", sbuf_data(sb)); 2956 sbuf_delete(sb); 2957 2958 vi_sysctls(vi); 2959 2960 pa.pa_version = PFIL_VERSION; 2961 pa.pa_flags = PFIL_IN; 2962 pa.pa_type = PFIL_TYPE_ETHERNET; 2963 pa.pa_headname = if_name(ifp); 2964 vi->pfil = pfil_head_register(&pa); 2965 } 2966 2967 static int 2968 cxgbe_attach(device_t dev) 2969 { 2970 struct port_info *pi = device_get_softc(dev); 2971 struct adapter *sc = pi->adapter; 2972 struct vi_info *vi; 2973 int i; 2974 2975 sysctl_ctx_init(&pi->ctx); 2976 2977 cxgbe_vi_attach(dev, &pi->vi[0]); 2978 2979 for_each_vi(pi, i, vi) { 2980 if (i == 0) 2981 continue; 2982 vi->dev = device_add_child(dev, sc->names->vi_ifnet_name, DEVICE_UNIT_ANY); 2983 if (vi->dev == NULL) { 2984 device_printf(dev, "failed to add VI %d\n", i); 2985 continue; 2986 } 2987 device_set_softc(vi->dev, vi); 2988 } 2989 2990 cxgbe_sysctls(pi); 2991 2992 bus_attach_children(dev); 2993 2994 return (0); 2995 } 2996 2997 static void 2998 cxgbe_vi_detach(struct vi_info *vi) 2999 { 3000 if_t ifp = vi->ifp; 3001 3002 if (vi->pfil != NULL) { 3003 pfil_head_unregister(vi->pfil); 3004 vi->pfil = NULL; 3005 } 3006 3007 ether_ifdetach(ifp); 3008 3009 /* Let detach proceed even if these fail. */ 3010 #ifdef DEV_NETMAP 3011 if (if_getcapabilities(ifp) & IFCAP_NETMAP) 3012 cxgbe_nm_detach(vi); 3013 #endif 3014 cxgbe_uninit_synchronized(vi); 3015 callout_drain(&vi->tick); 3016 mtx_destroy(&vi->tick_mtx); 3017 sysctl_ctx_free(&vi->ctx); 3018 vi_full_uninit(vi); 3019 3020 if_free(vi->ifp); 3021 vi->ifp = NULL; 3022 } 3023 3024 static int 3025 cxgbe_detach(device_t dev) 3026 { 3027 struct port_info *pi = device_get_softc(dev); 3028 struct adapter *sc = pi->adapter; 3029 int rc; 3030 3031 /* Detach the extra VIs first. */ 3032 rc = bus_generic_detach(dev); 3033 if (rc) 3034 return (rc); 3035 3036 sysctl_ctx_free(&pi->ctx); 3037 begin_vi_detach(sc, &pi->vi[0]); 3038 if (pi->flags & HAS_TRACEQ) { 3039 sc->traceq = -1; /* cloner should not create ifnet */ 3040 t4_tracer_port_detach(sc); 3041 } 3042 cxgbe_vi_detach(&pi->vi[0]); 3043 ifmedia_removeall(&pi->media); 3044 end_vi_detach(sc, &pi->vi[0]); 3045 3046 return (0); 3047 } 3048 3049 static void 3050 cxgbe_init(void *arg) 3051 { 3052 struct vi_info *vi = arg; 3053 struct adapter *sc = vi->adapter; 3054 3055 if (begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4init") != 0) 3056 return; 3057 cxgbe_init_synchronized(vi); 3058 end_synchronized_op(sc, 0); 3059 } 3060 3061 static int 3062 cxgbe_ioctl(if_t ifp, unsigned long cmd, caddr_t data) 3063 { 3064 int rc = 0, mtu, flags; 3065 struct vi_info *vi = if_getsoftc(ifp); 3066 struct port_info *pi = vi->pi; 3067 struct adapter *sc = pi->adapter; 3068 struct ifreq *ifr = (struct ifreq *)data; 3069 uint32_t mask, mask2; 3070 3071 switch (cmd) { 3072 case SIOCSIFMTU: 3073 mtu = ifr->ifr_mtu; 3074 if (mtu < ETHERMIN || mtu > MAX_MTU) 3075 return (EINVAL); 3076 3077 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4mtu"); 3078 if (rc) 3079 return (rc); 3080 if_setmtu(ifp, mtu); 3081 if (vi->flags & VI_INIT_DONE) { 3082 t4_update_fl_bufsize(ifp); 3083 if (hw_all_ok(sc) && 3084 if_getdrvflags(ifp) & IFF_DRV_RUNNING) 3085 rc = update_mac_settings(ifp, XGMAC_MTU); 3086 } 3087 end_synchronized_op(sc, 0); 3088 break; 3089 3090 case SIOCSIFFLAGS: 3091 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4flg"); 3092 if (rc) 3093 return (rc); 3094 3095 if (!hw_all_ok(sc)) { 3096 rc = ENXIO; 3097 goto fail; 3098 } 3099 3100 if (if_getflags(ifp) & IFF_UP) { 3101 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 3102 flags = vi->if_flags; 3103 if ((if_getflags(ifp) ^ flags) & 3104 (IFF_PROMISC | IFF_ALLMULTI)) { 3105 rc = update_mac_settings(ifp, 3106 XGMAC_PROMISC | XGMAC_ALLMULTI); 3107 } 3108 } else { 3109 rc = cxgbe_init_synchronized(vi); 3110 } 3111 vi->if_flags = if_getflags(ifp); 3112 } else if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 3113 rc = cxgbe_uninit_synchronized(vi); 3114 } 3115 end_synchronized_op(sc, 0); 3116 break; 3117 3118 case SIOCADDMULTI: 3119 case SIOCDELMULTI: 3120 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4multi"); 3121 if (rc) 3122 return (rc); 3123 if (hw_all_ok(sc) && if_getdrvflags(ifp) & IFF_DRV_RUNNING) 3124 rc = update_mac_settings(ifp, XGMAC_MCADDRS); 3125 end_synchronized_op(sc, 0); 3126 break; 3127 3128 case SIOCGIFCAPNV: 3129 break; 3130 case SIOCSIFCAPNV: 3131 case SIOCSIFCAP: 3132 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4cap"); 3133 if (rc) 3134 return (rc); 3135 3136 if (cmd == SIOCSIFCAPNV) { 3137 const struct siocsifcapnv_driver_data *ifr_nv = 3138 (struct siocsifcapnv_driver_data *)data; 3139 3140 mask = ifr_nv->reqcap ^ if_getcapenable(ifp); 3141 mask2 = ifr_nv->reqcap2 ^ if_getcapenable2(ifp); 3142 } else { 3143 mask = ifr->ifr_reqcap ^ if_getcapenable(ifp); 3144 mask2 = 0; 3145 } 3146 if (mask & IFCAP_TXCSUM) { 3147 if_togglecapenable(ifp, IFCAP_TXCSUM); 3148 if_togglehwassist(ifp, CSUM_TCP | CSUM_UDP | CSUM_IP); 3149 3150 if (IFCAP_TSO4 & if_getcapenable(ifp) && 3151 !(IFCAP_TXCSUM & if_getcapenable(ifp))) { 3152 mask &= ~IFCAP_TSO4; 3153 if_setcapenablebit(ifp, 0, IFCAP_TSO4); 3154 if_printf(ifp, 3155 "tso4 disabled due to -txcsum.\n"); 3156 } 3157 } 3158 if (mask & IFCAP_TXCSUM_IPV6) { 3159 if_togglecapenable(ifp, IFCAP_TXCSUM_IPV6); 3160 if_togglehwassist(ifp, CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 3161 3162 if (IFCAP_TSO6 & if_getcapenable(ifp) && 3163 !(IFCAP_TXCSUM_IPV6 & if_getcapenable(ifp))) { 3164 mask &= ~IFCAP_TSO6; 3165 if_setcapenablebit(ifp, 0, IFCAP_TSO6); 3166 if_printf(ifp, 3167 "tso6 disabled due to -txcsum6.\n"); 3168 } 3169 } 3170 if (mask & IFCAP_RXCSUM) 3171 if_togglecapenable(ifp, IFCAP_RXCSUM); 3172 if (mask & IFCAP_RXCSUM_IPV6) 3173 if_togglecapenable(ifp, IFCAP_RXCSUM_IPV6); 3174 3175 /* 3176 * Note that we leave CSUM_TSO alone (it is always set). The 3177 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 3178 * sending a TSO request our way, so it's sufficient to toggle 3179 * IFCAP_TSOx only. 3180 */ 3181 if (mask & IFCAP_TSO4) { 3182 if (!(IFCAP_TSO4 & if_getcapenable(ifp)) && 3183 !(IFCAP_TXCSUM & if_getcapenable(ifp))) { 3184 if_printf(ifp, "enable txcsum first.\n"); 3185 rc = EAGAIN; 3186 goto fail; 3187 } 3188 if_togglecapenable(ifp, IFCAP_TSO4); 3189 } 3190 if (mask & IFCAP_TSO6) { 3191 if (!(IFCAP_TSO6 & if_getcapenable(ifp)) && 3192 !(IFCAP_TXCSUM_IPV6 & if_getcapenable(ifp))) { 3193 if_printf(ifp, "enable txcsum6 first.\n"); 3194 rc = EAGAIN; 3195 goto fail; 3196 } 3197 if_togglecapenable(ifp, IFCAP_TSO6); 3198 } 3199 if (mask & IFCAP_LRO) { 3200 #if defined(INET) || defined(INET6) 3201 int i; 3202 struct sge_rxq *rxq; 3203 3204 if_togglecapenable(ifp, IFCAP_LRO); 3205 for_each_rxq(vi, i, rxq) { 3206 if (if_getcapenable(ifp) & IFCAP_LRO) 3207 rxq->iq.flags |= IQ_LRO_ENABLED; 3208 else 3209 rxq->iq.flags &= ~IQ_LRO_ENABLED; 3210 } 3211 #endif 3212 } 3213 #ifdef TCP_OFFLOAD 3214 if (mask & IFCAP_TOE) { 3215 int enable = (if_getcapenable(ifp) ^ mask) & IFCAP_TOE; 3216 3217 rc = toe_capability(vi, enable); 3218 if (rc != 0) 3219 goto fail; 3220 3221 if_togglecapenable(ifp, mask); 3222 } 3223 #endif 3224 if (mask & IFCAP_VLAN_HWTAGGING) { 3225 if_togglecapenable(ifp, IFCAP_VLAN_HWTAGGING); 3226 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) 3227 rc = update_mac_settings(ifp, XGMAC_VLANEX); 3228 } 3229 if (mask & IFCAP_VLAN_MTU) { 3230 if_togglecapenable(ifp, IFCAP_VLAN_MTU); 3231 3232 /* Need to find out how to disable auto-mtu-inflation */ 3233 } 3234 if (mask & IFCAP_VLAN_HWTSO) 3235 if_togglecapenable(ifp, IFCAP_VLAN_HWTSO); 3236 if (mask & IFCAP_VLAN_HWCSUM) 3237 if_togglecapenable(ifp, IFCAP_VLAN_HWCSUM); 3238 #ifdef RATELIMIT 3239 if (mask & IFCAP_TXRTLMT) 3240 if_togglecapenable(ifp, IFCAP_TXRTLMT); 3241 #endif 3242 if (mask & IFCAP_HWRXTSTMP) { 3243 int i; 3244 struct sge_rxq *rxq; 3245 3246 if_togglecapenable(ifp, IFCAP_HWRXTSTMP); 3247 for_each_rxq(vi, i, rxq) { 3248 if (if_getcapenable(ifp) & IFCAP_HWRXTSTMP) 3249 rxq->iq.flags |= IQ_RX_TIMESTAMP; 3250 else 3251 rxq->iq.flags &= ~IQ_RX_TIMESTAMP; 3252 } 3253 } 3254 if (mask & IFCAP_MEXTPG) 3255 if_togglecapenable(ifp, IFCAP_MEXTPG); 3256 3257 #ifdef KERN_TLS 3258 if (mask & IFCAP_TXTLS) { 3259 int enable = (if_getcapenable(ifp) ^ mask) & IFCAP_TXTLS; 3260 3261 rc = ktls_capability(sc, enable); 3262 if (rc != 0) 3263 goto fail; 3264 3265 if_togglecapenable(ifp, mask & IFCAP_TXTLS); 3266 } 3267 #endif 3268 if (mask & IFCAP_VXLAN_HWCSUM) { 3269 if_togglecapenable(ifp, IFCAP_VXLAN_HWCSUM); 3270 if_togglehwassist(ifp, CSUM_INNER_IP6_UDP | 3271 CSUM_INNER_IP6_TCP | CSUM_INNER_IP | 3272 CSUM_INNER_IP_UDP | CSUM_INNER_IP_TCP); 3273 } 3274 if (mask & IFCAP_VXLAN_HWTSO) { 3275 if_togglecapenable(ifp, IFCAP_VXLAN_HWTSO); 3276 if_togglehwassist(ifp, CSUM_INNER_IP6_TSO | 3277 CSUM_INNER_IP_TSO); 3278 } 3279 3280 MPASS(mask2 == 0); 3281 (void)mask2; 3282 3283 #ifdef VLAN_CAPABILITIES 3284 VLAN_CAPABILITIES(ifp); 3285 #endif 3286 fail: 3287 end_synchronized_op(sc, 0); 3288 break; 3289 3290 case SIOCSIFMEDIA: 3291 case SIOCGIFMEDIA: 3292 case SIOCGIFXMEDIA: 3293 rc = ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 3294 break; 3295 3296 case SIOCGI2C: { 3297 struct ifi2creq i2c; 3298 3299 rc = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c)); 3300 if (rc != 0) 3301 break; 3302 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) { 3303 rc = EPERM; 3304 break; 3305 } 3306 if (i2c.len > sizeof(i2c.data)) { 3307 rc = EINVAL; 3308 break; 3309 } 3310 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4i2c"); 3311 if (rc) 3312 return (rc); 3313 if (!hw_all_ok(sc)) 3314 rc = ENXIO; 3315 else 3316 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr, 3317 i2c.offset, i2c.len, &i2c.data[0]); 3318 end_synchronized_op(sc, 0); 3319 if (rc == 0) 3320 rc = copyout(&i2c, ifr_data_get_ptr(ifr), sizeof(i2c)); 3321 break; 3322 } 3323 3324 default: 3325 rc = ether_ioctl(ifp, cmd, data); 3326 } 3327 3328 return (rc); 3329 } 3330 3331 static int 3332 cxgbe_transmit(if_t ifp, struct mbuf *m) 3333 { 3334 struct vi_info *vi = if_getsoftc(ifp); 3335 struct port_info *pi = vi->pi; 3336 struct adapter *sc; 3337 struct sge_txq *txq; 3338 void *items[1]; 3339 int rc; 3340 3341 M_ASSERTPKTHDR(m); 3342 MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */ 3343 #if defined(KERN_TLS) || defined(RATELIMIT) 3344 if (m->m_pkthdr.csum_flags & CSUM_SND_TAG) 3345 MPASS(m->m_pkthdr.snd_tag->ifp == ifp); 3346 #endif 3347 3348 if (__predict_false(pi->link_cfg.link_ok == false)) { 3349 m_freem(m); 3350 return (ENETDOWN); 3351 } 3352 3353 rc = parse_pkt(&m, vi->flags & TX_USES_VM_WR); 3354 if (__predict_false(rc != 0)) { 3355 if (__predict_true(rc == EINPROGRESS)) { 3356 /* queued by parse_pkt */ 3357 MPASS(m != NULL); 3358 return (0); 3359 } 3360 3361 MPASS(m == NULL); /* was freed already */ 3362 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */ 3363 return (rc); 3364 } 3365 3366 /* Select a txq. */ 3367 sc = vi->adapter; 3368 txq = &sc->sge.txq[vi->first_txq]; 3369 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 3370 txq += ((m->m_pkthdr.flowid % (vi->ntxq - vi->rsrv_noflowq)) + 3371 vi->rsrv_noflowq); 3372 3373 items[0] = m; 3374 rc = mp_ring_enqueue(txq->r, items, 1, 256); 3375 if (__predict_false(rc != 0)) 3376 m_freem(m); 3377 3378 return (rc); 3379 } 3380 3381 static void 3382 cxgbe_qflush(if_t ifp) 3383 { 3384 struct vi_info *vi = if_getsoftc(ifp); 3385 struct sge_txq *txq; 3386 int i; 3387 3388 /* queues do not exist if !VI_INIT_DONE. */ 3389 if (vi->flags & VI_INIT_DONE) { 3390 for_each_txq(vi, i, txq) { 3391 TXQ_LOCK(txq); 3392 txq->eq.flags |= EQ_QFLUSH; 3393 TXQ_UNLOCK(txq); 3394 while (!mp_ring_is_idle(txq->r)) { 3395 mp_ring_check_drainage(txq->r, 4096); 3396 pause("qflush", 1); 3397 } 3398 TXQ_LOCK(txq); 3399 txq->eq.flags &= ~EQ_QFLUSH; 3400 TXQ_UNLOCK(txq); 3401 } 3402 } 3403 if_qflush(ifp); 3404 } 3405 3406 static uint64_t 3407 vi_get_counter(if_t ifp, ift_counter c) 3408 { 3409 struct vi_info *vi = if_getsoftc(ifp); 3410 struct fw_vi_stats_vf *s = &vi->stats; 3411 3412 mtx_lock(&vi->tick_mtx); 3413 vi_refresh_stats(vi); 3414 mtx_unlock(&vi->tick_mtx); 3415 3416 switch (c) { 3417 case IFCOUNTER_IPACKETS: 3418 return (s->rx_bcast_frames + s->rx_mcast_frames + 3419 s->rx_ucast_frames); 3420 case IFCOUNTER_IERRORS: 3421 return (s->rx_err_frames); 3422 case IFCOUNTER_OPACKETS: 3423 return (s->tx_bcast_frames + s->tx_mcast_frames + 3424 s->tx_ucast_frames + s->tx_offload_frames); 3425 case IFCOUNTER_OERRORS: 3426 return (s->tx_drop_frames); 3427 case IFCOUNTER_IBYTES: 3428 return (s->rx_bcast_bytes + s->rx_mcast_bytes + 3429 s->rx_ucast_bytes); 3430 case IFCOUNTER_OBYTES: 3431 return (s->tx_bcast_bytes + s->tx_mcast_bytes + 3432 s->tx_ucast_bytes + s->tx_offload_bytes); 3433 case IFCOUNTER_IMCASTS: 3434 return (s->rx_mcast_frames); 3435 case IFCOUNTER_OMCASTS: 3436 return (s->tx_mcast_frames); 3437 case IFCOUNTER_OQDROPS: { 3438 uint64_t drops; 3439 3440 drops = 0; 3441 if (vi->flags & VI_INIT_DONE) { 3442 int i; 3443 struct sge_txq *txq; 3444 3445 for_each_txq(vi, i, txq) 3446 drops += counter_u64_fetch(txq->r->dropped); 3447 } 3448 3449 return (drops); 3450 3451 } 3452 3453 default: 3454 return (if_get_counter_default(ifp, c)); 3455 } 3456 } 3457 3458 static uint64_t 3459 cxgbe_get_counter(if_t ifp, ift_counter c) 3460 { 3461 struct vi_info *vi = if_getsoftc(ifp); 3462 struct port_info *pi = vi->pi; 3463 struct port_stats *s = &pi->stats; 3464 3465 mtx_lock(&vi->tick_mtx); 3466 cxgbe_refresh_stats(vi); 3467 mtx_unlock(&vi->tick_mtx); 3468 3469 switch (c) { 3470 case IFCOUNTER_IPACKETS: 3471 return (s->rx_frames); 3472 3473 case IFCOUNTER_IERRORS: 3474 return (s->rx_jabber + s->rx_runt + s->rx_too_long + 3475 s->rx_fcs_err + s->rx_len_err); 3476 3477 case IFCOUNTER_OPACKETS: 3478 return (s->tx_frames); 3479 3480 case IFCOUNTER_OERRORS: 3481 return (s->tx_error_frames); 3482 3483 case IFCOUNTER_IBYTES: 3484 return (s->rx_octets); 3485 3486 case IFCOUNTER_OBYTES: 3487 return (s->tx_octets); 3488 3489 case IFCOUNTER_IMCASTS: 3490 return (s->rx_mcast_frames); 3491 3492 case IFCOUNTER_OMCASTS: 3493 return (s->tx_mcast_frames); 3494 3495 case IFCOUNTER_IQDROPS: 3496 return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 3497 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 3498 s->rx_trunc3 + pi->tnl_cong_drops); 3499 3500 case IFCOUNTER_OQDROPS: { 3501 uint64_t drops; 3502 3503 drops = s->tx_drop; 3504 if (vi->flags & VI_INIT_DONE) { 3505 int i; 3506 struct sge_txq *txq; 3507 3508 for_each_txq(vi, i, txq) 3509 drops += counter_u64_fetch(txq->r->dropped); 3510 } 3511 3512 return (drops); 3513 3514 } 3515 3516 default: 3517 return (if_get_counter_default(ifp, c)); 3518 } 3519 } 3520 3521 #if defined(KERN_TLS) || defined(RATELIMIT) 3522 static int 3523 cxgbe_snd_tag_alloc(if_t ifp, union if_snd_tag_alloc_params *params, 3524 struct m_snd_tag **pt) 3525 { 3526 int error; 3527 3528 switch (params->hdr.type) { 3529 #ifdef RATELIMIT 3530 case IF_SND_TAG_TYPE_RATE_LIMIT: 3531 error = cxgbe_rate_tag_alloc(ifp, params, pt); 3532 break; 3533 #endif 3534 #ifdef KERN_TLS 3535 case IF_SND_TAG_TYPE_TLS: 3536 { 3537 struct vi_info *vi = if_getsoftc(ifp); 3538 3539 if (is_t6(vi->pi->adapter)) 3540 error = t6_tls_tag_alloc(ifp, params, pt); 3541 else 3542 error = t7_tls_tag_alloc(ifp, params, pt); 3543 break; 3544 } 3545 #endif 3546 default: 3547 error = EOPNOTSUPP; 3548 } 3549 return (error); 3550 } 3551 #endif 3552 3553 /* 3554 * The kernel picks a media from the list we had provided but we still validate 3555 * the requeste. 3556 */ 3557 int 3558 cxgbe_media_change(if_t ifp) 3559 { 3560 struct vi_info *vi = if_getsoftc(ifp); 3561 struct port_info *pi = vi->pi; 3562 struct ifmedia *ifm = &pi->media; 3563 struct link_config *lc = &pi->link_cfg; 3564 struct adapter *sc = pi->adapter; 3565 int rc; 3566 3567 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4mec"); 3568 if (rc != 0) 3569 return (rc); 3570 PORT_LOCK(pi); 3571 if (IFM_SUBTYPE(ifm->ifm_media) == IFM_AUTO) { 3572 /* ifconfig .. media autoselect */ 3573 if (!(lc->pcaps & FW_PORT_CAP32_ANEG)) { 3574 rc = ENOTSUP; /* AN not supported by transceiver */ 3575 goto done; 3576 } 3577 lc->requested_aneg = AUTONEG_ENABLE; 3578 lc->requested_speed = 0; 3579 lc->requested_fc |= PAUSE_AUTONEG; 3580 } else { 3581 lc->requested_aneg = AUTONEG_DISABLE; 3582 lc->requested_speed = 3583 ifmedia_baudrate(ifm->ifm_media) / 1000000; 3584 lc->requested_fc = 0; 3585 if (IFM_OPTIONS(ifm->ifm_media) & IFM_ETH_RXPAUSE) 3586 lc->requested_fc |= PAUSE_RX; 3587 if (IFM_OPTIONS(ifm->ifm_media) & IFM_ETH_TXPAUSE) 3588 lc->requested_fc |= PAUSE_TX; 3589 } 3590 if (pi->up_vis > 0 && hw_all_ok(sc)) { 3591 fixup_link_config(pi); 3592 rc = apply_link_config(pi); 3593 } 3594 done: 3595 PORT_UNLOCK(pi); 3596 end_synchronized_op(sc, 0); 3597 return (rc); 3598 } 3599 3600 /* 3601 * Base media word (without ETHER, pause, link active, etc.) for the port at the 3602 * given speed. 3603 */ 3604 static int 3605 port_mword(struct port_info *pi, uint32_t speed) 3606 { 3607 3608 MPASS(speed & M_FW_PORT_CAP32_SPEED); 3609 MPASS(powerof2(speed)); 3610 3611 switch(pi->port_type) { 3612 case FW_PORT_TYPE_BT_SGMII: 3613 case FW_PORT_TYPE_BT_XFI: 3614 case FW_PORT_TYPE_BT_XAUI: 3615 /* BaseT */ 3616 switch (speed) { 3617 case FW_PORT_CAP32_SPEED_100M: 3618 return (IFM_100_T); 3619 case FW_PORT_CAP32_SPEED_1G: 3620 return (IFM_1000_T); 3621 case FW_PORT_CAP32_SPEED_10G: 3622 return (IFM_10G_T); 3623 } 3624 break; 3625 case FW_PORT_TYPE_KX4: 3626 if (speed == FW_PORT_CAP32_SPEED_10G) 3627 return (IFM_10G_KX4); 3628 break; 3629 case FW_PORT_TYPE_CX4: 3630 if (speed == FW_PORT_CAP32_SPEED_10G) 3631 return (IFM_10G_CX4); 3632 break; 3633 case FW_PORT_TYPE_KX: 3634 if (speed == FW_PORT_CAP32_SPEED_1G) 3635 return (IFM_1000_KX); 3636 break; 3637 case FW_PORT_TYPE_KR: 3638 case FW_PORT_TYPE_BP_AP: 3639 case FW_PORT_TYPE_BP4_AP: 3640 case FW_PORT_TYPE_BP40_BA: 3641 case FW_PORT_TYPE_KR4_100G: 3642 case FW_PORT_TYPE_KR_SFP28: 3643 case FW_PORT_TYPE_KR_XLAUI: 3644 switch (speed) { 3645 case FW_PORT_CAP32_SPEED_1G: 3646 return (IFM_1000_KX); 3647 case FW_PORT_CAP32_SPEED_10G: 3648 return (IFM_10G_KR); 3649 case FW_PORT_CAP32_SPEED_25G: 3650 return (IFM_25G_KR); 3651 case FW_PORT_CAP32_SPEED_40G: 3652 return (IFM_40G_KR4); 3653 case FW_PORT_CAP32_SPEED_50G: 3654 return (IFM_50G_KR2); 3655 case FW_PORT_CAP32_SPEED_100G: 3656 return (IFM_100G_KR4); 3657 } 3658 break; 3659 case FW_PORT_TYPE_FIBER_XFI: 3660 case FW_PORT_TYPE_FIBER_XAUI: 3661 case FW_PORT_TYPE_SFP: 3662 case FW_PORT_TYPE_QSFP_10G: 3663 case FW_PORT_TYPE_QSA: 3664 case FW_PORT_TYPE_QSFP: 3665 case FW_PORT_TYPE_CR4_QSFP: 3666 case FW_PORT_TYPE_CR_QSFP: 3667 case FW_PORT_TYPE_CR2_QSFP: 3668 case FW_PORT_TYPE_SFP28: 3669 case FW_PORT_TYPE_SFP56: 3670 case FW_PORT_TYPE_QSFP56: 3671 case FW_PORT_TYPE_QSFPDD: 3672 /* Pluggable transceiver */ 3673 switch (pi->mod_type) { 3674 case FW_PORT_MOD_TYPE_LR: 3675 case FW_PORT_MOD_TYPE_LR_SIMPLEX: 3676 switch (speed) { 3677 case FW_PORT_CAP32_SPEED_1G: 3678 return (IFM_1000_LX); 3679 case FW_PORT_CAP32_SPEED_10G: 3680 return (IFM_10G_LR); 3681 case FW_PORT_CAP32_SPEED_25G: 3682 return (IFM_25G_LR); 3683 case FW_PORT_CAP32_SPEED_40G: 3684 return (IFM_40G_LR4); 3685 case FW_PORT_CAP32_SPEED_50G: 3686 return (IFM_50G_LR2); 3687 case FW_PORT_CAP32_SPEED_100G: 3688 return (IFM_100G_LR4); 3689 case FW_PORT_CAP32_SPEED_200G: 3690 return (IFM_200G_LR4); 3691 case FW_PORT_CAP32_SPEED_400G: 3692 return (IFM_400G_LR8); 3693 } 3694 break; 3695 case FW_PORT_MOD_TYPE_SR: 3696 switch (speed) { 3697 case FW_PORT_CAP32_SPEED_1G: 3698 return (IFM_1000_SX); 3699 case FW_PORT_CAP32_SPEED_10G: 3700 return (IFM_10G_SR); 3701 case FW_PORT_CAP32_SPEED_25G: 3702 return (IFM_25G_SR); 3703 case FW_PORT_CAP32_SPEED_40G: 3704 return (IFM_40G_SR4); 3705 case FW_PORT_CAP32_SPEED_50G: 3706 return (IFM_50G_SR2); 3707 case FW_PORT_CAP32_SPEED_100G: 3708 return (IFM_100G_SR4); 3709 case FW_PORT_CAP32_SPEED_200G: 3710 return (IFM_200G_SR4); 3711 case FW_PORT_CAP32_SPEED_400G: 3712 return (IFM_400G_SR8); 3713 } 3714 break; 3715 case FW_PORT_MOD_TYPE_ER: 3716 if (speed == FW_PORT_CAP32_SPEED_10G) 3717 return (IFM_10G_ER); 3718 break; 3719 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 3720 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 3721 switch (speed) { 3722 case FW_PORT_CAP32_SPEED_1G: 3723 return (IFM_1000_CX); 3724 case FW_PORT_CAP32_SPEED_10G: 3725 return (IFM_10G_TWINAX); 3726 case FW_PORT_CAP32_SPEED_25G: 3727 return (IFM_25G_CR); 3728 case FW_PORT_CAP32_SPEED_40G: 3729 return (IFM_40G_CR4); 3730 case FW_PORT_CAP32_SPEED_50G: 3731 return (IFM_50G_CR2); 3732 case FW_PORT_CAP32_SPEED_100G: 3733 return (IFM_100G_CR4); 3734 case FW_PORT_CAP32_SPEED_200G: 3735 return (IFM_200G_CR4_PAM4); 3736 case FW_PORT_CAP32_SPEED_400G: 3737 return (IFM_400G_CR8); 3738 } 3739 break; 3740 case FW_PORT_MOD_TYPE_LRM: 3741 if (speed == FW_PORT_CAP32_SPEED_10G) 3742 return (IFM_10G_LRM); 3743 break; 3744 case FW_PORT_MOD_TYPE_DR: 3745 if (speed == FW_PORT_CAP32_SPEED_100G) 3746 return (IFM_100G_DR); 3747 if (speed == FW_PORT_CAP32_SPEED_200G) 3748 return (IFM_200G_DR4); 3749 if (speed == FW_PORT_CAP32_SPEED_400G) 3750 return (IFM_400G_DR4); 3751 break; 3752 case FW_PORT_MOD_TYPE_NA: 3753 MPASS(0); /* Not pluggable? */ 3754 /* fall through */ 3755 case FW_PORT_MOD_TYPE_ERROR: 3756 case FW_PORT_MOD_TYPE_UNKNOWN: 3757 case FW_PORT_MOD_TYPE_NOTSUPPORTED: 3758 break; 3759 case FW_PORT_MOD_TYPE_NONE: 3760 return (IFM_NONE); 3761 } 3762 break; 3763 case M_FW_PORT_CMD_PTYPE: /* FW_PORT_TYPE_NONE for old firmware */ 3764 if (chip_id(pi->adapter) >= CHELSIO_T7) 3765 return (IFM_UNKNOWN); 3766 /* fall through */ 3767 case FW_PORT_TYPE_NONE: 3768 return (IFM_NONE); 3769 } 3770 3771 return (IFM_UNKNOWN); 3772 } 3773 3774 void 3775 cxgbe_media_status(if_t ifp, struct ifmediareq *ifmr) 3776 { 3777 struct vi_info *vi = if_getsoftc(ifp); 3778 struct port_info *pi = vi->pi; 3779 struct adapter *sc = pi->adapter; 3780 struct link_config *lc = &pi->link_cfg; 3781 3782 if (begin_synchronized_op(sc, vi , SLEEP_OK | INTR_OK, "t4med") != 0) 3783 return; 3784 PORT_LOCK(pi); 3785 3786 if (pi->up_vis == 0 && hw_all_ok(sc)) { 3787 /* 3788 * If all the interfaces are administratively down the firmware 3789 * does not report transceiver changes. Refresh port info here 3790 * so that ifconfig displays accurate ifmedia at all times. 3791 * This is the only reason we have a synchronized op in this 3792 * function. Just PORT_LOCK would have been enough otherwise. 3793 */ 3794 t4_update_port_info(pi); 3795 build_medialist(pi); 3796 } 3797 3798 /* ifm_status */ 3799 ifmr->ifm_status = IFM_AVALID; 3800 if (lc->link_ok == false) 3801 goto done; 3802 ifmr->ifm_status |= IFM_ACTIVE; 3803 3804 /* ifm_active */ 3805 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 3806 ifmr->ifm_active &= ~(IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE); 3807 if (lc->fc & PAUSE_RX) 3808 ifmr->ifm_active |= IFM_ETH_RXPAUSE; 3809 if (lc->fc & PAUSE_TX) 3810 ifmr->ifm_active |= IFM_ETH_TXPAUSE; 3811 ifmr->ifm_active |= port_mword(pi, speed_to_fwcap(lc->speed)); 3812 done: 3813 PORT_UNLOCK(pi); 3814 end_synchronized_op(sc, 0); 3815 } 3816 3817 static int 3818 vcxgbe_probe(device_t dev) 3819 { 3820 struct vi_info *vi = device_get_softc(dev); 3821 3822 device_set_descf(dev, "port %d vi %td", vi->pi->port_id, 3823 vi - vi->pi->vi); 3824 3825 return (BUS_PROBE_DEFAULT); 3826 } 3827 3828 static int 3829 alloc_extra_vi(struct adapter *sc, struct port_info *pi, struct vi_info *vi) 3830 { 3831 int func, index, rc; 3832 uint32_t param, val; 3833 3834 ASSERT_SYNCHRONIZED_OP(sc); 3835 3836 index = vi - pi->vi; 3837 MPASS(index > 0); /* This function deals with _extra_ VIs only */ 3838 KASSERT(index < nitems(vi_mac_funcs), 3839 ("%s: VI %s doesn't have a MAC func", __func__, 3840 device_get_nameunit(vi->dev))); 3841 func = vi_mac_funcs[index]; 3842 rc = t4_alloc_vi_func(sc, sc->mbox, pi->hw_port, sc->pf, 0, 1, 3843 vi->hw_addr, &vi->rss_size, &vi->vfvld, &vi->vin, func, 0); 3844 if (rc < 0) { 3845 CH_ERR(vi, "failed to allocate virtual interface %d" 3846 "for port %d: %d\n", index, pi->port_id, -rc); 3847 return (-rc); 3848 } 3849 vi->viid = rc; 3850 3851 if (vi->rss_size == 1) { 3852 /* 3853 * This VI didn't get a slice of the RSS table. Reduce the 3854 * number of VIs being created (hw.cxgbe.num_vis) or modify the 3855 * configuration file (nvi, rssnvi for this PF) if this is a 3856 * problem. 3857 */ 3858 device_printf(vi->dev, "RSS table not available.\n"); 3859 vi->rss_base = 0xffff; 3860 3861 return (0); 3862 } 3863 3864 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 3865 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_RSSINFO) | 3866 V_FW_PARAMS_PARAM_YZ(vi->viid); 3867 rc = t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 3868 if (rc) 3869 vi->rss_base = 0xffff; 3870 else { 3871 MPASS((val >> 16) == vi->rss_size); 3872 vi->rss_base = val & 0xffff; 3873 } 3874 3875 return (0); 3876 } 3877 3878 static int 3879 vcxgbe_attach(device_t dev) 3880 { 3881 struct vi_info *vi; 3882 struct port_info *pi; 3883 struct adapter *sc; 3884 int rc; 3885 3886 vi = device_get_softc(dev); 3887 pi = vi->pi; 3888 sc = pi->adapter; 3889 3890 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4via"); 3891 if (rc) 3892 return (rc); 3893 rc = alloc_extra_vi(sc, pi, vi); 3894 end_synchronized_op(sc, 0); 3895 if (rc) 3896 return (rc); 3897 3898 cxgbe_vi_attach(dev, vi); 3899 3900 return (0); 3901 } 3902 3903 static int 3904 vcxgbe_detach(device_t dev) 3905 { 3906 struct vi_info *vi; 3907 struct adapter *sc; 3908 3909 vi = device_get_softc(dev); 3910 sc = vi->adapter; 3911 3912 begin_vi_detach(sc, vi); 3913 cxgbe_vi_detach(vi); 3914 t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid); 3915 end_vi_detach(sc, vi); 3916 3917 return (0); 3918 } 3919 3920 static struct callout fatal_callout; 3921 static struct taskqueue *reset_tq; 3922 3923 static void 3924 delayed_panic(void *arg) 3925 { 3926 struct adapter *sc = arg; 3927 3928 panic("%s: panic on fatal error", device_get_nameunit(sc->dev)); 3929 } 3930 3931 static void 3932 fatal_error_task(void *arg, int pending) 3933 { 3934 struct adapter *sc = arg; 3935 int rc; 3936 3937 if (atomic_testandclear_int(&sc->error_flags, ilog2(ADAP_CIM_ERR))) { 3938 dump_cim_regs(sc); 3939 dump_cimla(sc); 3940 dump_devlog(sc); 3941 } 3942 3943 if (t4_reset_on_fatal_err) { 3944 CH_ALERT(sc, "resetting adapter after fatal error.\n"); 3945 rc = reset_adapter(sc); 3946 if (rc == 0 && t4_panic_on_fatal_err) { 3947 CH_ALERT(sc, "reset was successful, " 3948 "system will NOT panic.\n"); 3949 return; 3950 } 3951 } 3952 3953 if (t4_panic_on_fatal_err) { 3954 CH_ALERT(sc, "panicking on fatal error (after 30s).\n"); 3955 callout_reset(&fatal_callout, hz * 30, delayed_panic, sc); 3956 } 3957 } 3958 3959 void 3960 t4_fatal_err(struct adapter *sc, bool fw_error) 3961 { 3962 stop_adapter(sc); 3963 if (atomic_testandset_int(&sc->error_flags, ilog2(ADAP_FATAL_ERR))) 3964 return; 3965 if (fw_error) { 3966 /* 3967 * We are here because of a firmware error/timeout and not 3968 * because of a hardware interrupt. It is possible (although 3969 * not very likely) that an error interrupt was also raised but 3970 * this thread ran first and inhibited t4_intr_err. We walk the 3971 * main INT_CAUSE registers here to make sure we haven't missed 3972 * anything interesting. 3973 */ 3974 t4_slow_intr_handler(sc, sc->intr_flags); 3975 atomic_set_int(&sc->error_flags, ADAP_CIM_ERR); 3976 } 3977 t4_report_fw_error(sc); 3978 log(LOG_ALERT, "%s: encountered fatal error, adapter stopped (%d).\n", 3979 device_get_nameunit(sc->dev), fw_error); 3980 taskqueue_enqueue(reset_tq, &sc->fatal_error_task); 3981 } 3982 3983 void 3984 t4_add_adapter(struct adapter *sc) 3985 { 3986 sx_xlock(&t4_list_lock); 3987 SLIST_INSERT_HEAD(&t4_list, sc, link); 3988 sx_xunlock(&t4_list_lock); 3989 } 3990 3991 int 3992 t4_map_bars_0_and_4(struct adapter *sc) 3993 { 3994 sc->regs_rid = PCIR_BAR(0); 3995 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 3996 &sc->regs_rid, RF_ACTIVE); 3997 if (sc->regs_res == NULL) { 3998 device_printf(sc->dev, "cannot map registers.\n"); 3999 return (ENXIO); 4000 } 4001 sc->mmio_len = rman_get_size(sc->regs_res); 4002 setbit(&sc->doorbells, DOORBELL_KDB); 4003 4004 sc->msix_rid = PCIR_BAR(4); 4005 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 4006 &sc->msix_rid, RF_ACTIVE); 4007 if (sc->msix_res == NULL) { 4008 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 4009 return (ENXIO); 4010 } 4011 4012 return (0); 4013 } 4014 4015 int 4016 t4_map_bar_2(struct adapter *sc) 4017 { 4018 4019 /* 4020 * T4: only iWARP driver uses the userspace doorbells. There is no need 4021 * to map it if RDMA is disabled. 4022 */ 4023 if (is_t4(sc) && sc->rdmacaps == 0) 4024 return (0); 4025 4026 sc->udbs_rid = PCIR_BAR(2); 4027 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 4028 &sc->udbs_rid, RF_ACTIVE); 4029 if (sc->udbs_res == NULL) { 4030 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 4031 return (ENXIO); 4032 } 4033 sc->udbs_base = rman_get_virtual(sc->udbs_res); 4034 4035 if (chip_id(sc) >= CHELSIO_T5) { 4036 setbit(&sc->doorbells, DOORBELL_UDB); 4037 #if defined(__i386__) || defined(__amd64__) 4038 if (t5_write_combine) { 4039 int rc, mode; 4040 4041 /* 4042 * Enable write combining on BAR2. This is the 4043 * userspace doorbell BAR and is split into 128B 4044 * (UDBS_SEG_SIZE) doorbell regions, each associated 4045 * with an egress queue. The first 64B has the doorbell 4046 * and the second 64B can be used to submit a tx work 4047 * request with an implicit doorbell. 4048 */ 4049 4050 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 4051 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 4052 if (rc == 0) { 4053 clrbit(&sc->doorbells, DOORBELL_UDB); 4054 setbit(&sc->doorbells, DOORBELL_WCWR); 4055 setbit(&sc->doorbells, DOORBELL_UDBWC); 4056 } else { 4057 device_printf(sc->dev, 4058 "couldn't enable write combining: %d\n", 4059 rc); 4060 } 4061 4062 mode = is_t5(sc) ? V_STATMODE(0) : V_T6_STATMODE(0); 4063 t4_write_reg(sc, A_SGE_STAT_CFG, 4064 V_STATSOURCE_T5(7) | mode); 4065 } 4066 #endif 4067 } 4068 sc->iwt.wc_en = isset(&sc->doorbells, DOORBELL_UDBWC) ? 1 : 0; 4069 4070 return (0); 4071 } 4072 4073 int 4074 t4_adj_doorbells(struct adapter *sc) 4075 { 4076 if ((sc->doorbells & t4_doorbells_allowed) != 0) { 4077 sc->doorbells &= t4_doorbells_allowed; 4078 return (0); 4079 } 4080 CH_ERR(sc, "No usable doorbell (available = 0x%x, allowed = 0x%x).\n", 4081 sc->doorbells, t4_doorbells_allowed); 4082 return (EINVAL); 4083 } 4084 4085 struct memwin_init { 4086 uint32_t base; 4087 uint32_t aperture; 4088 }; 4089 4090 static const struct memwin_init t4_memwin[NUM_MEMWIN] = { 4091 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 4092 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 4093 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 4094 }; 4095 4096 static const struct memwin_init t5_memwin[NUM_MEMWIN] = { 4097 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 4098 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 4099 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 4100 }; 4101 4102 static void 4103 setup_memwin(struct adapter *sc) 4104 { 4105 const struct memwin_init *mw_init; 4106 struct memwin *mw; 4107 int i; 4108 uint32_t bar0, reg; 4109 4110 if (is_t4(sc)) { 4111 /* 4112 * Read low 32b of bar0 indirectly via the hardware backdoor 4113 * mechanism. Works from within PCI passthrough environments 4114 * too, where rman_get_start() can return a different value. We 4115 * need to program the T4 memory window decoders with the actual 4116 * addresses that will be coming across the PCIe link. 4117 */ 4118 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 4119 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 4120 4121 mw_init = &t4_memwin[0]; 4122 } else { 4123 /* T5+ use the relative offset inside the PCIe BAR */ 4124 bar0 = 0; 4125 4126 mw_init = &t5_memwin[0]; 4127 } 4128 4129 for (i = 0, mw = &sc->memwin[0]; i < NUM_MEMWIN; i++, mw_init++, mw++) { 4130 if (!rw_initialized(&mw->mw_lock)) { 4131 rw_init(&mw->mw_lock, "memory window access"); 4132 mw->mw_base = mw_init->base; 4133 mw->mw_aperture = mw_init->aperture; 4134 mw->mw_curpos = 0; 4135 } 4136 reg = chip_id(sc) > CHELSIO_T6 ? 4137 PCIE_MEM_ACCESS_T7_REG(A_T7_PCIE_MEM_ACCESS_BASE_WIN, i) : 4138 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i); 4139 t4_write_reg(sc, reg, (mw->mw_base + bar0) | V_BIR(0) | 4140 V_WINDOW(ilog2(mw->mw_aperture) - 10)); 4141 rw_wlock(&mw->mw_lock); 4142 position_memwin(sc, i, mw->mw_curpos); 4143 rw_wunlock(&mw->mw_lock); 4144 } 4145 4146 /* flush */ 4147 t4_read_reg(sc, reg); 4148 } 4149 4150 /* 4151 * Positions the memory window at the given address in the card's address space. 4152 * There are some alignment requirements and the actual position may be at an 4153 * address prior to the requested address. mw->mw_curpos always has the actual 4154 * position of the window. 4155 */ 4156 static void 4157 position_memwin(struct adapter *sc, int idx, uint32_t addr) 4158 { 4159 struct memwin *mw; 4160 uint32_t pf, reg, val; 4161 4162 MPASS(idx >= 0 && idx < NUM_MEMWIN); 4163 mw = &sc->memwin[idx]; 4164 rw_assert(&mw->mw_lock, RA_WLOCKED); 4165 4166 if (is_t4(sc)) { 4167 pf = 0; 4168 mw->mw_curpos = addr & ~0xf; /* start must be 16B aligned */ 4169 } else { 4170 pf = V_PFNUM(sc->pf); 4171 mw->mw_curpos = addr & ~0x7f; /* start must be 128B aligned */ 4172 } 4173 if (chip_id(sc) > CHELSIO_T6) { 4174 reg = PCIE_MEM_ACCESS_T7_REG(A_PCIE_MEM_ACCESS_OFFSET0, idx); 4175 val = (mw->mw_curpos >> X_T7_MEMOFST_SHIFT) | pf; 4176 } else { 4177 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, idx); 4178 val = mw->mw_curpos | pf; 4179 } 4180 t4_write_reg(sc, reg, val); 4181 t4_read_reg(sc, reg); /* flush */ 4182 } 4183 4184 int 4185 rw_via_memwin(struct adapter *sc, int idx, uint32_t addr, uint32_t *val, 4186 int len, int rw) 4187 { 4188 struct memwin *mw; 4189 uint32_t mw_end, v; 4190 4191 MPASS(idx >= 0 && idx < NUM_MEMWIN); 4192 4193 /* Memory can only be accessed in naturally aligned 4 byte units */ 4194 if (addr & 3 || len & 3 || len <= 0) 4195 return (EINVAL); 4196 4197 mw = &sc->memwin[idx]; 4198 while (len > 0) { 4199 rw_rlock(&mw->mw_lock); 4200 mw_end = mw->mw_curpos + mw->mw_aperture; 4201 if (addr >= mw_end || addr < mw->mw_curpos) { 4202 /* Will need to reposition the window */ 4203 if (!rw_try_upgrade(&mw->mw_lock)) { 4204 rw_runlock(&mw->mw_lock); 4205 rw_wlock(&mw->mw_lock); 4206 } 4207 rw_assert(&mw->mw_lock, RA_WLOCKED); 4208 position_memwin(sc, idx, addr); 4209 rw_downgrade(&mw->mw_lock); 4210 mw_end = mw->mw_curpos + mw->mw_aperture; 4211 } 4212 rw_assert(&mw->mw_lock, RA_RLOCKED); 4213 while (addr < mw_end && len > 0) { 4214 if (rw == 0) { 4215 v = t4_read_reg(sc, mw->mw_base + addr - 4216 mw->mw_curpos); 4217 *val++ = le32toh(v); 4218 } else { 4219 v = *val++; 4220 t4_write_reg(sc, mw->mw_base + addr - 4221 mw->mw_curpos, htole32(v)); 4222 } 4223 addr += 4; 4224 len -= 4; 4225 } 4226 rw_runlock(&mw->mw_lock); 4227 } 4228 4229 return (0); 4230 } 4231 4232 CTASSERT(M_TID_COOKIE == M_COOKIE); 4233 CTASSERT(MAX_ATIDS <= (M_TID_TID + 1)); 4234 4235 static void 4236 t4_init_atid_table(struct adapter *sc) 4237 { 4238 struct tid_info *t; 4239 int i; 4240 4241 t = &sc->tids; 4242 if (t->natids == 0) 4243 return; 4244 4245 MPASS(t->atid_tab == NULL); 4246 4247 t->atid_tab = malloc(t->natids * sizeof(*t->atid_tab), M_CXGBE, 4248 M_ZERO | M_WAITOK); 4249 mtx_init(&t->atid_lock, "atid lock", NULL, MTX_DEF); 4250 t->afree = t->atid_tab; 4251 t->atids_in_use = 0; 4252 t->atid_alloc_stopped = false; 4253 for (i = 1; i < t->natids; i++) 4254 t->atid_tab[i - 1].next = &t->atid_tab[i]; 4255 t->atid_tab[t->natids - 1].next = NULL; 4256 } 4257 4258 static void 4259 t4_free_atid_table(struct adapter *sc) 4260 { 4261 struct tid_info *t; 4262 4263 t = &sc->tids; 4264 4265 KASSERT(t->atids_in_use == 0, 4266 ("%s: %d atids still in use.", __func__, t->atids_in_use)); 4267 4268 if (mtx_initialized(&t->atid_lock)) 4269 mtx_destroy(&t->atid_lock); 4270 free(t->atid_tab, M_CXGBE); 4271 t->atid_tab = NULL; 4272 } 4273 4274 static void 4275 stop_atid_allocator(struct adapter *sc) 4276 { 4277 struct tid_info *t = &sc->tids; 4278 4279 if (t->natids == 0) 4280 return; 4281 mtx_lock(&t->atid_lock); 4282 t->atid_alloc_stopped = true; 4283 mtx_unlock(&t->atid_lock); 4284 } 4285 4286 static void 4287 restart_atid_allocator(struct adapter *sc) 4288 { 4289 struct tid_info *t = &sc->tids; 4290 4291 if (t->natids == 0) 4292 return; 4293 mtx_lock(&t->atid_lock); 4294 KASSERT(t->atids_in_use == 0, 4295 ("%s: %d atids still in use.", __func__, t->atids_in_use)); 4296 t->atid_alloc_stopped = false; 4297 mtx_unlock(&t->atid_lock); 4298 } 4299 4300 int 4301 alloc_atid(struct adapter *sc, void *ctx) 4302 { 4303 struct tid_info *t = &sc->tids; 4304 int atid = -1; 4305 4306 mtx_lock(&t->atid_lock); 4307 if (t->afree && !t->atid_alloc_stopped) { 4308 union aopen_entry *p = t->afree; 4309 4310 atid = p - t->atid_tab; 4311 MPASS(atid <= M_TID_TID); 4312 t->afree = p->next; 4313 p->data = ctx; 4314 t->atids_in_use++; 4315 } 4316 mtx_unlock(&t->atid_lock); 4317 return (atid); 4318 } 4319 4320 void * 4321 lookup_atid(struct adapter *sc, int atid) 4322 { 4323 struct tid_info *t = &sc->tids; 4324 4325 return (t->atid_tab[atid].data); 4326 } 4327 4328 void 4329 free_atid(struct adapter *sc, int atid) 4330 { 4331 struct tid_info *t = &sc->tids; 4332 union aopen_entry *p = &t->atid_tab[atid]; 4333 4334 mtx_lock(&t->atid_lock); 4335 p->next = t->afree; 4336 t->afree = p; 4337 t->atids_in_use--; 4338 mtx_unlock(&t->atid_lock); 4339 } 4340 4341 static void 4342 queue_tid_release(struct adapter *sc, int tid) 4343 { 4344 4345 CXGBE_UNIMPLEMENTED("deferred tid release"); 4346 } 4347 4348 void 4349 release_tid(struct adapter *sc, int tid, struct sge_wrq *ctrlq) 4350 { 4351 struct wrqe *wr; 4352 struct cpl_tid_release *req; 4353 4354 wr = alloc_wrqe(sizeof(*req), ctrlq); 4355 if (wr == NULL) { 4356 queue_tid_release(sc, tid); /* defer */ 4357 return; 4358 } 4359 req = wrtod(wr); 4360 4361 INIT_TP_WR_MIT_CPL(req, CPL_TID_RELEASE, tid); 4362 4363 t4_wrq_tx(sc, wr); 4364 } 4365 4366 static int 4367 t4_range_cmp(const void *a, const void *b) 4368 { 4369 return ((const struct t4_range *)a)->start - 4370 ((const struct t4_range *)b)->start; 4371 } 4372 4373 /* 4374 * Verify that the memory range specified by the addr/len pair is valid within 4375 * the card's address space. 4376 */ 4377 static int 4378 validate_mem_range(struct adapter *sc, uint32_t addr, uint32_t len) 4379 { 4380 struct t4_range mem_ranges[4], *r, *next; 4381 uint32_t em, addr_len; 4382 int i, n, remaining; 4383 4384 /* Memory can only be accessed in naturally aligned 4 byte units */ 4385 if (addr & 3 || len & 3 || len == 0) 4386 return (EINVAL); 4387 4388 /* Enabled memories */ 4389 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 4390 4391 r = &mem_ranges[0]; 4392 n = 0; 4393 bzero(r, sizeof(mem_ranges)); 4394 if (em & F_EDRAM0_ENABLE) { 4395 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 4396 r->size = G_EDRAM0_SIZE(addr_len) << 20; 4397 if (r->size > 0) { 4398 r->start = G_EDRAM0_BASE(addr_len) << 20; 4399 if (addr >= r->start && 4400 addr + len <= r->start + r->size) 4401 return (0); 4402 r++; 4403 n++; 4404 } 4405 } 4406 if (em & F_EDRAM1_ENABLE) { 4407 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 4408 r->size = G_EDRAM1_SIZE(addr_len) << 20; 4409 if (r->size > 0) { 4410 r->start = G_EDRAM1_BASE(addr_len) << 20; 4411 if (addr >= r->start && 4412 addr + len <= r->start + r->size) 4413 return (0); 4414 r++; 4415 n++; 4416 } 4417 } 4418 if (em & F_EXT_MEM_ENABLE) { 4419 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 4420 r->size = G_EXT_MEM_SIZE(addr_len) << 20; 4421 if (r->size > 0) { 4422 r->start = G_EXT_MEM_BASE(addr_len) << 20; 4423 if (addr >= r->start && 4424 addr + len <= r->start + r->size) 4425 return (0); 4426 r++; 4427 n++; 4428 } 4429 } 4430 if (is_t5(sc) && em & F_EXT_MEM1_ENABLE) { 4431 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 4432 r->size = G_EXT_MEM1_SIZE(addr_len) << 20; 4433 if (r->size > 0) { 4434 r->start = G_EXT_MEM1_BASE(addr_len) << 20; 4435 if (addr >= r->start && 4436 addr + len <= r->start + r->size) 4437 return (0); 4438 r++; 4439 n++; 4440 } 4441 } 4442 MPASS(n <= nitems(mem_ranges)); 4443 4444 if (n > 1) { 4445 /* Sort and merge the ranges. */ 4446 qsort(mem_ranges, n, sizeof(struct t4_range), t4_range_cmp); 4447 4448 /* Start from index 0 and examine the next n - 1 entries. */ 4449 r = &mem_ranges[0]; 4450 for (remaining = n - 1; remaining > 0; remaining--, r++) { 4451 4452 MPASS(r->size > 0); /* r is a valid entry. */ 4453 next = r + 1; 4454 MPASS(next->size > 0); /* and so is the next one. */ 4455 4456 while (r->start + r->size >= next->start) { 4457 /* Merge the next one into the current entry. */ 4458 r->size = max(r->start + r->size, 4459 next->start + next->size) - r->start; 4460 n--; /* One fewer entry in total. */ 4461 if (--remaining == 0) 4462 goto done; /* short circuit */ 4463 next++; 4464 } 4465 if (next != r + 1) { 4466 /* 4467 * Some entries were merged into r and next 4468 * points to the first valid entry that couldn't 4469 * be merged. 4470 */ 4471 MPASS(next->size > 0); /* must be valid */ 4472 memcpy(r + 1, next, remaining * sizeof(*r)); 4473 #ifdef INVARIANTS 4474 /* 4475 * This so that the foo->size assertion in the 4476 * next iteration of the loop do the right 4477 * thing for entries that were pulled up and are 4478 * no longer valid. 4479 */ 4480 MPASS(n < nitems(mem_ranges)); 4481 bzero(&mem_ranges[n], (nitems(mem_ranges) - n) * 4482 sizeof(struct t4_range)); 4483 #endif 4484 } 4485 } 4486 done: 4487 /* Done merging the ranges. */ 4488 MPASS(n > 0); 4489 r = &mem_ranges[0]; 4490 for (i = 0; i < n; i++, r++) { 4491 if (addr >= r->start && 4492 addr + len <= r->start + r->size) 4493 return (0); 4494 } 4495 } 4496 4497 return (EFAULT); 4498 } 4499 4500 static int 4501 fwmtype_to_hwmtype(int mtype) 4502 { 4503 4504 switch (mtype) { 4505 case FW_MEMTYPE_EDC0: 4506 return (MEM_EDC0); 4507 case FW_MEMTYPE_EDC1: 4508 return (MEM_EDC1); 4509 case FW_MEMTYPE_EXTMEM: 4510 return (MEM_MC0); 4511 case FW_MEMTYPE_EXTMEM1: 4512 return (MEM_MC1); 4513 default: 4514 panic("%s: cannot translate fw mtype %d.", __func__, mtype); 4515 } 4516 } 4517 4518 /* 4519 * Verify that the memory range specified by the memtype/offset/len pair is 4520 * valid and lies entirely within the memtype specified. The global address of 4521 * the start of the range is returned in addr. 4522 */ 4523 static int 4524 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, uint32_t len, 4525 uint32_t *addr) 4526 { 4527 uint32_t em, addr_len, maddr; 4528 4529 /* Memory can only be accessed in naturally aligned 4 byte units */ 4530 if (off & 3 || len & 3 || len == 0) 4531 return (EINVAL); 4532 4533 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 4534 switch (fwmtype_to_hwmtype(mtype)) { 4535 case MEM_EDC0: 4536 if (!(em & F_EDRAM0_ENABLE)) 4537 return (EINVAL); 4538 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 4539 maddr = G_EDRAM0_BASE(addr_len) << 20; 4540 break; 4541 case MEM_EDC1: 4542 if (!(em & F_EDRAM1_ENABLE)) 4543 return (EINVAL); 4544 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 4545 maddr = G_EDRAM1_BASE(addr_len) << 20; 4546 break; 4547 case MEM_MC: 4548 if (!(em & F_EXT_MEM_ENABLE)) 4549 return (EINVAL); 4550 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 4551 maddr = G_EXT_MEM_BASE(addr_len) << 20; 4552 break; 4553 case MEM_MC1: 4554 if (!is_t5(sc) || !(em & F_EXT_MEM1_ENABLE)) 4555 return (EINVAL); 4556 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 4557 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 4558 break; 4559 default: 4560 return (EINVAL); 4561 } 4562 4563 *addr = maddr + off; /* global address */ 4564 return (validate_mem_range(sc, *addr, len)); 4565 } 4566 4567 static int 4568 fixup_devlog_ncores_params(struct adapter *sc) 4569 { 4570 struct devlog_params *dparams = &sc->params.devlog; 4571 int rc; 4572 4573 #ifdef INVARIANTS 4574 if (sc->params.ncores > 1) 4575 MPASS(chip_id(sc) >= CHELSIO_T7); 4576 #endif 4577 rc = validate_mt_off_len(sc, dparams->memtype, dparams->start, 4578 dparams->size, &dparams->addr); 4579 4580 return (rc); 4581 } 4582 4583 static void 4584 update_nirq(struct intrs_and_queues *iaq, int nports) 4585 { 4586 4587 iaq->nirq = T4_EXTRA_INTR; 4588 iaq->nirq += nports * max(iaq->nrxq, iaq->nnmrxq); 4589 iaq->nirq += nports * iaq->nofldrxq; 4590 iaq->nirq += nports * (iaq->num_vis - 1) * 4591 max(iaq->nrxq_vi, iaq->nnmrxq_vi); 4592 iaq->nirq += nports * (iaq->num_vis - 1) * iaq->nofldrxq_vi; 4593 } 4594 4595 /* 4596 * Adjust requirements to fit the number of interrupts available. 4597 */ 4598 static void 4599 calculate_iaq(struct adapter *sc, struct intrs_and_queues *iaq, int itype, 4600 int navail) 4601 { 4602 int old_nirq; 4603 const int nports = sc->params.nports; 4604 4605 MPASS(nports > 0); 4606 MPASS(navail > 0); 4607 4608 bzero(iaq, sizeof(*iaq)); 4609 iaq->intr_type = itype; 4610 iaq->num_vis = t4_num_vis; 4611 iaq->ntxq = t4_ntxq; 4612 iaq->ntxq_vi = t4_ntxq_vi; 4613 iaq->nrxq = t4_nrxq; 4614 iaq->nrxq_vi = t4_nrxq_vi; 4615 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 4616 if (is_offload(sc) || is_ethoffload(sc)) { 4617 if (sc->params.tid_qid_sel_mask == 0) { 4618 iaq->nofldtxq = t4_nofldtxq; 4619 iaq->nofldtxq_vi = t4_nofldtxq_vi; 4620 } else { 4621 iaq->nofldtxq = roundup(t4_nofldtxq, sc->params.ncores); 4622 iaq->nofldtxq_vi = roundup(t4_nofldtxq_vi, 4623 sc->params.ncores); 4624 if (iaq->nofldtxq != t4_nofldtxq) 4625 device_printf(sc->dev, 4626 "nofldtxq updated (%d -> %d) for correct" 4627 " operation with %d firmware cores.\n", 4628 t4_nofldtxq, iaq->nofldtxq, 4629 sc->params.ncores); 4630 if (iaq->num_vis > 1 && 4631 iaq->nofldtxq_vi != t4_nofldtxq_vi) 4632 device_printf(sc->dev, 4633 "nofldtxq_vi updated (%d -> %d) for correct" 4634 " operation with %d firmware cores.\n", 4635 t4_nofldtxq_vi, iaq->nofldtxq_vi, 4636 sc->params.ncores); 4637 } 4638 } 4639 #endif 4640 #ifdef TCP_OFFLOAD 4641 if (is_offload(sc)) { 4642 iaq->nofldrxq = t4_nofldrxq; 4643 iaq->nofldrxq_vi = t4_nofldrxq_vi; 4644 } 4645 #endif 4646 #ifdef DEV_NETMAP 4647 if (t4_native_netmap & NN_MAIN_VI) { 4648 iaq->nnmtxq = t4_nnmtxq; 4649 iaq->nnmrxq = t4_nnmrxq; 4650 } 4651 if (t4_native_netmap & NN_EXTRA_VI) { 4652 iaq->nnmtxq_vi = t4_nnmtxq_vi; 4653 iaq->nnmrxq_vi = t4_nnmrxq_vi; 4654 } 4655 #endif 4656 4657 update_nirq(iaq, nports); 4658 if (iaq->nirq <= navail && 4659 (itype != INTR_MSI || powerof2(iaq->nirq))) { 4660 /* 4661 * This is the normal case -- there are enough interrupts for 4662 * everything. 4663 */ 4664 goto done; 4665 } 4666 4667 /* 4668 * If extra VIs have been configured try reducing their count and see if 4669 * that works. 4670 */ 4671 while (iaq->num_vis > 1) { 4672 iaq->num_vis--; 4673 update_nirq(iaq, nports); 4674 if (iaq->nirq <= navail && 4675 (itype != INTR_MSI || powerof2(iaq->nirq))) { 4676 device_printf(sc->dev, "virtual interfaces per port " 4677 "reduced to %d from %d. nrxq=%u, nofldrxq=%u, " 4678 "nrxq_vi=%u nofldrxq_vi=%u, nnmrxq_vi=%u. " 4679 "itype %d, navail %u, nirq %d.\n", 4680 iaq->num_vis, t4_num_vis, iaq->nrxq, iaq->nofldrxq, 4681 iaq->nrxq_vi, iaq->nofldrxq_vi, iaq->nnmrxq_vi, 4682 itype, navail, iaq->nirq); 4683 goto done; 4684 } 4685 } 4686 4687 /* 4688 * Extra VIs will not be created. Log a message if they were requested. 4689 */ 4690 MPASS(iaq->num_vis == 1); 4691 iaq->ntxq_vi = iaq->nrxq_vi = 0; 4692 iaq->nofldtxq_vi = iaq->nofldrxq_vi = 0; 4693 iaq->nnmtxq_vi = iaq->nnmrxq_vi = 0; 4694 if (iaq->num_vis != t4_num_vis) { 4695 device_printf(sc->dev, "extra virtual interfaces disabled. " 4696 "nrxq=%u, nofldrxq=%u, nrxq_vi=%u nofldrxq_vi=%u, " 4697 "nnmrxq_vi=%u. itype %d, navail %u, nirq %d.\n", 4698 iaq->nrxq, iaq->nofldrxq, iaq->nrxq_vi, iaq->nofldrxq_vi, 4699 iaq->nnmrxq_vi, itype, navail, iaq->nirq); 4700 } 4701 4702 /* 4703 * Keep reducing the number of NIC rx queues to the next lower power of 4704 * 2 (for even RSS distribution) and halving the TOE rx queues and see 4705 * if that works. 4706 */ 4707 do { 4708 if (iaq->nrxq > 1) { 4709 iaq->nrxq = rounddown_pow_of_two(iaq->nrxq - 1); 4710 if (iaq->nnmrxq > iaq->nrxq) 4711 iaq->nnmrxq = iaq->nrxq; 4712 } 4713 if (iaq->nofldrxq > 1) 4714 iaq->nofldrxq >>= 1; 4715 4716 old_nirq = iaq->nirq; 4717 update_nirq(iaq, nports); 4718 if (iaq->nirq <= navail && 4719 (itype != INTR_MSI || powerof2(iaq->nirq))) { 4720 device_printf(sc->dev, "running with reduced number of " 4721 "rx queues because of shortage of interrupts. " 4722 "nrxq=%u, nofldrxq=%u. " 4723 "itype %d, navail %u, nirq %d.\n", iaq->nrxq, 4724 iaq->nofldrxq, itype, navail, iaq->nirq); 4725 goto done; 4726 } 4727 } while (old_nirq != iaq->nirq); 4728 4729 /* One interrupt for everything. Ugh. */ 4730 device_printf(sc->dev, "running with minimal number of queues. " 4731 "itype %d, navail %u.\n", itype, navail); 4732 iaq->nirq = 1; 4733 iaq->nrxq = 1; 4734 iaq->ntxq = 1; 4735 if (iaq->nofldrxq > 0) { 4736 iaq->nofldrxq = 1; 4737 iaq->nofldtxq = 1; 4738 if (sc->params.tid_qid_sel_mask == 0) 4739 iaq->nofldtxq = 1; 4740 else 4741 iaq->nofldtxq = sc->params.ncores; 4742 } 4743 iaq->nnmtxq = 0; 4744 iaq->nnmrxq = 0; 4745 done: 4746 MPASS(iaq->num_vis > 0); 4747 if (iaq->num_vis > 1) { 4748 MPASS(iaq->nrxq_vi > 0); 4749 MPASS(iaq->ntxq_vi > 0); 4750 } 4751 MPASS(iaq->nirq > 0); 4752 MPASS(iaq->nrxq > 0); 4753 MPASS(iaq->ntxq > 0); 4754 if (itype == INTR_MSI) 4755 MPASS(powerof2(iaq->nirq)); 4756 if (sc->params.tid_qid_sel_mask != 0) 4757 MPASS(iaq->nofldtxq % sc->params.ncores == 0); 4758 } 4759 4760 static int 4761 cfg_itype_and_nqueues(struct adapter *sc, struct intrs_and_queues *iaq) 4762 { 4763 int rc, itype, navail, nalloc; 4764 4765 for (itype = INTR_MSIX; itype; itype >>= 1) { 4766 4767 if ((itype & t4_intr_types) == 0) 4768 continue; /* not allowed */ 4769 4770 if (itype == INTR_MSIX) 4771 navail = pci_msix_count(sc->dev); 4772 else if (itype == INTR_MSI) 4773 navail = pci_msi_count(sc->dev); 4774 else 4775 navail = 1; 4776 restart: 4777 if (navail == 0) 4778 continue; 4779 4780 calculate_iaq(sc, iaq, itype, navail); 4781 nalloc = iaq->nirq; 4782 rc = 0; 4783 if (itype == INTR_MSIX) 4784 rc = pci_alloc_msix(sc->dev, &nalloc); 4785 else if (itype == INTR_MSI) 4786 rc = pci_alloc_msi(sc->dev, &nalloc); 4787 4788 if (rc == 0 && nalloc > 0) { 4789 if (nalloc == iaq->nirq) 4790 return (0); 4791 4792 /* 4793 * Didn't get the number requested. Use whatever number 4794 * the kernel is willing to allocate. 4795 */ 4796 device_printf(sc->dev, "fewer vectors than requested, " 4797 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 4798 itype, iaq->nirq, nalloc); 4799 pci_release_msi(sc->dev); 4800 navail = nalloc; 4801 goto restart; 4802 } 4803 4804 device_printf(sc->dev, 4805 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 4806 itype, rc, iaq->nirq, nalloc); 4807 } 4808 4809 device_printf(sc->dev, 4810 "failed to find a usable interrupt type. " 4811 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 4812 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 4813 4814 return (ENXIO); 4815 } 4816 4817 #define FW_VERSION(chip) ( \ 4818 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ 4819 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ 4820 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ 4821 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) 4822 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) 4823 4824 /* Just enough of fw_hdr to cover all version info. */ 4825 struct fw_h { 4826 __u8 ver; 4827 __u8 chip; 4828 __be16 len512; 4829 __be32 fw_ver; 4830 __be32 tp_microcode_ver; 4831 __u8 intfver_nic; 4832 __u8 intfver_vnic; 4833 __u8 intfver_ofld; 4834 __u8 intfver_ri; 4835 __u8 intfver_iscsipdu; 4836 __u8 intfver_iscsi; 4837 __u8 intfver_fcoepdu; 4838 __u8 intfver_fcoe; 4839 }; 4840 /* Spot check a couple of fields. */ 4841 CTASSERT(offsetof(struct fw_h, fw_ver) == offsetof(struct fw_hdr, fw_ver)); 4842 CTASSERT(offsetof(struct fw_h, intfver_nic) == offsetof(struct fw_hdr, intfver_nic)); 4843 CTASSERT(offsetof(struct fw_h, intfver_fcoe) == offsetof(struct fw_hdr, intfver_fcoe)); 4844 4845 struct fw_info { 4846 uint8_t chip; 4847 char *kld_name; 4848 char *fw_mod_name; 4849 struct fw_h fw_h; 4850 } fw_info[] = { 4851 { 4852 .chip = CHELSIO_T4, 4853 .kld_name = "t4fw_cfg", 4854 .fw_mod_name = "t4fw", 4855 .fw_h = { 4856 .chip = FW_HDR_CHIP_T4, 4857 .fw_ver = htobe32(FW_VERSION(T4)), 4858 .intfver_nic = FW_INTFVER(T4, NIC), 4859 .intfver_vnic = FW_INTFVER(T4, VNIC), 4860 .intfver_ofld = FW_INTFVER(T4, OFLD), 4861 .intfver_ri = FW_INTFVER(T4, RI), 4862 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 4863 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 4864 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 4865 .intfver_fcoe = FW_INTFVER(T4, FCOE), 4866 }, 4867 }, { 4868 .chip = CHELSIO_T5, 4869 .kld_name = "t5fw_cfg", 4870 .fw_mod_name = "t5fw", 4871 .fw_h = { 4872 .chip = FW_HDR_CHIP_T5, 4873 .fw_ver = htobe32(FW_VERSION(T5)), 4874 .intfver_nic = FW_INTFVER(T5, NIC), 4875 .intfver_vnic = FW_INTFVER(T5, VNIC), 4876 .intfver_ofld = FW_INTFVER(T5, OFLD), 4877 .intfver_ri = FW_INTFVER(T5, RI), 4878 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 4879 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 4880 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 4881 .intfver_fcoe = FW_INTFVER(T5, FCOE), 4882 }, 4883 }, { 4884 .chip = CHELSIO_T6, 4885 .kld_name = "t6fw_cfg", 4886 .fw_mod_name = "t6fw", 4887 .fw_h = { 4888 .chip = FW_HDR_CHIP_T6, 4889 .fw_ver = htobe32(FW_VERSION(T6)), 4890 .intfver_nic = FW_INTFVER(T6, NIC), 4891 .intfver_vnic = FW_INTFVER(T6, VNIC), 4892 .intfver_ofld = FW_INTFVER(T6, OFLD), 4893 .intfver_ri = FW_INTFVER(T6, RI), 4894 .intfver_iscsipdu = FW_INTFVER(T6, ISCSIPDU), 4895 .intfver_iscsi = FW_INTFVER(T6, ISCSI), 4896 .intfver_fcoepdu = FW_INTFVER(T6, FCOEPDU), 4897 .intfver_fcoe = FW_INTFVER(T6, FCOE), 4898 }, 4899 }, { 4900 .chip = CHELSIO_T7, 4901 .kld_name = "t7fw_cfg", 4902 .fw_mod_name = "t7fw", 4903 .fw_h = { 4904 .chip = FW_HDR_CHIP_T7, 4905 .fw_ver = htobe32(FW_VERSION(T7)), 4906 .intfver_nic = FW_INTFVER(T7, NIC), 4907 .intfver_vnic = FW_INTFVER(T7, VNIC), 4908 .intfver_ofld = FW_INTFVER(T7, OFLD), 4909 .intfver_ri = FW_INTFVER(T7, RI), 4910 .intfver_iscsipdu = FW_INTFVER(T7, ISCSIPDU), 4911 .intfver_iscsi = FW_INTFVER(T7, ISCSI), 4912 .intfver_fcoepdu = FW_INTFVER(T7, FCOEPDU), 4913 .intfver_fcoe = FW_INTFVER(T7, FCOE), 4914 }, 4915 } 4916 }; 4917 4918 static struct fw_info * 4919 find_fw_info(int chip) 4920 { 4921 int i; 4922 4923 for (i = 0; i < nitems(fw_info); i++) { 4924 if (fw_info[i].chip == chip) 4925 return (&fw_info[i]); 4926 } 4927 return (NULL); 4928 } 4929 4930 /* 4931 * Is the given firmware API compatible with the one the driver was compiled 4932 * with? 4933 */ 4934 static int 4935 fw_compatible(const struct fw_h *hdr1, const struct fw_h *hdr2) 4936 { 4937 4938 /* short circuit if it's the exact same firmware version */ 4939 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 4940 return (1); 4941 4942 /* 4943 * XXX: Is this too conservative? Perhaps I should limit this to the 4944 * features that are supported in the driver. 4945 */ 4946 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 4947 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 4948 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 4949 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 4950 return (1); 4951 #undef SAME_INTF 4952 4953 return (0); 4954 } 4955 4956 static int 4957 load_fw_module(struct adapter *sc, const struct firmware **dcfg, 4958 const struct firmware **fw) 4959 { 4960 struct fw_info *fw_info; 4961 4962 *dcfg = NULL; 4963 if (fw != NULL) 4964 *fw = NULL; 4965 4966 fw_info = find_fw_info(chip_id(sc)); 4967 if (fw_info == NULL) { 4968 device_printf(sc->dev, 4969 "unable to look up firmware information for chip %d.\n", 4970 chip_id(sc)); 4971 return (EINVAL); 4972 } 4973 4974 *dcfg = firmware_get(fw_info->kld_name); 4975 if (*dcfg != NULL) { 4976 if (fw != NULL) 4977 *fw = firmware_get(fw_info->fw_mod_name); 4978 return (0); 4979 } 4980 4981 return (ENOENT); 4982 } 4983 4984 static void 4985 unload_fw_module(struct adapter *sc, const struct firmware *dcfg, 4986 const struct firmware *fw) 4987 { 4988 4989 if (fw != NULL) 4990 firmware_put(fw, FIRMWARE_UNLOAD); 4991 if (dcfg != NULL) 4992 firmware_put(dcfg, FIRMWARE_UNLOAD); 4993 } 4994 4995 /* 4996 * Return values: 4997 * 0 means no firmware install attempted. 4998 * ERESTART means a firmware install was attempted and was successful. 4999 * +ve errno means a firmware install was attempted but failed. 5000 */ 5001 static int 5002 install_kld_firmware(struct adapter *sc, struct fw_h *card_fw, 5003 const struct fw_h *drv_fw, const char *reason, int *already) 5004 { 5005 const struct firmware *cfg, *fw; 5006 const uint32_t c = be32toh(card_fw->fw_ver); 5007 uint32_t d, k; 5008 int rc, fw_install; 5009 struct fw_h bundled_fw; 5010 bool load_attempted; 5011 5012 cfg = fw = NULL; 5013 load_attempted = false; 5014 fw_install = t4_fw_install < 0 ? -t4_fw_install : t4_fw_install; 5015 5016 memcpy(&bundled_fw, drv_fw, sizeof(bundled_fw)); 5017 if (t4_fw_install < 0) { 5018 rc = load_fw_module(sc, &cfg, &fw); 5019 if (rc != 0 || fw == NULL) { 5020 device_printf(sc->dev, 5021 "failed to load firmware module: %d. cfg %p, fw %p;" 5022 " will use compiled-in firmware version for" 5023 "hw.cxgbe.fw_install checks.\n", 5024 rc, cfg, fw); 5025 } else { 5026 memcpy(&bundled_fw, fw->data, sizeof(bundled_fw)); 5027 } 5028 load_attempted = true; 5029 } 5030 d = be32toh(bundled_fw.fw_ver); 5031 5032 if (reason != NULL) 5033 goto install; 5034 5035 if ((sc->flags & FW_OK) == 0) { 5036 5037 if (c == 0xffffffff) { 5038 reason = "missing"; 5039 goto install; 5040 } 5041 5042 rc = 0; 5043 goto done; 5044 } 5045 5046 if (!fw_compatible(card_fw, &bundled_fw)) { 5047 reason = "incompatible or unusable"; 5048 goto install; 5049 } 5050 5051 if (d > c) { 5052 reason = "older than the version bundled with this driver"; 5053 goto install; 5054 } 5055 5056 if (fw_install == 2 && d != c) { 5057 reason = "different than the version bundled with this driver"; 5058 goto install; 5059 } 5060 5061 /* No reason to do anything to the firmware already on the card. */ 5062 rc = 0; 5063 goto done; 5064 5065 install: 5066 rc = 0; 5067 if ((*already)++) 5068 goto done; 5069 5070 if (fw_install == 0) { 5071 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 5072 "but the driver is prohibited from installing a firmware " 5073 "on the card.\n", 5074 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 5075 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 5076 5077 goto done; 5078 } 5079 5080 /* 5081 * We'll attempt to install a firmware. Load the module first (if it 5082 * hasn't been loaded already). 5083 */ 5084 if (!load_attempted) { 5085 rc = load_fw_module(sc, &cfg, &fw); 5086 if (rc != 0 || fw == NULL) { 5087 device_printf(sc->dev, 5088 "failed to load firmware module: %d. cfg %p, fw %p\n", 5089 rc, cfg, fw); 5090 /* carry on */ 5091 } 5092 } 5093 if (fw == NULL) { 5094 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 5095 "but the driver cannot take corrective action because it " 5096 "is unable to load the firmware module.\n", 5097 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 5098 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 5099 rc = sc->flags & FW_OK ? 0 : ENOENT; 5100 goto done; 5101 } 5102 k = be32toh(((const struct fw_hdr *)fw->data)->fw_ver); 5103 if (k != d) { 5104 MPASS(t4_fw_install > 0); 5105 device_printf(sc->dev, 5106 "firmware in KLD (%u.%u.%u.%u) is not what the driver was " 5107 "expecting (%u.%u.%u.%u) and will not be used.\n", 5108 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 5109 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k), 5110 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 5111 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d)); 5112 rc = sc->flags & FW_OK ? 0 : EINVAL; 5113 goto done; 5114 } 5115 5116 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 5117 "installing firmware %u.%u.%u.%u on card.\n", 5118 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 5119 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 5120 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 5121 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d)); 5122 5123 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 5124 if (rc != 0) { 5125 device_printf(sc->dev, "failed to install firmware: %d\n", rc); 5126 } else { 5127 /* Installed successfully, update the cached header too. */ 5128 rc = ERESTART; 5129 memcpy(card_fw, fw->data, sizeof(*card_fw)); 5130 } 5131 done: 5132 unload_fw_module(sc, cfg, fw); 5133 5134 return (rc); 5135 } 5136 5137 /* 5138 * Establish contact with the firmware and attempt to become the master driver. 5139 * 5140 * A firmware will be installed to the card if needed (if the driver is allowed 5141 * to do so). 5142 */ 5143 static int 5144 contact_firmware(struct adapter *sc) 5145 { 5146 int rc, already = 0; 5147 enum dev_state state; 5148 struct fw_info *fw_info; 5149 struct fw_hdr *card_fw; /* fw on the card */ 5150 const struct fw_h *drv_fw; 5151 5152 fw_info = find_fw_info(chip_id(sc)); 5153 if (fw_info == NULL) { 5154 device_printf(sc->dev, 5155 "unable to look up firmware information for chip %d.\n", 5156 chip_id(sc)); 5157 return (EINVAL); 5158 } 5159 drv_fw = &fw_info->fw_h; 5160 5161 /* Read the header of the firmware on the card */ 5162 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 5163 restart: 5164 rc = -t4_get_fw_hdr(sc, card_fw); 5165 if (rc != 0) { 5166 device_printf(sc->dev, 5167 "unable to read firmware header from card's flash: %d\n", 5168 rc); 5169 goto done; 5170 } 5171 5172 rc = install_kld_firmware(sc, (struct fw_h *)card_fw, drv_fw, NULL, 5173 &already); 5174 if (rc == ERESTART) 5175 goto restart; 5176 if (rc != 0) 5177 goto done; 5178 5179 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 5180 if (rc < 0 || state == DEV_STATE_ERR) { 5181 rc = -rc; 5182 device_printf(sc->dev, 5183 "failed to connect to the firmware: %d, %d. " 5184 "PCIE_FW 0x%08x\n", rc, state, t4_read_reg(sc, A_PCIE_FW)); 5185 #if 0 5186 if (install_kld_firmware(sc, (struct fw_h *)card_fw, drv_fw, 5187 "not responding properly to HELLO", &already) == ERESTART) 5188 goto restart; 5189 #endif 5190 goto done; 5191 } 5192 MPASS(be32toh(card_fw->flags) & FW_HDR_FLAGS_RESET_HALT); 5193 sc->flags |= FW_OK; /* The firmware responded to the FW_HELLO. */ 5194 5195 if (rc == sc->pf) { 5196 sc->flags |= MASTER_PF; 5197 rc = install_kld_firmware(sc, (struct fw_h *)card_fw, drv_fw, 5198 NULL, &already); 5199 if (rc == ERESTART) 5200 rc = 0; 5201 else if (rc != 0) 5202 goto done; 5203 } else if (state == DEV_STATE_UNINIT) { 5204 /* 5205 * We didn't get to be the master so we definitely won't be 5206 * configuring the chip. It's a bug if someone else hasn't 5207 * configured it already. 5208 */ 5209 device_printf(sc->dev, "couldn't be master(%d), " 5210 "device not already initialized either(%d). " 5211 "PCIE_FW 0x%08x\n", rc, state, t4_read_reg(sc, A_PCIE_FW)); 5212 rc = EPROTO; 5213 goto done; 5214 } else { 5215 /* 5216 * Some other PF is the master and has configured the chip. 5217 * This is allowed but untested. 5218 */ 5219 device_printf(sc->dev, "PF%d is master, device state %d. " 5220 "PCIE_FW 0x%08x\n", rc, state, t4_read_reg(sc, A_PCIE_FW)); 5221 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", rc); 5222 sc->cfcsum = 0; 5223 rc = 0; 5224 } 5225 done: 5226 if (rc != 0 && sc->flags & FW_OK) { 5227 t4_fw_bye(sc, sc->mbox); 5228 sc->flags &= ~FW_OK; 5229 } 5230 free(card_fw, M_CXGBE); 5231 return (rc); 5232 } 5233 5234 static int 5235 copy_cfg_file_to_card(struct adapter *sc, char *cfg_file, 5236 uint32_t mtype, uint32_t moff, u_int maxlen) 5237 { 5238 struct fw_info *fw_info; 5239 const struct firmware *dcfg, *rcfg = NULL; 5240 const uint32_t *cfdata; 5241 uint32_t cflen, addr; 5242 int rc; 5243 5244 load_fw_module(sc, &dcfg, NULL); 5245 5246 /* Card specific interpretation of "default". */ 5247 if (strncmp(cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 5248 if (pci_get_device(sc->dev) == 0x440a) 5249 snprintf(cfg_file, sizeof(t4_cfg_file), UWIRE_CF); 5250 if (is_fpga(sc)) 5251 snprintf(cfg_file, sizeof(t4_cfg_file), FPGA_CF); 5252 } 5253 5254 if (strncmp(cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 5255 if (dcfg == NULL) { 5256 device_printf(sc->dev, 5257 "KLD with default config is not available.\n"); 5258 rc = ENOENT; 5259 goto done; 5260 } 5261 cfdata = dcfg->data; 5262 cflen = dcfg->datasize & ~3; 5263 } else { 5264 char s[32]; 5265 5266 fw_info = find_fw_info(chip_id(sc)); 5267 if (fw_info == NULL) { 5268 device_printf(sc->dev, 5269 "unable to look up firmware information for chip %d.\n", 5270 chip_id(sc)); 5271 rc = EINVAL; 5272 goto done; 5273 } 5274 snprintf(s, sizeof(s), "%s_%s", fw_info->kld_name, cfg_file); 5275 5276 rcfg = firmware_get(s); 5277 if (rcfg == NULL) { 5278 device_printf(sc->dev, 5279 "unable to load module \"%s\" for configuration " 5280 "profile \"%s\".\n", s, cfg_file); 5281 rc = ENOENT; 5282 goto done; 5283 } 5284 cfdata = rcfg->data; 5285 cflen = rcfg->datasize & ~3; 5286 } 5287 5288 if (cflen > maxlen) { 5289 device_printf(sc->dev, 5290 "config file too long (%d, max allowed is %d).\n", 5291 cflen, maxlen); 5292 rc = EINVAL; 5293 goto done; 5294 } 5295 5296 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 5297 if (rc != 0) { 5298 device_printf(sc->dev, 5299 "%s: addr (%d/0x%x) or len %d is not valid: %d.\n", 5300 __func__, mtype, moff, cflen, rc); 5301 rc = EINVAL; 5302 goto done; 5303 } 5304 write_via_memwin(sc, 2, addr, cfdata, cflen); 5305 done: 5306 if (rcfg != NULL) 5307 firmware_put(rcfg, FIRMWARE_UNLOAD); 5308 unload_fw_module(sc, dcfg, NULL); 5309 return (rc); 5310 } 5311 5312 struct caps_allowed { 5313 uint16_t nbmcaps; 5314 uint16_t linkcaps; 5315 uint16_t switchcaps; 5316 uint16_t nvmecaps; 5317 uint16_t niccaps; 5318 uint16_t toecaps; 5319 uint16_t rdmacaps; 5320 uint16_t cryptocaps; 5321 uint16_t iscsicaps; 5322 uint16_t fcoecaps; 5323 }; 5324 5325 #define FW_PARAM_DEV(param) \ 5326 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 5327 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 5328 #define FW_PARAM_PFVF(param) \ 5329 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 5330 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 5331 5332 /* 5333 * Provide a configuration profile to the firmware and have it initialize the 5334 * chip accordingly. This may involve uploading a configuration file to the 5335 * card. 5336 */ 5337 static int 5338 apply_cfg_and_initialize(struct adapter *sc, char *cfg_file, 5339 const struct caps_allowed *caps_allowed) 5340 { 5341 int rc; 5342 struct fw_caps_config_cmd caps; 5343 uint32_t mtype, moff, finicsum, cfcsum, param, val; 5344 unsigned int maxlen = 0; 5345 const int cfg_addr = t4_flash_cfg_addr(sc, &maxlen); 5346 5347 rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST); 5348 if (rc != 0) { 5349 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 5350 return (rc); 5351 } 5352 5353 bzero(&caps, sizeof(caps)); 5354 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 5355 F_FW_CMD_REQUEST | F_FW_CMD_READ); 5356 if (strncmp(cfg_file, BUILTIN_CF, sizeof(t4_cfg_file)) == 0) { 5357 mtype = 0; 5358 moff = 0; 5359 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 5360 } else if (strncmp(cfg_file, FLASH_CF, sizeof(t4_cfg_file)) == 0) { 5361 mtype = FW_MEMTYPE_FLASH; 5362 moff = cfg_addr; 5363 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 5364 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 5365 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | 5366 FW_LEN16(caps)); 5367 } else { 5368 /* 5369 * Ask the firmware where it wants us to upload the config file. 5370 */ 5371 param = FW_PARAM_DEV(CF); 5372 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 5373 if (rc != 0) { 5374 /* No support for config file? Shouldn't happen. */ 5375 device_printf(sc->dev, 5376 "failed to query config file location: %d.\n", rc); 5377 goto done; 5378 } 5379 mtype = G_FW_PARAMS_PARAM_Y(val); 5380 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 5381 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 5382 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 5383 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | 5384 FW_LEN16(caps)); 5385 5386 rc = copy_cfg_file_to_card(sc, cfg_file, mtype, moff, maxlen); 5387 if (rc != 0) { 5388 device_printf(sc->dev, 5389 "failed to upload config file to card: %d.\n", rc); 5390 goto done; 5391 } 5392 } 5393 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 5394 if (rc != 0) { 5395 device_printf(sc->dev, "failed to pre-process config file: %d " 5396 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 5397 goto done; 5398 } 5399 5400 finicsum = be32toh(caps.finicsum); 5401 cfcsum = be32toh(caps.cfcsum); /* actual */ 5402 if (finicsum != cfcsum) { 5403 device_printf(sc->dev, 5404 "WARNING: config file checksum mismatch: %08x %08x\n", 5405 finicsum, cfcsum); 5406 } 5407 sc->cfcsum = cfcsum; 5408 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", cfg_file); 5409 5410 /* 5411 * Let the firmware know what features will (not) be used so it can tune 5412 * things accordingly. 5413 */ 5414 #define LIMIT_CAPS(x) do { \ 5415 caps.x##caps &= htobe16(caps_allowed->x##caps); \ 5416 } while (0) 5417 LIMIT_CAPS(nbm); 5418 LIMIT_CAPS(link); 5419 LIMIT_CAPS(switch); 5420 LIMIT_CAPS(nvme); 5421 LIMIT_CAPS(nic); 5422 LIMIT_CAPS(toe); 5423 LIMIT_CAPS(rdma); 5424 LIMIT_CAPS(crypto); 5425 LIMIT_CAPS(iscsi); 5426 LIMIT_CAPS(fcoe); 5427 #undef LIMIT_CAPS 5428 if (caps.niccaps & htobe16(FW_CAPS_CONFIG_NIC_HASHFILTER)) { 5429 /* 5430 * TOE and hashfilters are mutually exclusive. It is a config 5431 * file or firmware bug if both are reported as available. Try 5432 * to cope with the situation in non-debug builds by disabling 5433 * TOE. 5434 */ 5435 MPASS(caps.toecaps == 0); 5436 5437 caps.toecaps = 0; 5438 caps.rdmacaps = 0; 5439 caps.iscsicaps = 0; 5440 caps.nvmecaps = 0; 5441 } 5442 5443 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 5444 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 5445 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 5446 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 5447 if (rc != 0) { 5448 device_printf(sc->dev, 5449 "failed to process config file: %d.\n", rc); 5450 goto done; 5451 } 5452 5453 t4_tweak_chip_settings(sc); 5454 set_params__pre_init(sc); 5455 5456 /* get basic stuff going */ 5457 rc = -t4_fw_initialize(sc, sc->mbox); 5458 if (rc != 0) { 5459 device_printf(sc->dev, "fw_initialize failed: %d.\n", rc); 5460 goto done; 5461 } 5462 done: 5463 return (rc); 5464 } 5465 5466 /* 5467 * Partition chip resources for use between various PFs, VFs, etc. 5468 */ 5469 static int 5470 partition_resources(struct adapter *sc) 5471 { 5472 char cfg_file[sizeof(t4_cfg_file)]; 5473 struct caps_allowed caps_allowed; 5474 int rc; 5475 bool fallback; 5476 5477 /* Only the master driver gets to configure the chip resources. */ 5478 MPASS(sc->flags & MASTER_PF); 5479 5480 #define COPY_CAPS(x) do { \ 5481 caps_allowed.x##caps = t4_##x##caps_allowed; \ 5482 } while (0) 5483 bzero(&caps_allowed, sizeof(caps_allowed)); 5484 COPY_CAPS(nbm); 5485 COPY_CAPS(link); 5486 COPY_CAPS(switch); 5487 COPY_CAPS(nvme); 5488 COPY_CAPS(nic); 5489 COPY_CAPS(toe); 5490 COPY_CAPS(rdma); 5491 COPY_CAPS(crypto); 5492 COPY_CAPS(iscsi); 5493 COPY_CAPS(fcoe); 5494 fallback = sc->debug_flags & DF_DISABLE_CFG_RETRY ? false : true; 5495 snprintf(cfg_file, sizeof(cfg_file), "%s", t4_cfg_file); 5496 retry: 5497 rc = apply_cfg_and_initialize(sc, cfg_file, &caps_allowed); 5498 if (rc != 0 && fallback) { 5499 dump_devlog(sc); 5500 device_printf(sc->dev, 5501 "failed (%d) to configure card with \"%s\" profile, " 5502 "will fall back to a basic configuration and retry.\n", 5503 rc, cfg_file); 5504 snprintf(cfg_file, sizeof(cfg_file), "%s", BUILTIN_CF); 5505 bzero(&caps_allowed, sizeof(caps_allowed)); 5506 COPY_CAPS(switch); 5507 caps_allowed.niccaps = FW_CAPS_CONFIG_NIC; 5508 fallback = false; 5509 goto retry; 5510 } 5511 #undef COPY_CAPS 5512 return (rc); 5513 } 5514 5515 /* 5516 * Retrieve parameters that are needed (or nice to have) very early. 5517 */ 5518 static int 5519 get_params__pre_init(struct adapter *sc) 5520 { 5521 int rc; 5522 uint32_t param[2], val[2]; 5523 5524 t4_get_version_info(sc); 5525 5526 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 5527 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 5528 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 5529 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 5530 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 5531 5532 snprintf(sc->bs_version, sizeof(sc->bs_version), "%u.%u.%u.%u", 5533 G_FW_HDR_FW_VER_MAJOR(sc->params.bs_vers), 5534 G_FW_HDR_FW_VER_MINOR(sc->params.bs_vers), 5535 G_FW_HDR_FW_VER_MICRO(sc->params.bs_vers), 5536 G_FW_HDR_FW_VER_BUILD(sc->params.bs_vers)); 5537 5538 snprintf(sc->tp_version, sizeof(sc->tp_version), "%u.%u.%u.%u", 5539 G_FW_HDR_FW_VER_MAJOR(sc->params.tp_vers), 5540 G_FW_HDR_FW_VER_MINOR(sc->params.tp_vers), 5541 G_FW_HDR_FW_VER_MICRO(sc->params.tp_vers), 5542 G_FW_HDR_FW_VER_BUILD(sc->params.tp_vers)); 5543 5544 snprintf(sc->er_version, sizeof(sc->er_version), "%u.%u.%u.%u", 5545 G_FW_HDR_FW_VER_MAJOR(sc->params.er_vers), 5546 G_FW_HDR_FW_VER_MINOR(sc->params.er_vers), 5547 G_FW_HDR_FW_VER_MICRO(sc->params.er_vers), 5548 G_FW_HDR_FW_VER_BUILD(sc->params.er_vers)); 5549 5550 param[0] = FW_PARAM_DEV(PORTVEC); 5551 param[1] = FW_PARAM_DEV(CCLK); 5552 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 5553 if (rc != 0) { 5554 device_printf(sc->dev, 5555 "failed to query parameters (pre_init): %d.\n", rc); 5556 return (rc); 5557 } 5558 5559 sc->params.portvec = val[0]; 5560 sc->params.nports = bitcount32(val[0]); 5561 sc->params.vpd.cclk = val[1]; 5562 5563 /* Read device log parameters. */ 5564 rc = -t4_init_devlog_ncores_params(sc, 1); 5565 if (rc == 0) 5566 fixup_devlog_ncores_params(sc); 5567 else { 5568 device_printf(sc->dev, 5569 "failed to get devlog parameters: %d.\n", rc); 5570 rc = 0; /* devlog isn't critical for device operation */ 5571 } 5572 5573 return (rc); 5574 } 5575 5576 /* 5577 * Any params that need to be set before FW_INITIALIZE. 5578 */ 5579 static int 5580 set_params__pre_init(struct adapter *sc) 5581 { 5582 int rc = 0; 5583 uint32_t param, val; 5584 5585 if (chip_id(sc) >= CHELSIO_T6) { 5586 param = FW_PARAM_DEV(HPFILTER_REGION_SUPPORT); 5587 val = 1; 5588 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 5589 /* firmwares < 1.20.1.0 do not have this param. */ 5590 if (rc == FW_EINVAL && 5591 sc->params.fw_vers < FW_VERSION32(1, 20, 1, 0)) { 5592 rc = 0; 5593 } 5594 if (rc != 0) { 5595 device_printf(sc->dev, 5596 "failed to enable high priority filters :%d.\n", 5597 rc); 5598 } 5599 5600 param = FW_PARAM_DEV(PPOD_EDRAM); 5601 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 5602 if (rc == 0 && val == 1) { 5603 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, 5604 &val); 5605 if (rc != 0) { 5606 device_printf(sc->dev, 5607 "failed to set PPOD_EDRAM: %d.\n", rc); 5608 } 5609 } 5610 } 5611 5612 /* Enable opaque VIIDs with firmwares that support it. */ 5613 param = FW_PARAM_DEV(OPAQUE_VIID_SMT_EXTN); 5614 val = 1; 5615 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 5616 if (rc == 0 && val == 1) 5617 sc->params.viid_smt_extn_support = true; 5618 else 5619 sc->params.viid_smt_extn_support = false; 5620 5621 return (rc); 5622 } 5623 5624 /* 5625 * Retrieve various parameters that are of interest to the driver. The device 5626 * has been initialized by the firmware at this point. 5627 */ 5628 static int 5629 get_params__post_init(struct adapter *sc) 5630 { 5631 int rc; 5632 uint32_t param[7], val[7]; 5633 struct fw_caps_config_cmd caps; 5634 5635 param[0] = FW_PARAM_PFVF(IQFLINT_START); 5636 param[1] = FW_PARAM_PFVF(EQ_START); 5637 param[2] = FW_PARAM_PFVF(FILTER_START); 5638 param[3] = FW_PARAM_PFVF(FILTER_END); 5639 param[4] = FW_PARAM_PFVF(L2T_START); 5640 param[5] = FW_PARAM_PFVF(L2T_END); 5641 param[6] = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 5642 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 5643 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_VDD); 5644 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 7, param, val); 5645 if (rc != 0) { 5646 device_printf(sc->dev, 5647 "failed to query parameters (post_init): %d.\n", rc); 5648 return (rc); 5649 } 5650 5651 sc->sge.iq_start = val[0]; 5652 sc->sge.eq_start = val[1]; 5653 if ((int)val[3] > (int)val[2]) { 5654 sc->tids.ftid_base = val[2]; 5655 sc->tids.ftid_end = val[3]; 5656 sc->tids.nftids = val[3] - val[2] + 1; 5657 } 5658 sc->vres.l2t.start = val[4]; 5659 sc->vres.l2t.size = val[5] - val[4] + 1; 5660 /* val[5] is the last hwidx and it must not collide with F_SYNC_WR */ 5661 if (sc->vres.l2t.size > 0) 5662 MPASS(fls(val[5]) <= S_SYNC_WR); 5663 sc->params.core_vdd = val[6]; 5664 5665 param[0] = FW_PARAM_PFVF(IQFLINT_END); 5666 param[1] = FW_PARAM_PFVF(EQ_END); 5667 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 5668 if (rc != 0) { 5669 device_printf(sc->dev, 5670 "failed to query parameters (post_init2): %d.\n", rc); 5671 return (rc); 5672 } 5673 MPASS((int)val[0] >= sc->sge.iq_start); 5674 sc->sge.iqmap_sz = val[0] - sc->sge.iq_start + 1; 5675 MPASS((int)val[1] >= sc->sge.eq_start); 5676 sc->sge.eqmap_sz = val[1] - sc->sge.eq_start + 1; 5677 5678 if (chip_id(sc) >= CHELSIO_T6) { 5679 5680 sc->tids.tid_base = t4_read_reg(sc, 5681 A_LE_DB_ACTIVE_TABLE_START_INDEX); 5682 5683 param[0] = FW_PARAM_PFVF(HPFILTER_START); 5684 param[1] = FW_PARAM_PFVF(HPFILTER_END); 5685 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 5686 if (rc != 0) { 5687 device_printf(sc->dev, 5688 "failed to query hpfilter parameters: %d.\n", rc); 5689 return (rc); 5690 } 5691 if ((int)val[1] > (int)val[0]) { 5692 sc->tids.hpftid_base = val[0]; 5693 sc->tids.hpftid_end = val[1]; 5694 sc->tids.nhpftids = val[1] - val[0] + 1; 5695 5696 /* 5697 * These should go off if the layout changes and the 5698 * driver needs to catch up. 5699 */ 5700 MPASS(sc->tids.hpftid_base == 0); 5701 MPASS(sc->tids.tid_base == sc->tids.nhpftids); 5702 } 5703 5704 param[0] = FW_PARAM_PFVF(RAWF_START); 5705 param[1] = FW_PARAM_PFVF(RAWF_END); 5706 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 5707 if (rc != 0) { 5708 device_printf(sc->dev, 5709 "failed to query rawf parameters: %d.\n", rc); 5710 return (rc); 5711 } 5712 if ((int)val[1] > (int)val[0]) { 5713 sc->rawf_base = val[0]; 5714 sc->nrawf = val[1] - val[0] + 1; 5715 } 5716 } 5717 5718 if (sc->params.ncores > 1) { 5719 param[0] = FW_PARAM_DEV(TID_QID_SEL_MASK); 5720 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val); 5721 sc->params.tid_qid_sel_mask = rc == 0 ? val[0] : 0; 5722 } 5723 5724 /* 5725 * The parameters that follow may not be available on all firmwares. We 5726 * query them individually rather than in a compound query because old 5727 * firmwares fail the entire query if an unknown parameter is queried. 5728 */ 5729 5730 /* 5731 * MPS buffer group configuration. 5732 */ 5733 param[0] = FW_PARAM_DEV(MPSBGMAP); 5734 val[0] = 0; 5735 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val); 5736 if (rc == 0) 5737 sc->params.mps_bg_map = val[0]; 5738 else 5739 sc->params.mps_bg_map = UINT32_MAX; /* Not a legal value. */ 5740 5741 param[0] = FW_PARAM_DEV(TPCHMAP); 5742 val[0] = 0; 5743 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val); 5744 if (rc == 0) 5745 sc->params.tp_ch_map = val[0]; 5746 else 5747 sc->params.tp_ch_map = UINT32_MAX; /* Not a legal value. */ 5748 5749 param[0] = FW_PARAM_DEV(TX_TPCHMAP); 5750 val[0] = 0; 5751 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val); 5752 if (rc == 0) 5753 sc->params.tx_tp_ch_map = val[0]; 5754 else 5755 sc->params.tx_tp_ch_map = UINT32_MAX; /* Not a legal value. */ 5756 5757 /* 5758 * Determine whether the firmware supports the filter2 work request. 5759 */ 5760 param[0] = FW_PARAM_DEV(FILTER2_WR); 5761 val[0] = 0; 5762 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val); 5763 if (rc == 0) 5764 sc->params.filter2_wr_support = val[0] != 0; 5765 else 5766 sc->params.filter2_wr_support = 0; 5767 5768 /* 5769 * Find out whether we're allowed to use the ULPTX MEMWRITE DSGL. 5770 */ 5771 param[0] = FW_PARAM_DEV(ULPTX_MEMWRITE_DSGL); 5772 val[0] = 0; 5773 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val); 5774 if (rc == 0) 5775 sc->params.ulptx_memwrite_dsgl = val[0] != 0; 5776 else 5777 sc->params.ulptx_memwrite_dsgl = false; 5778 5779 /* FW_RI_FR_NSMR_TPTE_WR support */ 5780 param[0] = FW_PARAM_DEV(RI_FR_NSMR_TPTE_WR); 5781 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val); 5782 if (rc == 0) 5783 sc->params.fr_nsmr_tpte_wr_support = val[0] != 0; 5784 else 5785 sc->params.fr_nsmr_tpte_wr_support = false; 5786 5787 /* Support for 512 SGL entries per FR MR. */ 5788 param[0] = FW_PARAM_DEV(DEV_512SGL_MR); 5789 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val); 5790 if (rc == 0) 5791 sc->params.dev_512sgl_mr = val[0] != 0; 5792 else 5793 sc->params.dev_512sgl_mr = false; 5794 5795 param[0] = FW_PARAM_PFVF(MAX_PKTS_PER_ETH_TX_PKTS_WR); 5796 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val); 5797 if (rc == 0) 5798 sc->params.max_pkts_per_eth_tx_pkts_wr = val[0]; 5799 else 5800 sc->params.max_pkts_per_eth_tx_pkts_wr = 15; 5801 5802 param[0] = FW_PARAM_DEV(NUM_TM_CLASS); 5803 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val); 5804 if (rc == 0) { 5805 MPASS(val[0] > 0 && val[0] < 256); /* nsched_cls is 8b */ 5806 sc->params.nsched_cls = val[0]; 5807 } else 5808 sc->params.nsched_cls = sc->chip_params->nsched_cls; 5809 5810 /* get capabilites */ 5811 bzero(&caps, sizeof(caps)); 5812 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 5813 F_FW_CMD_REQUEST | F_FW_CMD_READ); 5814 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 5815 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 5816 if (rc != 0) { 5817 device_printf(sc->dev, 5818 "failed to get card capabilities: %d.\n", rc); 5819 return (rc); 5820 } 5821 5822 #define READ_CAPS(x) do { \ 5823 sc->x = htobe16(caps.x); \ 5824 } while (0) 5825 READ_CAPS(nbmcaps); 5826 READ_CAPS(linkcaps); 5827 READ_CAPS(switchcaps); 5828 READ_CAPS(nvmecaps); 5829 READ_CAPS(niccaps); 5830 READ_CAPS(toecaps); 5831 READ_CAPS(rdmacaps); 5832 READ_CAPS(cryptocaps); 5833 READ_CAPS(iscsicaps); 5834 READ_CAPS(fcoecaps); 5835 5836 if (sc->niccaps & FW_CAPS_CONFIG_NIC_HASHFILTER) { 5837 MPASS(chip_id(sc) > CHELSIO_T4); 5838 MPASS(sc->toecaps == 0); 5839 sc->toecaps = 0; 5840 5841 param[0] = FW_PARAM_DEV(NTID); 5842 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val); 5843 if (rc != 0) { 5844 device_printf(sc->dev, 5845 "failed to query HASHFILTER parameters: %d.\n", rc); 5846 return (rc); 5847 } 5848 sc->tids.ntids = val[0]; 5849 if (sc->params.fw_vers < FW_VERSION32(1, 20, 5, 0)) { 5850 MPASS(sc->tids.ntids >= sc->tids.nhpftids); 5851 sc->tids.ntids -= sc->tids.nhpftids; 5852 } 5853 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 5854 sc->params.hash_filter = 1; 5855 } 5856 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { 5857 param[0] = FW_PARAM_PFVF(ETHOFLD_START); 5858 param[1] = FW_PARAM_PFVF(ETHOFLD_END); 5859 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 5860 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); 5861 if (rc != 0) { 5862 device_printf(sc->dev, 5863 "failed to query NIC parameters: %d.\n", rc); 5864 return (rc); 5865 } 5866 if ((int)val[1] > (int)val[0]) { 5867 sc->tids.etid_base = val[0]; 5868 sc->tids.etid_end = val[1]; 5869 sc->tids.netids = val[1] - val[0] + 1; 5870 sc->params.eo_wr_cred = val[2]; 5871 sc->params.ethoffload = 1; 5872 } 5873 } 5874 if (sc->toecaps) { 5875 /* query offload-related parameters */ 5876 param[0] = FW_PARAM_DEV(NTID); 5877 param[1] = FW_PARAM_PFVF(SERVER_START); 5878 param[2] = FW_PARAM_PFVF(SERVER_END); 5879 param[3] = FW_PARAM_PFVF(TDDP_START); 5880 param[4] = FW_PARAM_PFVF(TDDP_END); 5881 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 5882 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 5883 if (rc != 0) { 5884 device_printf(sc->dev, 5885 "failed to query TOE parameters: %d.\n", rc); 5886 return (rc); 5887 } 5888 sc->tids.ntids = val[0]; 5889 if (sc->params.fw_vers < FW_VERSION32(1, 20, 5, 0)) { 5890 MPASS(sc->tids.ntids >= sc->tids.nhpftids); 5891 sc->tids.ntids -= sc->tids.nhpftids; 5892 } 5893 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 5894 if ((int)val[2] > (int)val[1]) { 5895 sc->tids.stid_base = val[1]; 5896 sc->tids.nstids = val[2] - val[1] + 1; 5897 } 5898 sc->vres.ddp.start = val[3]; 5899 sc->vres.ddp.size = val[4] - val[3] + 1; 5900 sc->params.ofldq_wr_cred = val[5]; 5901 sc->params.offload = 1; 5902 } else { 5903 /* 5904 * The firmware attempts memfree TOE configuration for -SO cards 5905 * and will report toecaps=0 if it runs out of resources (this 5906 * depends on the config file). It may not report 0 for other 5907 * capabilities dependent on the TOE in this case. Set them to 5908 * 0 here so that the driver doesn't bother tracking resources 5909 * that will never be used. 5910 */ 5911 sc->iscsicaps = 0; 5912 sc->nvmecaps = 0; 5913 sc->rdmacaps = 0; 5914 } 5915 if (sc->nvmecaps || sc->rdmacaps) { 5916 param[0] = FW_PARAM_PFVF(STAG_START); 5917 param[1] = FW_PARAM_PFVF(STAG_END); 5918 param[2] = FW_PARAM_PFVF(PBL_START); 5919 param[3] = FW_PARAM_PFVF(PBL_END); 5920 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 4, param, val); 5921 if (rc != 0) { 5922 device_printf(sc->dev, 5923 "failed to query NVMe/RDMA parameters: %d.\n", rc); 5924 return (rc); 5925 } 5926 sc->vres.stag.start = val[0]; 5927 sc->vres.stag.size = val[1] - val[0] + 1; 5928 sc->vres.pbl.start = val[2]; 5929 sc->vres.pbl.size = val[3] - val[2] + 1; 5930 } 5931 if (sc->rdmacaps) { 5932 param[0] = FW_PARAM_PFVF(RQ_START); 5933 param[1] = FW_PARAM_PFVF(RQ_END); 5934 param[2] = FW_PARAM_PFVF(SQRQ_START); 5935 param[3] = FW_PARAM_PFVF(SQRQ_END); 5936 param[4] = FW_PARAM_PFVF(CQ_START); 5937 param[5] = FW_PARAM_PFVF(CQ_END); 5938 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 5939 if (rc != 0) { 5940 device_printf(sc->dev, 5941 "failed to query RDMA parameters(1): %d.\n", rc); 5942 return (rc); 5943 } 5944 sc->vres.rq.start = val[0]; 5945 sc->vres.rq.size = val[1] - val[0] + 1; 5946 sc->vres.qp.start = val[2]; 5947 sc->vres.qp.size = val[3] - val[2] + 1; 5948 sc->vres.cq.start = val[4]; 5949 sc->vres.cq.size = val[5] - val[4] + 1; 5950 5951 param[0] = FW_PARAM_PFVF(OCQ_START); 5952 param[1] = FW_PARAM_PFVF(OCQ_END); 5953 param[2] = FW_PARAM_PFVF(SRQ_START); 5954 param[3] = FW_PARAM_PFVF(SRQ_END); 5955 param[4] = FW_PARAM_DEV(MAXORDIRD_QP); 5956 param[5] = FW_PARAM_DEV(MAXIRD_ADAPTER); 5957 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 5958 if (rc != 0) { 5959 device_printf(sc->dev, 5960 "failed to query RDMA parameters(2): %d.\n", rc); 5961 return (rc); 5962 } 5963 sc->vres.ocq.start = val[0]; 5964 sc->vres.ocq.size = val[1] - val[0] + 1; 5965 sc->vres.srq.start = val[2]; 5966 sc->vres.srq.size = val[3] - val[2] + 1; 5967 sc->params.max_ordird_qp = val[4]; 5968 sc->params.max_ird_adapter = val[5]; 5969 } 5970 if (sc->iscsicaps) { 5971 param[0] = FW_PARAM_PFVF(ISCSI_START); 5972 param[1] = FW_PARAM_PFVF(ISCSI_END); 5973 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 5974 if (rc != 0) { 5975 device_printf(sc->dev, 5976 "failed to query iSCSI parameters: %d.\n", rc); 5977 return (rc); 5978 } 5979 sc->vres.iscsi.start = val[0]; 5980 sc->vres.iscsi.size = val[1] - val[0] + 1; 5981 } 5982 if (sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS) { 5983 param[0] = FW_PARAM_PFVF(TLS_START); 5984 param[1] = FW_PARAM_PFVF(TLS_END); 5985 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 5986 if (rc != 0) { 5987 device_printf(sc->dev, 5988 "failed to query TLS parameters: %d.\n", rc); 5989 return (rc); 5990 } 5991 sc->vres.key.start = val[0]; 5992 sc->vres.key.size = val[1] - val[0] + 1; 5993 } 5994 5995 /* 5996 * We've got the params we wanted to query directly from the firmware. 5997 * Grab some others via other means. 5998 */ 5999 t4_init_sge_params(sc); 6000 t4_init_tp_params(sc); 6001 t4_read_mtu_tbl(sc, sc->params.mtus, NULL); 6002 t4_load_mtus(sc, sc->params.mtus, sc->params.a_wnd, sc->params.b_wnd); 6003 6004 rc = t4_verify_chip_settings(sc); 6005 if (rc != 0) 6006 return (rc); 6007 t4_init_rx_buf_info(sc); 6008 6009 return (rc); 6010 } 6011 6012 #ifdef KERN_TLS 6013 static void 6014 ktls_tick(void *arg) 6015 { 6016 struct adapter *sc; 6017 uint32_t tstamp; 6018 6019 sc = arg; 6020 tstamp = tcp_ts_getticks(); 6021 t4_write_reg(sc, A_TP_SYNC_TIME_HI, tstamp >> 1); 6022 t4_write_reg(sc, A_TP_SYNC_TIME_LO, tstamp << 31); 6023 callout_schedule_sbt(&sc->ktls_tick, SBT_1MS, 0, C_HARDCLOCK); 6024 } 6025 6026 static int 6027 t6_config_kern_tls(struct adapter *sc, bool enable) 6028 { 6029 int rc; 6030 uint32_t param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 6031 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_KTLS_HW) | 6032 V_FW_PARAMS_PARAM_Y(enable ? 1 : 0) | 6033 V_FW_PARAMS_PARAM_Z(FW_PARAMS_PARAM_DEV_KTLS_HW_USER_ENABLE); 6034 6035 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, ¶m); 6036 if (rc != 0) { 6037 CH_ERR(sc, "failed to %s NIC TLS: %d\n", 6038 enable ? "enable" : "disable", rc); 6039 return (rc); 6040 } 6041 6042 if (enable) { 6043 sc->flags |= KERN_TLS_ON; 6044 callout_reset_sbt(&sc->ktls_tick, SBT_1MS, 0, ktls_tick, sc, 6045 C_HARDCLOCK); 6046 } else { 6047 sc->flags &= ~KERN_TLS_ON; 6048 callout_stop(&sc->ktls_tick); 6049 } 6050 6051 return (rc); 6052 } 6053 #endif 6054 6055 static int 6056 set_params__post_init(struct adapter *sc) 6057 { 6058 uint32_t mask, param, val; 6059 #ifdef TCP_OFFLOAD 6060 int i, v, shift; 6061 #endif 6062 6063 /* ask for encapsulated CPLs */ 6064 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 6065 val = 1; 6066 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 6067 6068 /* Enable 32b port caps if the firmware supports it. */ 6069 param = FW_PARAM_PFVF(PORT_CAPS32); 6070 val = 1; 6071 if (t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val) == 0) 6072 sc->params.port_caps32 = 1; 6073 6074 /* Let filter + maskhash steer to a part of the VI's RSS region. */ 6075 val = 1 << (G_MASKSIZE(t4_read_reg(sc, A_TP_RSS_CONFIG_TNL)) - 1); 6076 t4_set_reg_field(sc, A_TP_RSS_CONFIG_TNL, V_MASKFILTER(M_MASKFILTER), 6077 V_MASKFILTER(val - 1)); 6078 6079 mask = F_DROPERRORANY | F_DROPERRORMAC | F_DROPERRORIPVER | 6080 F_DROPERRORFRAG | F_DROPERRORATTACK | F_DROPERRORETHHDRLEN | 6081 F_DROPERRORIPHDRLEN | F_DROPERRORTCPHDRLEN | F_DROPERRORPKTLEN | 6082 F_DROPERRORTCPOPT | F_DROPERRORCSUMIP | F_DROPERRORCSUM; 6083 val = 0; 6084 if (chip_id(sc) < CHELSIO_T6 && t4_attack_filter != 0) { 6085 t4_set_reg_field(sc, A_TP_GLOBAL_CONFIG, F_ATTACKFILTERENABLE, 6086 F_ATTACKFILTERENABLE); 6087 val |= F_DROPERRORATTACK; 6088 } 6089 if (t4_drop_ip_fragments != 0) { 6090 t4_set_reg_field(sc, A_TP_GLOBAL_CONFIG, F_FRAGMENTDROP, 6091 F_FRAGMENTDROP); 6092 val |= F_DROPERRORFRAG; 6093 } 6094 if (t4_drop_pkts_with_l2_errors != 0) 6095 val |= F_DROPERRORMAC | F_DROPERRORETHHDRLEN; 6096 if (t4_drop_pkts_with_l3_errors != 0) { 6097 val |= F_DROPERRORIPVER | F_DROPERRORIPHDRLEN | 6098 F_DROPERRORCSUMIP; 6099 } 6100 if (t4_drop_pkts_with_l4_errors != 0) { 6101 val |= F_DROPERRORTCPHDRLEN | F_DROPERRORPKTLEN | 6102 F_DROPERRORTCPOPT | F_DROPERRORCSUM; 6103 } 6104 t4_set_reg_field(sc, A_TP_ERR_CONFIG, mask, val); 6105 6106 #ifdef TCP_OFFLOAD 6107 /* 6108 * Override the TOE timers with user provided tunables. This is not the 6109 * recommended way to change the timers (the firmware config file is) so 6110 * these tunables are not documented. 6111 * 6112 * All the timer tunables are in microseconds. 6113 */ 6114 if (t4_toe_keepalive_idle != 0) { 6115 v = us_to_tcp_ticks(sc, t4_toe_keepalive_idle); 6116 v &= M_KEEPALIVEIDLE; 6117 t4_set_reg_field(sc, A_TP_KEEP_IDLE, 6118 V_KEEPALIVEIDLE(M_KEEPALIVEIDLE), V_KEEPALIVEIDLE(v)); 6119 } 6120 if (t4_toe_keepalive_interval != 0) { 6121 v = us_to_tcp_ticks(sc, t4_toe_keepalive_interval); 6122 v &= M_KEEPALIVEINTVL; 6123 t4_set_reg_field(sc, A_TP_KEEP_INTVL, 6124 V_KEEPALIVEINTVL(M_KEEPALIVEINTVL), V_KEEPALIVEINTVL(v)); 6125 } 6126 if (t4_toe_keepalive_count != 0) { 6127 v = t4_toe_keepalive_count & M_KEEPALIVEMAXR2; 6128 t4_set_reg_field(sc, A_TP_SHIFT_CNT, 6129 V_KEEPALIVEMAXR1(M_KEEPALIVEMAXR1) | 6130 V_KEEPALIVEMAXR2(M_KEEPALIVEMAXR2), 6131 V_KEEPALIVEMAXR1(1) | V_KEEPALIVEMAXR2(v)); 6132 } 6133 if (t4_toe_rexmt_min != 0) { 6134 v = us_to_tcp_ticks(sc, t4_toe_rexmt_min); 6135 v &= M_RXTMIN; 6136 t4_set_reg_field(sc, A_TP_RXT_MIN, 6137 V_RXTMIN(M_RXTMIN), V_RXTMIN(v)); 6138 } 6139 if (t4_toe_rexmt_max != 0) { 6140 v = us_to_tcp_ticks(sc, t4_toe_rexmt_max); 6141 v &= M_RXTMAX; 6142 t4_set_reg_field(sc, A_TP_RXT_MAX, 6143 V_RXTMAX(M_RXTMAX), V_RXTMAX(v)); 6144 } 6145 if (t4_toe_rexmt_count != 0) { 6146 v = t4_toe_rexmt_count & M_RXTSHIFTMAXR2; 6147 t4_set_reg_field(sc, A_TP_SHIFT_CNT, 6148 V_RXTSHIFTMAXR1(M_RXTSHIFTMAXR1) | 6149 V_RXTSHIFTMAXR2(M_RXTSHIFTMAXR2), 6150 V_RXTSHIFTMAXR1(1) | V_RXTSHIFTMAXR2(v)); 6151 } 6152 for (i = 0; i < nitems(t4_toe_rexmt_backoff); i++) { 6153 if (t4_toe_rexmt_backoff[i] != -1) { 6154 v = t4_toe_rexmt_backoff[i] & M_TIMERBACKOFFINDEX0; 6155 shift = (i & 3) << 3; 6156 t4_set_reg_field(sc, A_TP_TCP_BACKOFF_REG0 + (i & ~3), 6157 M_TIMERBACKOFFINDEX0 << shift, v << shift); 6158 } 6159 } 6160 #endif 6161 6162 /* 6163 * Limit TOE connections to 2 reassembly "islands". This is 6164 * required to permit migrating TOE connections to either 6165 * ULP_MODE_TCPDDP or UPL_MODE_TLS. 6166 */ 6167 t4_tp_wr_bits_indirect(sc, A_TP_FRAG_CONFIG, V_PASSMODE(M_PASSMODE), 6168 V_PASSMODE(2)); 6169 6170 #ifdef KERN_TLS 6171 if (is_ktls(sc)) { 6172 sc->tlst.inline_keys = t4_tls_inline_keys; 6173 if (t4_kern_tls != 0 && is_t6(sc)) { 6174 sc->tlst.combo_wrs = t4_tls_combo_wrs; 6175 t6_config_kern_tls(sc, true); 6176 } else { 6177 sc->tlst.short_records = t4_tls_short_records; 6178 sc->tlst.partial_ghash = t4_tls_partial_ghash; 6179 } 6180 } 6181 #endif 6182 return (0); 6183 } 6184 6185 #undef FW_PARAM_PFVF 6186 #undef FW_PARAM_DEV 6187 6188 static void 6189 t4_set_desc(struct adapter *sc) 6190 { 6191 struct adapter_params *p = &sc->params; 6192 6193 device_set_descf(sc->dev, "Chelsio %s", p->vpd.id); 6194 } 6195 6196 static inline void 6197 ifmedia_add4(struct ifmedia *ifm, int m) 6198 { 6199 6200 ifmedia_add(ifm, m, 0, NULL); 6201 ifmedia_add(ifm, m | IFM_ETH_TXPAUSE, 0, NULL); 6202 ifmedia_add(ifm, m | IFM_ETH_RXPAUSE, 0, NULL); 6203 ifmedia_add(ifm, m | IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE, 0, NULL); 6204 } 6205 6206 /* 6207 * This is the selected media, which is not quite the same as the active media. 6208 * The media line in ifconfig is "media: Ethernet selected (active)" if selected 6209 * and active are not the same, and "media: Ethernet selected" otherwise. 6210 */ 6211 static void 6212 set_current_media(struct port_info *pi) 6213 { 6214 struct link_config *lc; 6215 struct ifmedia *ifm; 6216 int mword; 6217 u_int speed; 6218 6219 PORT_LOCK_ASSERT_OWNED(pi); 6220 6221 /* Leave current media alone if it's already set to IFM_NONE. */ 6222 ifm = &pi->media; 6223 if (ifm->ifm_cur != NULL && 6224 IFM_SUBTYPE(ifm->ifm_cur->ifm_media) == IFM_NONE) 6225 return; 6226 6227 lc = &pi->link_cfg; 6228 if (lc->requested_aneg != AUTONEG_DISABLE && 6229 lc->pcaps & FW_PORT_CAP32_ANEG) { 6230 ifmedia_set(ifm, IFM_ETHER | IFM_AUTO); 6231 return; 6232 } 6233 mword = IFM_ETHER | IFM_FDX; 6234 if (lc->requested_fc & PAUSE_TX) 6235 mword |= IFM_ETH_TXPAUSE; 6236 if (lc->requested_fc & PAUSE_RX) 6237 mword |= IFM_ETH_RXPAUSE; 6238 if (lc->requested_speed == 0) 6239 speed = port_top_speed(pi) * 1000; /* Gbps -> Mbps */ 6240 else 6241 speed = lc->requested_speed; 6242 mword |= port_mword(pi, speed_to_fwcap(speed)); 6243 ifmedia_set(ifm, mword); 6244 } 6245 6246 /* 6247 * Returns true if the ifmedia list for the port cannot change. 6248 */ 6249 static bool 6250 fixed_ifmedia(struct port_info *pi) 6251 { 6252 6253 return (pi->port_type == FW_PORT_TYPE_BT_SGMII || 6254 pi->port_type == FW_PORT_TYPE_BT_XFI || 6255 pi->port_type == FW_PORT_TYPE_BT_XAUI || 6256 pi->port_type == FW_PORT_TYPE_KX4 || 6257 pi->port_type == FW_PORT_TYPE_KX || 6258 pi->port_type == FW_PORT_TYPE_KR || 6259 pi->port_type == FW_PORT_TYPE_BP_AP || 6260 pi->port_type == FW_PORT_TYPE_BP4_AP || 6261 pi->port_type == FW_PORT_TYPE_BP40_BA || 6262 pi->port_type == FW_PORT_TYPE_KR4_100G || 6263 pi->port_type == FW_PORT_TYPE_KR_SFP28 || 6264 pi->port_type == FW_PORT_TYPE_KR_XLAUI); 6265 } 6266 6267 static void 6268 build_medialist(struct port_info *pi) 6269 { 6270 uint32_t ss, speed; 6271 int unknown, mword, bit; 6272 struct link_config *lc; 6273 struct ifmedia *ifm; 6274 6275 PORT_LOCK_ASSERT_OWNED(pi); 6276 6277 if (pi->flags & FIXED_IFMEDIA) 6278 return; 6279 6280 /* 6281 * Rebuild the ifmedia list. 6282 */ 6283 ifm = &pi->media; 6284 ifmedia_removeall(ifm); 6285 lc = &pi->link_cfg; 6286 ss = G_FW_PORT_CAP32_SPEED(lc->pcaps); /* Supported Speeds */ 6287 if (__predict_false(ss == 0)) { /* not supposed to happen. */ 6288 MPASS(ss != 0); 6289 no_media: 6290 MPASS(LIST_EMPTY(&ifm->ifm_list)); 6291 ifmedia_add(ifm, IFM_ETHER | IFM_NONE, 0, NULL); 6292 ifmedia_set(ifm, IFM_ETHER | IFM_NONE); 6293 return; 6294 } 6295 6296 unknown = 0; 6297 for (bit = S_FW_PORT_CAP32_SPEED; bit < fls(ss); bit++) { 6298 speed = 1 << bit; 6299 MPASS(speed & M_FW_PORT_CAP32_SPEED); 6300 if (ss & speed) { 6301 mword = port_mword(pi, speed); 6302 if (mword == IFM_NONE) { 6303 goto no_media; 6304 } else if (mword == IFM_UNKNOWN) 6305 unknown++; 6306 else 6307 ifmedia_add4(ifm, IFM_ETHER | IFM_FDX | mword); 6308 } 6309 } 6310 if (unknown > 0) /* Add one unknown for all unknown media types. */ 6311 ifmedia_add4(ifm, IFM_ETHER | IFM_FDX | IFM_UNKNOWN); 6312 if (lc->pcaps & FW_PORT_CAP32_ANEG) 6313 ifmedia_add(ifm, IFM_ETHER | IFM_AUTO, 0, NULL); 6314 6315 set_current_media(pi); 6316 } 6317 6318 /* 6319 * Initialize the requested fields in the link config based on driver tunables. 6320 */ 6321 static void 6322 init_link_config(struct port_info *pi) 6323 { 6324 struct link_config *lc = &pi->link_cfg; 6325 6326 PORT_LOCK_ASSERT_OWNED(pi); 6327 6328 lc->requested_caps = 0; 6329 lc->requested_speed = 0; 6330 6331 if (t4_autoneg == 0) 6332 lc->requested_aneg = AUTONEG_DISABLE; 6333 else if (t4_autoneg == 1) 6334 lc->requested_aneg = AUTONEG_ENABLE; 6335 else 6336 lc->requested_aneg = AUTONEG_AUTO; 6337 6338 lc->requested_fc = t4_pause_settings & (PAUSE_TX | PAUSE_RX | 6339 PAUSE_AUTONEG); 6340 6341 if (t4_fec & FEC_AUTO) 6342 lc->requested_fec = FEC_AUTO; 6343 else if (t4_fec == 0) 6344 lc->requested_fec = FEC_NONE; 6345 else { 6346 /* -1 is handled by the FEC_AUTO block above and not here. */ 6347 lc->requested_fec = t4_fec & 6348 (FEC_RS | FEC_BASER_RS | FEC_NONE | FEC_MODULE); 6349 if (lc->requested_fec == 0) 6350 lc->requested_fec = FEC_AUTO; 6351 } 6352 if (t4_force_fec < 0) 6353 lc->force_fec = -1; 6354 else if (t4_force_fec > 0) 6355 lc->force_fec = 1; 6356 else 6357 lc->force_fec = 0; 6358 } 6359 6360 /* 6361 * Makes sure that all requested settings comply with what's supported by the 6362 * port. Returns the number of settings that were invalid and had to be fixed. 6363 */ 6364 static int 6365 fixup_link_config(struct port_info *pi) 6366 { 6367 int n = 0; 6368 struct link_config *lc = &pi->link_cfg; 6369 uint32_t fwspeed; 6370 6371 PORT_LOCK_ASSERT_OWNED(pi); 6372 6373 /* Speed (when not autonegotiating) */ 6374 if (lc->requested_speed != 0) { 6375 fwspeed = speed_to_fwcap(lc->requested_speed); 6376 if ((fwspeed & lc->pcaps) == 0) { 6377 n++; 6378 lc->requested_speed = 0; 6379 } 6380 } 6381 6382 /* Link autonegotiation */ 6383 MPASS(lc->requested_aneg == AUTONEG_ENABLE || 6384 lc->requested_aneg == AUTONEG_DISABLE || 6385 lc->requested_aneg == AUTONEG_AUTO); 6386 if (lc->requested_aneg == AUTONEG_ENABLE && 6387 !(lc->pcaps & FW_PORT_CAP32_ANEG)) { 6388 n++; 6389 lc->requested_aneg = AUTONEG_AUTO; 6390 } 6391 6392 /* Flow control */ 6393 MPASS((lc->requested_fc & ~(PAUSE_TX | PAUSE_RX | PAUSE_AUTONEG)) == 0); 6394 if (lc->requested_fc & PAUSE_TX && 6395 !(lc->pcaps & FW_PORT_CAP32_FC_TX)) { 6396 n++; 6397 lc->requested_fc &= ~PAUSE_TX; 6398 } 6399 if (lc->requested_fc & PAUSE_RX && 6400 !(lc->pcaps & FW_PORT_CAP32_FC_RX)) { 6401 n++; 6402 lc->requested_fc &= ~PAUSE_RX; 6403 } 6404 if (!(lc->requested_fc & PAUSE_AUTONEG) && 6405 !(lc->pcaps & FW_PORT_CAP32_FORCE_PAUSE)) { 6406 n++; 6407 lc->requested_fc |= PAUSE_AUTONEG; 6408 } 6409 6410 /* FEC */ 6411 if ((lc->requested_fec & FEC_RS && 6412 !(lc->pcaps & FW_PORT_CAP32_FEC_RS)) || 6413 (lc->requested_fec & FEC_BASER_RS && 6414 !(lc->pcaps & FW_PORT_CAP32_FEC_BASER_RS))) { 6415 n++; 6416 lc->requested_fec = FEC_AUTO; 6417 } 6418 6419 return (n); 6420 } 6421 6422 /* 6423 * Apply the requested L1 settings, which are expected to be valid, to the 6424 * hardware. 6425 */ 6426 static int 6427 apply_link_config(struct port_info *pi) 6428 { 6429 struct adapter *sc = pi->adapter; 6430 struct link_config *lc = &pi->link_cfg; 6431 int rc; 6432 6433 #ifdef INVARIANTS 6434 ASSERT_SYNCHRONIZED_OP(sc); 6435 PORT_LOCK_ASSERT_OWNED(pi); 6436 6437 if (lc->requested_aneg == AUTONEG_ENABLE) 6438 MPASS(lc->pcaps & FW_PORT_CAP32_ANEG); 6439 if (!(lc->requested_fc & PAUSE_AUTONEG)) 6440 MPASS(lc->pcaps & FW_PORT_CAP32_FORCE_PAUSE); 6441 if (lc->requested_fc & PAUSE_TX) 6442 MPASS(lc->pcaps & FW_PORT_CAP32_FC_TX); 6443 if (lc->requested_fc & PAUSE_RX) 6444 MPASS(lc->pcaps & FW_PORT_CAP32_FC_RX); 6445 if (lc->requested_fec & FEC_RS) 6446 MPASS(lc->pcaps & FW_PORT_CAP32_FEC_RS); 6447 if (lc->requested_fec & FEC_BASER_RS) 6448 MPASS(lc->pcaps & FW_PORT_CAP32_FEC_BASER_RS); 6449 #endif 6450 if (!(sc->flags & IS_VF)) { 6451 rc = -t4_link_l1cfg(sc, sc->mbox, pi->hw_port, lc); 6452 if (rc != 0) { 6453 device_printf(pi->dev, "l1cfg failed: %d\n", rc); 6454 return (rc); 6455 } 6456 } 6457 6458 /* 6459 * An L1_CFG will almost always result in a link-change event if the 6460 * link is up, and the driver will refresh the actual fec/fc/etc. when 6461 * the notification is processed. If the link is down then the actual 6462 * settings are meaningless. 6463 * 6464 * This takes care of the case where a change in the L1 settings may not 6465 * result in a notification. 6466 */ 6467 if (lc->link_ok && !(lc->requested_fc & PAUSE_AUTONEG)) 6468 lc->fc = lc->requested_fc & (PAUSE_TX | PAUSE_RX); 6469 6470 return (0); 6471 } 6472 6473 #define FW_MAC_EXACT_CHUNK 7 6474 struct mcaddr_ctx { 6475 if_t ifp; 6476 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 6477 uint64_t hash; 6478 int i; 6479 int del; 6480 int rc; 6481 }; 6482 6483 static u_int 6484 add_maddr(void *arg, struct sockaddr_dl *sdl, u_int cnt) 6485 { 6486 struct mcaddr_ctx *ctx = arg; 6487 struct vi_info *vi = if_getsoftc(ctx->ifp); 6488 struct port_info *pi = vi->pi; 6489 struct adapter *sc = pi->adapter; 6490 6491 if (ctx->rc < 0) 6492 return (0); 6493 6494 ctx->mcaddr[ctx->i] = LLADDR(sdl); 6495 MPASS(ETHER_IS_MULTICAST(ctx->mcaddr[ctx->i])); 6496 ctx->i++; 6497 6498 if (ctx->i == FW_MAC_EXACT_CHUNK) { 6499 ctx->rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid, ctx->del, 6500 ctx->i, ctx->mcaddr, NULL, &ctx->hash, 0); 6501 if (ctx->rc < 0) { 6502 int j; 6503 6504 for (j = 0; j < ctx->i; j++) { 6505 if_printf(ctx->ifp, 6506 "failed to add mc address" 6507 " %02x:%02x:%02x:" 6508 "%02x:%02x:%02x rc=%d\n", 6509 ctx->mcaddr[j][0], ctx->mcaddr[j][1], 6510 ctx->mcaddr[j][2], ctx->mcaddr[j][3], 6511 ctx->mcaddr[j][4], ctx->mcaddr[j][5], 6512 -ctx->rc); 6513 } 6514 return (0); 6515 } 6516 ctx->del = 0; 6517 ctx->i = 0; 6518 } 6519 6520 return (1); 6521 } 6522 6523 /* 6524 * Program the port's XGMAC based on parameters in ifnet. The caller also 6525 * indicates which parameters should be programmed (the rest are left alone). 6526 */ 6527 int 6528 update_mac_settings(if_t ifp, int flags) 6529 { 6530 int rc = 0; 6531 struct vi_info *vi = if_getsoftc(ifp); 6532 struct port_info *pi = vi->pi; 6533 struct adapter *sc = pi->adapter; 6534 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 6535 uint8_t match_all_mac[ETHER_ADDR_LEN] = {0}; 6536 6537 ASSERT_SYNCHRONIZED_OP(sc); 6538 KASSERT(flags, ("%s: not told what to update.", __func__)); 6539 6540 if (flags & XGMAC_MTU) 6541 mtu = if_getmtu(ifp); 6542 6543 if (flags & XGMAC_PROMISC) 6544 promisc = if_getflags(ifp) & IFF_PROMISC ? 1 : 0; 6545 6546 if (flags & XGMAC_ALLMULTI) 6547 allmulti = if_getflags(ifp) & IFF_ALLMULTI ? 1 : 0; 6548 6549 if (flags & XGMAC_VLANEX) 6550 vlanex = if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING ? 1 : 0; 6551 6552 if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) { 6553 rc = -t4_set_rxmode(sc, sc->mbox, vi->viid, mtu, promisc, 6554 allmulti, 1, vlanex, false); 6555 if (rc) { 6556 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, 6557 rc); 6558 return (rc); 6559 } 6560 } 6561 6562 if (flags & XGMAC_UCADDR) { 6563 uint8_t ucaddr[ETHER_ADDR_LEN]; 6564 6565 bcopy(if_getlladdr(ifp), ucaddr, sizeof(ucaddr)); 6566 rc = t4_change_mac(sc, sc->mbox, vi->viid, vi->xact_addr_filt, 6567 ucaddr, true, &vi->smt_idx); 6568 if (rc < 0) { 6569 rc = -rc; 6570 if_printf(ifp, "change_mac failed: %d\n", rc); 6571 return (rc); 6572 } else { 6573 vi->xact_addr_filt = rc; 6574 rc = 0; 6575 } 6576 } 6577 6578 if (flags & XGMAC_MCADDRS) { 6579 struct epoch_tracker et; 6580 struct mcaddr_ctx ctx; 6581 int j; 6582 6583 ctx.ifp = ifp; 6584 ctx.hash = 0; 6585 ctx.i = 0; 6586 ctx.del = 1; 6587 ctx.rc = 0; 6588 /* 6589 * Unlike other drivers, we accumulate list of pointers into 6590 * interface address lists and we need to keep it safe even 6591 * after if_foreach_llmaddr() returns, thus we must enter the 6592 * network epoch. 6593 */ 6594 NET_EPOCH_ENTER(et); 6595 if_foreach_llmaddr(ifp, add_maddr, &ctx); 6596 if (ctx.rc < 0) { 6597 NET_EPOCH_EXIT(et); 6598 rc = -ctx.rc; 6599 return (rc); 6600 } 6601 if (ctx.i > 0) { 6602 rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid, 6603 ctx.del, ctx.i, ctx.mcaddr, NULL, &ctx.hash, 0); 6604 NET_EPOCH_EXIT(et); 6605 if (rc < 0) { 6606 rc = -rc; 6607 for (j = 0; j < ctx.i; j++) { 6608 if_printf(ifp, 6609 "failed to add mcast address" 6610 " %02x:%02x:%02x:" 6611 "%02x:%02x:%02x rc=%d\n", 6612 ctx.mcaddr[j][0], ctx.mcaddr[j][1], 6613 ctx.mcaddr[j][2], ctx.mcaddr[j][3], 6614 ctx.mcaddr[j][4], ctx.mcaddr[j][5], 6615 rc); 6616 } 6617 return (rc); 6618 } 6619 ctx.del = 0; 6620 } else 6621 NET_EPOCH_EXIT(et); 6622 6623 rc = -t4_set_addr_hash(sc, sc->mbox, vi->viid, 0, ctx.hash, 0); 6624 if (rc != 0) 6625 if_printf(ifp, "failed to set mcast address hash: %d\n", 6626 rc); 6627 if (ctx.del == 0) { 6628 /* We clobbered the VXLAN entry if there was one. */ 6629 pi->vxlan_tcam_entry = false; 6630 } 6631 } 6632 6633 if (IS_MAIN_VI(vi) && sc->vxlan_refcount > 0 && 6634 pi->vxlan_tcam_entry == false) { 6635 rc = t4_alloc_raw_mac_filt(sc, vi->viid, match_all_mac, 6636 match_all_mac, sc->rawf_base + pi->port_id, 1, pi->port_id, 6637 true); 6638 if (rc < 0) { 6639 rc = -rc; 6640 if_printf(ifp, "failed to add VXLAN TCAM entry: %d.\n", 6641 rc); 6642 } else { 6643 MPASS(rc == sc->rawf_base + pi->port_id); 6644 rc = 0; 6645 pi->vxlan_tcam_entry = true; 6646 } 6647 } 6648 6649 return (rc); 6650 } 6651 6652 /* 6653 * {begin|end}_synchronized_op must be called from the same thread. 6654 */ 6655 int 6656 begin_synchronized_op(struct adapter *sc, struct vi_info *vi, int flags, 6657 char *wmesg) 6658 { 6659 int rc; 6660 6661 #ifdef WITNESS 6662 /* the caller thinks it's ok to sleep, but is it really? */ 6663 if (flags & SLEEP_OK) 6664 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, __func__); 6665 #endif 6666 ADAPTER_LOCK(sc); 6667 for (;;) { 6668 6669 if (vi && IS_DETACHING(vi)) { 6670 rc = ENXIO; 6671 goto done; 6672 } 6673 6674 if (!IS_BUSY(sc)) { 6675 rc = 0; 6676 break; 6677 } 6678 6679 if (!(flags & SLEEP_OK)) { 6680 rc = EBUSY; 6681 goto done; 6682 } 6683 6684 if (mtx_sleep(&sc->flags, &sc->sc_lock, 6685 flags & INTR_OK ? PCATCH : 0, wmesg, 0)) { 6686 rc = EINTR; 6687 goto done; 6688 } 6689 } 6690 6691 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 6692 SET_BUSY(sc); 6693 #ifdef INVARIANTS 6694 sc->last_op = wmesg; 6695 sc->last_op_thr = curthread; 6696 sc->last_op_flags = flags; 6697 #endif 6698 6699 done: 6700 if (!(flags & HOLD_LOCK) || rc) 6701 ADAPTER_UNLOCK(sc); 6702 6703 return (rc); 6704 } 6705 6706 /* 6707 * Tell if_ioctl and if_init that the VI is going away. This is 6708 * special variant of begin_synchronized_op and must be paired with a 6709 * call to end_vi_detach. 6710 */ 6711 void 6712 begin_vi_detach(struct adapter *sc, struct vi_info *vi) 6713 { 6714 ADAPTER_LOCK(sc); 6715 SET_DETACHING(vi); 6716 wakeup(&sc->flags); 6717 while (IS_BUSY(sc)) 6718 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 6719 SET_BUSY(sc); 6720 #ifdef INVARIANTS 6721 sc->last_op = "t4detach"; 6722 sc->last_op_thr = curthread; 6723 sc->last_op_flags = 0; 6724 #endif 6725 ADAPTER_UNLOCK(sc); 6726 } 6727 6728 void 6729 end_vi_detach(struct adapter *sc, struct vi_info *vi) 6730 { 6731 ADAPTER_LOCK(sc); 6732 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 6733 CLR_BUSY(sc); 6734 CLR_DETACHING(vi); 6735 wakeup(&sc->flags); 6736 ADAPTER_UNLOCK(sc); 6737 } 6738 6739 /* 6740 * {begin|end}_synchronized_op must be called from the same thread. 6741 */ 6742 void 6743 end_synchronized_op(struct adapter *sc, int flags) 6744 { 6745 6746 if (flags & LOCK_HELD) 6747 ADAPTER_LOCK_ASSERT_OWNED(sc); 6748 else 6749 ADAPTER_LOCK(sc); 6750 6751 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 6752 CLR_BUSY(sc); 6753 wakeup(&sc->flags); 6754 ADAPTER_UNLOCK(sc); 6755 } 6756 6757 static int 6758 cxgbe_init_synchronized(struct vi_info *vi) 6759 { 6760 struct port_info *pi = vi->pi; 6761 struct adapter *sc = pi->adapter; 6762 if_t ifp = vi->ifp; 6763 int rc = 0, i; 6764 struct sge_txq *txq; 6765 6766 ASSERT_SYNCHRONIZED_OP(sc); 6767 6768 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) 6769 return (0); /* already running */ 6770 6771 if (!(sc->flags & FULL_INIT_DONE) && ((rc = adapter_init(sc)) != 0)) 6772 return (rc); /* error message displayed already */ 6773 6774 if (!(vi->flags & VI_INIT_DONE) && ((rc = vi_init(vi)) != 0)) 6775 return (rc); /* error message displayed already */ 6776 6777 rc = update_mac_settings(ifp, XGMAC_ALL); 6778 if (rc) 6779 goto done; /* error message displayed already */ 6780 6781 PORT_LOCK(pi); 6782 if (pi->up_vis == 0) { 6783 t4_update_port_info(pi); 6784 fixup_link_config(pi); 6785 build_medialist(pi); 6786 apply_link_config(pi); 6787 } 6788 6789 rc = -t4_enable_vi(sc, sc->mbox, vi->viid, true, true); 6790 if (rc != 0) { 6791 if_printf(ifp, "enable_vi failed: %d\n", rc); 6792 PORT_UNLOCK(pi); 6793 goto done; 6794 } 6795 6796 /* 6797 * Can't fail from this point onwards. Review cxgbe_uninit_synchronized 6798 * if this changes. 6799 */ 6800 6801 for_each_txq(vi, i, txq) { 6802 TXQ_LOCK(txq); 6803 txq->eq.flags |= EQ_ENABLED; 6804 TXQ_UNLOCK(txq); 6805 } 6806 6807 /* 6808 * The first iq of the first port to come up is used for tracing. 6809 */ 6810 if (sc->traceq < 0 && IS_MAIN_VI(vi)) { 6811 sc->traceq = sc->sge.rxq[vi->first_rxq].iq.abs_id; 6812 t4_set_trace_rss_control(sc, pi->tx_chan, sc->traceq); 6813 pi->flags |= HAS_TRACEQ; 6814 } 6815 6816 /* all ok */ 6817 pi->up_vis++; 6818 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, 0); 6819 if (pi->link_cfg.link_ok) 6820 t4_os_link_changed(pi); 6821 PORT_UNLOCK(pi); 6822 6823 mtx_lock(&vi->tick_mtx); 6824 if (vi->pi->nvi > 1 || sc->flags & IS_VF) 6825 callout_reset(&vi->tick, hz, vi_tick, vi); 6826 else 6827 callout_reset(&vi->tick, hz, cxgbe_tick, vi); 6828 mtx_unlock(&vi->tick_mtx); 6829 done: 6830 if (rc != 0) 6831 cxgbe_uninit_synchronized(vi); 6832 6833 return (rc); 6834 } 6835 6836 /* 6837 * Idempotent. 6838 */ 6839 static int 6840 cxgbe_uninit_synchronized(struct vi_info *vi) 6841 { 6842 struct port_info *pi = vi->pi; 6843 struct adapter *sc = pi->adapter; 6844 if_t ifp = vi->ifp; 6845 int rc, i; 6846 struct sge_txq *txq; 6847 6848 ASSERT_SYNCHRONIZED_OP(sc); 6849 6850 if (!(vi->flags & VI_INIT_DONE)) { 6851 if (__predict_false(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) { 6852 KASSERT(0, ("uninited VI is running")); 6853 if_printf(ifp, "uninited VI with running ifnet. " 6854 "vi->flags 0x%016lx, if_flags 0x%08x, " 6855 "if_drv_flags 0x%08x\n", vi->flags, if_getflags(ifp), 6856 if_getdrvflags(ifp)); 6857 } 6858 return (0); 6859 } 6860 6861 /* 6862 * Disable the VI so that all its data in either direction is discarded 6863 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 6864 * tick) intact as the TP can deliver negative advice or data that it's 6865 * holding in its RAM (for an offloaded connection) even after the VI is 6866 * disabled. 6867 */ 6868 rc = -t4_enable_vi(sc, sc->mbox, vi->viid, false, false); 6869 if (rc) { 6870 if_printf(ifp, "disable_vi failed: %d\n", rc); 6871 return (rc); 6872 } 6873 6874 for_each_txq(vi, i, txq) { 6875 TXQ_LOCK(txq); 6876 txq->eq.flags &= ~EQ_ENABLED; 6877 TXQ_UNLOCK(txq); 6878 } 6879 6880 mtx_lock(&vi->tick_mtx); 6881 callout_stop(&vi->tick); 6882 mtx_unlock(&vi->tick_mtx); 6883 6884 PORT_LOCK(pi); 6885 if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) { 6886 PORT_UNLOCK(pi); 6887 return (0); 6888 } 6889 if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING); 6890 pi->up_vis--; 6891 if (pi->up_vis > 0) { 6892 PORT_UNLOCK(pi); 6893 return (0); 6894 } 6895 6896 pi->link_cfg.link_ok = false; 6897 pi->link_cfg.speed = 0; 6898 pi->link_cfg.link_down_rc = 255; 6899 t4_os_link_changed(pi); 6900 PORT_UNLOCK(pi); 6901 6902 return (0); 6903 } 6904 6905 /* 6906 * It is ok for this function to fail midway and return right away. t4_detach 6907 * will walk the entire sc->irq list and clean up whatever is valid. 6908 */ 6909 int 6910 t4_setup_intr_handlers(struct adapter *sc) 6911 { 6912 int rc, rid, p, q, v; 6913 char s[8]; 6914 struct irq *irq; 6915 struct port_info *pi; 6916 struct vi_info *vi; 6917 struct sge *sge = &sc->sge; 6918 struct sge_rxq *rxq; 6919 #ifdef TCP_OFFLOAD 6920 struct sge_ofld_rxq *ofld_rxq; 6921 #endif 6922 #ifdef DEV_NETMAP 6923 struct sge_nm_rxq *nm_rxq; 6924 #endif 6925 #ifdef RSS 6926 int nbuckets = rss_getnumbuckets(); 6927 #endif 6928 6929 /* 6930 * Setup interrupts. 6931 */ 6932 irq = &sc->irq[0]; 6933 rid = sc->intr_type == INTR_INTX ? 0 : 1; 6934 if (forwarding_intr_to_fwq(sc)) 6935 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all")); 6936 6937 /* Multiple interrupts. */ 6938 if (sc->flags & IS_VF) 6939 KASSERT(sc->intr_count >= T4VF_EXTRA_INTR + sc->params.nports, 6940 ("%s: too few intr.", __func__)); 6941 else 6942 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 6943 ("%s: too few intr.", __func__)); 6944 6945 /* The first one is always error intr on PFs */ 6946 if (!(sc->flags & IS_VF)) { 6947 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 6948 if (rc != 0) 6949 return (rc); 6950 irq++; 6951 rid++; 6952 } 6953 6954 /* The second one is always the firmware event queue (first on VFs) */ 6955 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sge->fwq, "evt"); 6956 if (rc != 0) 6957 return (rc); 6958 irq++; 6959 rid++; 6960 6961 for_each_port(sc, p) { 6962 pi = sc->port[p]; 6963 for_each_vi(pi, v, vi) { 6964 vi->first_intr = rid - 1; 6965 6966 if (vi->nnmrxq > 0) { 6967 int n = max(vi->nrxq, vi->nnmrxq); 6968 6969 rxq = &sge->rxq[vi->first_rxq]; 6970 #ifdef DEV_NETMAP 6971 nm_rxq = &sge->nm_rxq[vi->first_nm_rxq]; 6972 #endif 6973 for (q = 0; q < n; q++) { 6974 snprintf(s, sizeof(s), "%x%c%x", p, 6975 'a' + v, q); 6976 if (q < vi->nrxq) 6977 irq->rxq = rxq++; 6978 #ifdef DEV_NETMAP 6979 if (q < vi->nnmrxq) 6980 irq->nm_rxq = nm_rxq++; 6981 6982 if (irq->nm_rxq != NULL && 6983 irq->rxq == NULL) { 6984 /* Netmap rx only */ 6985 rc = t4_alloc_irq(sc, irq, rid, 6986 t4_nm_intr, irq->nm_rxq, s); 6987 } 6988 if (irq->nm_rxq != NULL && 6989 irq->rxq != NULL) { 6990 /* NIC and Netmap rx */ 6991 rc = t4_alloc_irq(sc, irq, rid, 6992 t4_vi_intr, irq, s); 6993 } 6994 #endif 6995 if (irq->rxq != NULL && 6996 irq->nm_rxq == NULL) { 6997 /* NIC rx only */ 6998 rc = t4_alloc_irq(sc, irq, rid, 6999 t4_intr, irq->rxq, s); 7000 } 7001 if (rc != 0) 7002 return (rc); 7003 #ifdef RSS 7004 if (q < vi->nrxq) { 7005 bus_bind_intr(sc->dev, irq->res, 7006 rss_getcpu(q % nbuckets)); 7007 } 7008 #endif 7009 irq++; 7010 rid++; 7011 vi->nintr++; 7012 } 7013 } else { 7014 for_each_rxq(vi, q, rxq) { 7015 snprintf(s, sizeof(s), "%x%c%x", p, 7016 'a' + v, q); 7017 rc = t4_alloc_irq(sc, irq, rid, 7018 t4_intr, rxq, s); 7019 if (rc != 0) 7020 return (rc); 7021 #ifdef RSS 7022 bus_bind_intr(sc->dev, irq->res, 7023 rss_getcpu(q % nbuckets)); 7024 #endif 7025 irq++; 7026 rid++; 7027 vi->nintr++; 7028 } 7029 } 7030 #ifdef TCP_OFFLOAD 7031 for_each_ofld_rxq(vi, q, ofld_rxq) { 7032 snprintf(s, sizeof(s), "%x%c%x", p, 'A' + v, q); 7033 rc = t4_alloc_irq(sc, irq, rid, t4_intr, 7034 ofld_rxq, s); 7035 if (rc != 0) 7036 return (rc); 7037 irq++; 7038 rid++; 7039 vi->nintr++; 7040 } 7041 #endif 7042 } 7043 } 7044 MPASS(irq == &sc->irq[sc->intr_count]); 7045 7046 return (0); 7047 } 7048 7049 static void 7050 write_global_rss_key(struct adapter *sc) 7051 { 7052 int i; 7053 uint32_t raw_rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; 7054 uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; 7055 7056 CTASSERT(RSS_KEYSIZE == 40); 7057 7058 rss_getkey((void *)&raw_rss_key[0]); 7059 for (i = 0; i < nitems(rss_key); i++) { 7060 rss_key[i] = htobe32(raw_rss_key[nitems(rss_key) - 1 - i]); 7061 } 7062 t4_write_rss_key(sc, &rss_key[0], -1, 1); 7063 } 7064 7065 /* 7066 * Idempotent. 7067 */ 7068 static int 7069 adapter_full_init(struct adapter *sc) 7070 { 7071 int rc, i; 7072 7073 ASSERT_SYNCHRONIZED_OP(sc); 7074 7075 /* 7076 * queues that belong to the adapter (not any particular port). 7077 */ 7078 rc = t4_setup_adapter_queues(sc); 7079 if (rc != 0) 7080 return (rc); 7081 7082 MPASS(sc->params.nports <= nitems(sc->tq)); 7083 for (i = 0; i < sc->params.nports; i++) { 7084 if (sc->tq[i] != NULL) 7085 continue; 7086 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 7087 taskqueue_thread_enqueue, &sc->tq[i]); 7088 if (sc->tq[i] == NULL) { 7089 CH_ERR(sc, "failed to allocate task queue %d\n", i); 7090 return (ENOMEM); 7091 } 7092 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 7093 device_get_nameunit(sc->dev), i); 7094 } 7095 7096 if (!(sc->flags & IS_VF)) { 7097 write_global_rss_key(sc); 7098 t4_intr_enable(sc); 7099 } 7100 return (0); 7101 } 7102 7103 int 7104 adapter_init(struct adapter *sc) 7105 { 7106 int rc; 7107 7108 ASSERT_SYNCHRONIZED_OP(sc); 7109 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 7110 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 7111 ("%s: FULL_INIT_DONE already", __func__)); 7112 7113 rc = adapter_full_init(sc); 7114 if (rc != 0) 7115 adapter_full_uninit(sc); 7116 else 7117 sc->flags |= FULL_INIT_DONE; 7118 7119 return (rc); 7120 } 7121 7122 /* 7123 * Idempotent. 7124 */ 7125 static void 7126 adapter_full_uninit(struct adapter *sc) 7127 { 7128 int i; 7129 7130 t4_teardown_adapter_queues(sc); 7131 7132 for (i = 0; i < nitems(sc->tq); i++) { 7133 if (sc->tq[i] == NULL) 7134 continue; 7135 taskqueue_free(sc->tq[i]); 7136 sc->tq[i] = NULL; 7137 } 7138 7139 sc->flags &= ~FULL_INIT_DONE; 7140 } 7141 7142 #define SUPPORTED_RSS_HASHTYPES (RSS_HASHTYPE_RSS_IPV4 | \ 7143 RSS_HASHTYPE_RSS_TCP_IPV4 | RSS_HASHTYPE_RSS_IPV6 | \ 7144 RSS_HASHTYPE_RSS_TCP_IPV6 | RSS_HASHTYPE_RSS_UDP_IPV4 | \ 7145 RSS_HASHTYPE_RSS_UDP_IPV6) 7146 7147 /* Translates kernel hash types to hardware. */ 7148 static int 7149 hashconfig_to_hashen(int hashconfig) 7150 { 7151 int hashen = 0; 7152 7153 if (hashconfig & RSS_HASHTYPE_RSS_IPV4) 7154 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN; 7155 if (hashconfig & RSS_HASHTYPE_RSS_IPV6) 7156 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN; 7157 if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV4) { 7158 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN | 7159 F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN; 7160 } 7161 if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV6) { 7162 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN | 7163 F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN; 7164 } 7165 if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV4) 7166 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN; 7167 if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV6) 7168 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN; 7169 7170 return (hashen); 7171 } 7172 7173 /* Translates hardware hash types to kernel. */ 7174 static int 7175 hashen_to_hashconfig(int hashen) 7176 { 7177 int hashconfig = 0; 7178 7179 if (hashen & F_FW_RSS_VI_CONFIG_CMD_UDPEN) { 7180 /* 7181 * If UDP hashing was enabled it must have been enabled for 7182 * either IPv4 or IPv6 (inclusive or). Enabling UDP without 7183 * enabling any 4-tuple hash is nonsense configuration. 7184 */ 7185 MPASS(hashen & (F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN | 7186 F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)); 7187 7188 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) 7189 hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV4; 7190 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) 7191 hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV6; 7192 } 7193 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) 7194 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV4; 7195 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) 7196 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV6; 7197 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN) 7198 hashconfig |= RSS_HASHTYPE_RSS_IPV4; 7199 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN) 7200 hashconfig |= RSS_HASHTYPE_RSS_IPV6; 7201 7202 return (hashconfig); 7203 } 7204 7205 /* 7206 * Idempotent. 7207 */ 7208 static int 7209 vi_full_init(struct vi_info *vi) 7210 { 7211 struct adapter *sc = vi->adapter; 7212 struct sge_rxq *rxq; 7213 int rc, i, j, extra; 7214 int hashconfig = rss_gethashconfig(); 7215 #ifdef RSS 7216 int nbuckets = rss_getnumbuckets(); 7217 #endif 7218 7219 ASSERT_SYNCHRONIZED_OP(sc); 7220 7221 /* 7222 * Allocate tx/rx/fl queues for this VI. 7223 */ 7224 rc = t4_setup_vi_queues(vi); 7225 if (rc != 0) 7226 return (rc); 7227 7228 /* 7229 * Setup RSS for this VI. Save a copy of the RSS table for later use. 7230 */ 7231 if (vi->nrxq > vi->rss_size) { 7232 CH_ALERT(vi, "nrxq (%d) > hw RSS table size (%d); " 7233 "some queues will never receive traffic.\n", vi->nrxq, 7234 vi->rss_size); 7235 } else if (vi->rss_size % vi->nrxq) { 7236 CH_ALERT(vi, "nrxq (%d), hw RSS table size (%d); " 7237 "expect uneven traffic distribution.\n", vi->nrxq, 7238 vi->rss_size); 7239 } 7240 #ifdef RSS 7241 if (vi->nrxq != nbuckets) { 7242 CH_ALERT(vi, "nrxq (%d) != kernel RSS buckets (%d);" 7243 "performance will be impacted.\n", vi->nrxq, nbuckets); 7244 } 7245 #endif 7246 if (vi->rss == NULL) 7247 vi->rss = malloc(vi->rss_size * sizeof (*vi->rss), M_CXGBE, 7248 M_ZERO | M_WAITOK); 7249 for (i = 0; i < vi->rss_size;) { 7250 #ifdef RSS 7251 j = rss_get_indirection_to_bucket(i); 7252 j %= vi->nrxq; 7253 rxq = &sc->sge.rxq[vi->first_rxq + j]; 7254 vi->rss[i++] = rxq->iq.abs_id; 7255 #else 7256 for_each_rxq(vi, j, rxq) { 7257 vi->rss[i++] = rxq->iq.abs_id; 7258 if (i == vi->rss_size) 7259 break; 7260 } 7261 #endif 7262 } 7263 7264 rc = -t4_config_rss_range(sc, sc->mbox, vi->viid, 0, vi->rss_size, 7265 vi->rss, vi->rss_size); 7266 if (rc != 0) { 7267 CH_ERR(vi, "rss_config failed: %d\n", rc); 7268 return (rc); 7269 } 7270 7271 vi->hashen = hashconfig_to_hashen(hashconfig); 7272 7273 /* 7274 * We may have had to enable some hashes even though the global config 7275 * wants them disabled. This is a potential problem that must be 7276 * reported to the user. 7277 */ 7278 extra = hashen_to_hashconfig(vi->hashen) ^ hashconfig; 7279 7280 /* 7281 * If we consider only the supported hash types, then the enabled hashes 7282 * are a superset of the requested hashes. In other words, there cannot 7283 * be any supported hash that was requested but not enabled, but there 7284 * can be hashes that were not requested but had to be enabled. 7285 */ 7286 extra &= SUPPORTED_RSS_HASHTYPES; 7287 MPASS((extra & hashconfig) == 0); 7288 7289 if (extra) { 7290 CH_ALERT(vi, 7291 "global RSS config (0x%x) cannot be accommodated.\n", 7292 hashconfig); 7293 } 7294 if (extra & RSS_HASHTYPE_RSS_IPV4) 7295 CH_ALERT(vi, "IPv4 2-tuple hashing forced on.\n"); 7296 if (extra & RSS_HASHTYPE_RSS_TCP_IPV4) 7297 CH_ALERT(vi, "TCP/IPv4 4-tuple hashing forced on.\n"); 7298 if (extra & RSS_HASHTYPE_RSS_IPV6) 7299 CH_ALERT(vi, "IPv6 2-tuple hashing forced on.\n"); 7300 if (extra & RSS_HASHTYPE_RSS_TCP_IPV6) 7301 CH_ALERT(vi, "TCP/IPv6 4-tuple hashing forced on.\n"); 7302 if (extra & RSS_HASHTYPE_RSS_UDP_IPV4) 7303 CH_ALERT(vi, "UDP/IPv4 4-tuple hashing forced on.\n"); 7304 if (extra & RSS_HASHTYPE_RSS_UDP_IPV6) 7305 CH_ALERT(vi, "UDP/IPv6 4-tuple hashing forced on.\n"); 7306 7307 rc = -t4_config_vi_rss(sc, sc->mbox, vi->viid, vi->hashen, vi->rss[0], 7308 0, 0); 7309 if (rc != 0) { 7310 CH_ERR(vi, "rss hash/defaultq config failed: %d\n", rc); 7311 return (rc); 7312 } 7313 7314 return (0); 7315 } 7316 7317 int 7318 vi_init(struct vi_info *vi) 7319 { 7320 int rc; 7321 7322 ASSERT_SYNCHRONIZED_OP(vi->adapter); 7323 KASSERT((vi->flags & VI_INIT_DONE) == 0, 7324 ("%s: VI_INIT_DONE already", __func__)); 7325 7326 rc = vi_full_init(vi); 7327 if (rc != 0) 7328 vi_full_uninit(vi); 7329 else 7330 vi->flags |= VI_INIT_DONE; 7331 7332 return (rc); 7333 } 7334 7335 /* 7336 * Idempotent. 7337 */ 7338 static void 7339 vi_full_uninit(struct vi_info *vi) 7340 { 7341 7342 if (vi->flags & VI_INIT_DONE) { 7343 quiesce_vi(vi); 7344 free(vi->rss, M_CXGBE); 7345 free(vi->nm_rss, M_CXGBE); 7346 } 7347 7348 t4_teardown_vi_queues(vi); 7349 vi->flags &= ~VI_INIT_DONE; 7350 } 7351 7352 static void 7353 quiesce_txq(struct sge_txq *txq) 7354 { 7355 struct sge_eq *eq = &txq->eq; 7356 struct sge_qstat *spg = (void *)&eq->desc[eq->sidx]; 7357 7358 MPASS(eq->flags & EQ_SW_ALLOCATED); 7359 MPASS(!(eq->flags & EQ_ENABLED)); 7360 7361 /* Wait for the mp_ring to empty. */ 7362 while (!mp_ring_is_idle(txq->r)) { 7363 mp_ring_check_drainage(txq->r, 4096); 7364 pause("rquiesce", 1); 7365 } 7366 MPASS(txq->txp.npkt == 0); 7367 7368 if (eq->flags & EQ_HW_ALLOCATED) { 7369 /* 7370 * Hardware is alive and working normally. Wait for it to 7371 * finish and then wait for the driver to catch up and reclaim 7372 * all descriptors. 7373 */ 7374 while (spg->cidx != htobe16(eq->pidx)) 7375 pause("equiesce", 1); 7376 while (eq->cidx != eq->pidx) 7377 pause("dquiesce", 1); 7378 } else { 7379 /* 7380 * Hardware is unavailable. Discard all pending tx and reclaim 7381 * descriptors directly. 7382 */ 7383 TXQ_LOCK(txq); 7384 while (eq->cidx != eq->pidx) { 7385 struct mbuf *m, *nextpkt; 7386 struct tx_sdesc *txsd; 7387 7388 txsd = &txq->sdesc[eq->cidx]; 7389 for (m = txsd->m; m != NULL; m = nextpkt) { 7390 nextpkt = m->m_nextpkt; 7391 m->m_nextpkt = NULL; 7392 m_freem(m); 7393 } 7394 IDXINCR(eq->cidx, txsd->desc_used, eq->sidx); 7395 } 7396 spg->pidx = spg->cidx = htobe16(eq->cidx); 7397 TXQ_UNLOCK(txq); 7398 } 7399 } 7400 7401 static void 7402 quiesce_wrq(struct sge_wrq *wrq) 7403 { 7404 struct wrqe *wr; 7405 7406 TXQ_LOCK(wrq); 7407 while ((wr = STAILQ_FIRST(&wrq->wr_list)) != NULL) { 7408 STAILQ_REMOVE_HEAD(&wrq->wr_list, link); 7409 #ifdef INVARIANTS 7410 wrq->nwr_pending--; 7411 wrq->ndesc_needed -= howmany(wr->wr_len, EQ_ESIZE); 7412 #endif 7413 free(wr, M_CXGBE); 7414 } 7415 MPASS(wrq->nwr_pending == 0); 7416 MPASS(wrq->ndesc_needed == 0); 7417 wrq->nwr_pending = 0; 7418 wrq->ndesc_needed = 0; 7419 TXQ_UNLOCK(wrq); 7420 } 7421 7422 static void 7423 quiesce_iq_fl(struct adapter *sc, struct sge_iq *iq, struct sge_fl *fl) 7424 { 7425 /* Synchronize with the interrupt handler */ 7426 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 7427 pause("iqfree", 1); 7428 7429 if (fl != NULL) { 7430 MPASS(iq->flags & IQ_HAS_FL); 7431 7432 mtx_lock(&sc->sfl_lock); 7433 FL_LOCK(fl); 7434 fl->flags |= FL_DOOMED; 7435 FL_UNLOCK(fl); 7436 callout_stop(&sc->sfl_callout); 7437 mtx_unlock(&sc->sfl_lock); 7438 7439 KASSERT((fl->flags & FL_STARVING) == 0, 7440 ("%s: still starving", __func__)); 7441 7442 /* Release all buffers if hardware is no longer available. */ 7443 if (!(iq->flags & IQ_HW_ALLOCATED)) 7444 free_fl_buffers(sc, fl); 7445 } 7446 } 7447 7448 /* 7449 * Wait for all activity on all the queues of the VI to complete. It is assumed 7450 * that no new work is being enqueued by the hardware or the driver. That part 7451 * should be arranged before calling this function. 7452 */ 7453 static void 7454 quiesce_vi(struct vi_info *vi) 7455 { 7456 int i; 7457 struct adapter *sc = vi->adapter; 7458 struct sge_rxq *rxq; 7459 struct sge_txq *txq; 7460 #ifdef TCP_OFFLOAD 7461 struct sge_ofld_rxq *ofld_rxq; 7462 #endif 7463 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 7464 struct sge_ofld_txq *ofld_txq; 7465 #endif 7466 7467 if (!(vi->flags & VI_INIT_DONE)) 7468 return; 7469 7470 for_each_txq(vi, i, txq) { 7471 quiesce_txq(txq); 7472 } 7473 7474 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 7475 for_each_ofld_txq(vi, i, ofld_txq) { 7476 quiesce_wrq(&ofld_txq->wrq); 7477 } 7478 #endif 7479 7480 for_each_rxq(vi, i, rxq) { 7481 quiesce_iq_fl(sc, &rxq->iq, &rxq->fl); 7482 } 7483 7484 #ifdef TCP_OFFLOAD 7485 for_each_ofld_rxq(vi, i, ofld_rxq) { 7486 quiesce_iq_fl(sc, &ofld_rxq->iq, &ofld_rxq->fl); 7487 } 7488 #endif 7489 } 7490 7491 static int 7492 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 7493 driver_intr_t *handler, void *arg, char *name) 7494 { 7495 int rc; 7496 7497 irq->rid = rid; 7498 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 7499 RF_SHAREABLE | RF_ACTIVE); 7500 if (irq->res == NULL) { 7501 device_printf(sc->dev, 7502 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 7503 return (ENOMEM); 7504 } 7505 7506 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 7507 NULL, handler, arg, &irq->tag); 7508 if (rc != 0) { 7509 device_printf(sc->dev, 7510 "failed to setup interrupt for rid %d, name %s: %d\n", 7511 rid, name, rc); 7512 } else if (name) 7513 bus_describe_intr(sc->dev, irq->res, irq->tag, "%s", name); 7514 7515 return (rc); 7516 } 7517 7518 static int 7519 t4_free_irq(struct adapter *sc, struct irq *irq) 7520 { 7521 if (irq->tag) 7522 bus_teardown_intr(sc->dev, irq->res, irq->tag); 7523 if (irq->res) 7524 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 7525 7526 bzero(irq, sizeof(*irq)); 7527 7528 return (0); 7529 } 7530 7531 static void 7532 get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 7533 { 7534 7535 regs->version = chip_id(sc) | chip_rev(sc) << 10; 7536 t4_get_regs(sc, buf, regs->len); 7537 } 7538 7539 #define A_PL_INDIR_CMD 0x1f8 7540 7541 #define S_PL_AUTOINC 31 7542 #define M_PL_AUTOINC 0x1U 7543 #define V_PL_AUTOINC(x) ((x) << S_PL_AUTOINC) 7544 #define G_PL_AUTOINC(x) (((x) >> S_PL_AUTOINC) & M_PL_AUTOINC) 7545 7546 #define S_PL_VFID 20 7547 #define M_PL_VFID 0xffU 7548 #define V_PL_VFID(x) ((x) << S_PL_VFID) 7549 #define G_PL_VFID(x) (((x) >> S_PL_VFID) & M_PL_VFID) 7550 7551 #define S_PL_ADDR 0 7552 #define M_PL_ADDR 0xfffffU 7553 #define V_PL_ADDR(x) ((x) << S_PL_ADDR) 7554 #define G_PL_ADDR(x) (((x) >> S_PL_ADDR) & M_PL_ADDR) 7555 7556 #define A_PL_INDIR_DATA 0x1fc 7557 7558 static uint64_t 7559 read_vf_stat(struct adapter *sc, u_int vin, int reg) 7560 { 7561 u32 stats[2]; 7562 7563 if (sc->flags & IS_VF) { 7564 stats[0] = t4_read_reg(sc, VF_MPS_REG(reg)); 7565 stats[1] = t4_read_reg(sc, VF_MPS_REG(reg + 4)); 7566 } else { 7567 mtx_assert(&sc->reg_lock, MA_OWNED); 7568 t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) | 7569 V_PL_VFID(vin) | V_PL_ADDR(VF_MPS_REG(reg))); 7570 stats[0] = t4_read_reg(sc, A_PL_INDIR_DATA); 7571 stats[1] = t4_read_reg(sc, A_PL_INDIR_DATA); 7572 } 7573 return (((uint64_t)stats[1]) << 32 | stats[0]); 7574 } 7575 7576 static void 7577 t4_get_vi_stats(struct adapter *sc, u_int vin, struct fw_vi_stats_vf *stats) 7578 { 7579 7580 #define GET_STAT(name) \ 7581 read_vf_stat(sc, vin, A_MPS_VF_STAT_##name##_L) 7582 7583 if (!(sc->flags & IS_VF)) 7584 mtx_lock(&sc->reg_lock); 7585 stats->tx_bcast_bytes = GET_STAT(TX_VF_BCAST_BYTES); 7586 stats->tx_bcast_frames = GET_STAT(TX_VF_BCAST_FRAMES); 7587 stats->tx_mcast_bytes = GET_STAT(TX_VF_MCAST_BYTES); 7588 stats->tx_mcast_frames = GET_STAT(TX_VF_MCAST_FRAMES); 7589 stats->tx_ucast_bytes = GET_STAT(TX_VF_UCAST_BYTES); 7590 stats->tx_ucast_frames = GET_STAT(TX_VF_UCAST_FRAMES); 7591 stats->tx_drop_frames = GET_STAT(TX_VF_DROP_FRAMES); 7592 stats->tx_offload_bytes = GET_STAT(TX_VF_OFFLOAD_BYTES); 7593 stats->tx_offload_frames = GET_STAT(TX_VF_OFFLOAD_FRAMES); 7594 stats->rx_bcast_bytes = GET_STAT(RX_VF_BCAST_BYTES); 7595 stats->rx_bcast_frames = GET_STAT(RX_VF_BCAST_FRAMES); 7596 stats->rx_mcast_bytes = GET_STAT(RX_VF_MCAST_BYTES); 7597 stats->rx_mcast_frames = GET_STAT(RX_VF_MCAST_FRAMES); 7598 stats->rx_ucast_bytes = GET_STAT(RX_VF_UCAST_BYTES); 7599 stats->rx_ucast_frames = GET_STAT(RX_VF_UCAST_FRAMES); 7600 stats->rx_err_frames = GET_STAT(RX_VF_ERR_FRAMES); 7601 if (!(sc->flags & IS_VF)) 7602 mtx_unlock(&sc->reg_lock); 7603 7604 #undef GET_STAT 7605 } 7606 7607 static void 7608 t4_clr_vi_stats(struct adapter *sc, u_int vin) 7609 { 7610 int reg; 7611 7612 t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) | V_PL_VFID(vin) | 7613 V_PL_ADDR(VF_MPS_REG(A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L))); 7614 for (reg = A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L; 7615 reg <= A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H; reg += 4) 7616 t4_write_reg(sc, A_PL_INDIR_DATA, 0); 7617 } 7618 7619 static void 7620 vi_refresh_stats(struct vi_info *vi) 7621 { 7622 struct timeval tv; 7623 const struct timeval interval = {0, 250000}; /* 250ms */ 7624 7625 mtx_assert(&vi->tick_mtx, MA_OWNED); 7626 7627 if (vi->flags & VI_SKIP_STATS) 7628 return; 7629 7630 getmicrotime(&tv); 7631 timevalsub(&tv, &interval); 7632 if (timevalcmp(&tv, &vi->last_refreshed, <)) 7633 return; 7634 7635 t4_get_vi_stats(vi->adapter, vi->vin, &vi->stats); 7636 getmicrotime(&vi->last_refreshed); 7637 } 7638 7639 static void 7640 cxgbe_refresh_stats(struct vi_info *vi) 7641 { 7642 u_int i, v, tnl_cong_drops, chan_map; 7643 struct timeval tv; 7644 const struct timeval interval = {0, 250000}; /* 250ms */ 7645 struct port_info *pi; 7646 struct adapter *sc; 7647 7648 mtx_assert(&vi->tick_mtx, MA_OWNED); 7649 7650 if (vi->flags & VI_SKIP_STATS) 7651 return; 7652 7653 getmicrotime(&tv); 7654 timevalsub(&tv, &interval); 7655 if (timevalcmp(&tv, &vi->last_refreshed, <)) 7656 return; 7657 7658 pi = vi->pi; 7659 sc = vi->adapter; 7660 tnl_cong_drops = 0; 7661 t4_get_port_stats(sc, pi->hw_port, &pi->stats); 7662 chan_map = pi->rx_e_chan_map; 7663 while (chan_map) { 7664 i = ffs(chan_map) - 1; 7665 mtx_lock(&sc->reg_lock); 7666 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 1, 7667 A_TP_MIB_TNL_CNG_DROP_0 + i); 7668 mtx_unlock(&sc->reg_lock); 7669 tnl_cong_drops += v; 7670 chan_map &= ~(1 << i); 7671 } 7672 pi->tnl_cong_drops = tnl_cong_drops; 7673 getmicrotime(&vi->last_refreshed); 7674 } 7675 7676 static void 7677 cxgbe_tick(void *arg) 7678 { 7679 struct vi_info *vi = arg; 7680 7681 MPASS(IS_MAIN_VI(vi)); 7682 mtx_assert(&vi->tick_mtx, MA_OWNED); 7683 7684 cxgbe_refresh_stats(vi); 7685 callout_schedule(&vi->tick, hz); 7686 } 7687 7688 static void 7689 vi_tick(void *arg) 7690 { 7691 struct vi_info *vi = arg; 7692 7693 mtx_assert(&vi->tick_mtx, MA_OWNED); 7694 7695 vi_refresh_stats(vi); 7696 callout_schedule(&vi->tick, hz); 7697 } 7698 7699 /* CIM inbound queues */ 7700 static const char *t4_ibq[CIM_NUM_IBQ] = { 7701 "ibq_tp0", "ibq_tp1", "ibq_ulp", "ibq_sge0", "ibq_sge1", "ibq_ncsi" 7702 }; 7703 static const char *t7_ibq[CIM_NUM_IBQ_T7] = { 7704 "ibq_tp0", "ibq_tp1", "ibq_tp2", "ibq_tp3", "ibq_ulp", "ibq_sge0", 7705 "ibq_sge1", "ibq_ncsi", NULL, "ibq_ipc1", "ibq_ipc2", "ibq_ipc3", 7706 "ibq_ipc4", "ibq_ipc5", "ibq_ipc6", "ibq_ipc7" 7707 }; 7708 static const char *t7_ibq_sec[] = { 7709 "ibq_tp0", "ibq_tp1", "ibq_tp2", "ibq_tp3", "ibq_ulp", "ibq_sge0", 7710 NULL, NULL, NULL, "ibq_ipc0" 7711 }; 7712 7713 /* CIM outbound queues */ 7714 static const char *t4_obq[CIM_NUM_OBQ_T5] = { 7715 "obq_ulp0", "obq_ulp1", "obq_ulp2", "obq_ulp3", "obq_sge", "obq_ncsi", 7716 "obq_sge_rx_q0", "obq_sge_rx_q1" /* These two are T5/T6 only */ 7717 }; 7718 static const char *t7_obq[CIM_NUM_OBQ_T7] = { 7719 "obq_ulp0", "obq_ulp1", "obq_ulp2", "obq_ulp3", "obq_sge", "obq_ncsi", 7720 "obq_sge_rx_q0", NULL, NULL, "obq_ipc1", "obq_ipc2", "obq_ipc3", 7721 "obq_ipc4", "obq_ipc5", "obq_ipc6", "obq_ipc7" 7722 }; 7723 static const char *t7_obq_sec[] = { 7724 "obq_ulp0", "obq_ulp1", "obq_ulp2", "obq_ulp3", "obq_sge", NULL, 7725 "obq_sge_rx_q0", NULL, NULL, "obq_ipc0" 7726 }; 7727 7728 static void 7729 cim_sysctls(struct adapter *sc, struct sysctl_ctx_list *ctx, 7730 struct sysctl_oid_list *c0) 7731 { 7732 struct sysctl_oid *oid; 7733 struct sysctl_oid_list *children1; 7734 int i, j, qcount; 7735 char s[16]; 7736 const char **qname; 7737 7738 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "cim", 7739 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "CIM block"); 7740 c0 = SYSCTL_CHILDREN(oid); 7741 7742 SYSCTL_ADD_U8(ctx, c0, OID_AUTO, "ncores", CTLFLAG_RD, NULL, 7743 sc->params.ncores, "# of active CIM cores"); 7744 7745 for (i = 0; i < sc->params.ncores; i++) { 7746 snprintf(s, sizeof(s), "%u", i); 7747 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, s, 7748 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "CIM core"); 7749 children1 = SYSCTL_CHILDREN(oid); 7750 7751 /* 7752 * CTLFLAG_SKIP because the misc.devlog sysctl already displays 7753 * the log for all cores. Use this sysctl to get the log for a 7754 * particular core only. 7755 */ 7756 SYSCTL_ADD_PROC(ctx, children1, OID_AUTO, "devlog", 7757 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE | CTLFLAG_SKIP, 7758 sc, i, sysctl_devlog, "A", "firmware's device log"); 7759 7760 SYSCTL_ADD_PROC(ctx, children1, OID_AUTO, "loadavg", 7761 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, i, 7762 sysctl_loadavg, "A", 7763 "microprocessor load averages (select firmwares only)"); 7764 7765 SYSCTL_ADD_PROC(ctx, children1, OID_AUTO, "qcfg", 7766 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, i, 7767 chip_id(sc) > CHELSIO_T6 ? sysctl_cim_qcfg_t7 : sysctl_cim_qcfg, 7768 "A", "Queue configuration"); 7769 7770 SYSCTL_ADD_PROC(ctx, children1, OID_AUTO, "la", 7771 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, i, 7772 sysctl_cim_la, "A", "Logic analyzer"); 7773 7774 SYSCTL_ADD_PROC(ctx, children1, OID_AUTO, "ma_la", 7775 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, i, 7776 sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); 7777 7778 SYSCTL_ADD_PROC(ctx, children1, OID_AUTO, "pif_la", 7779 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, i, 7780 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); 7781 7782 /* IBQs */ 7783 switch (chip_id(sc)) { 7784 case CHELSIO_T4: 7785 case CHELSIO_T5: 7786 case CHELSIO_T6: 7787 qname = &t4_ibq[0]; 7788 qcount = nitems(t4_ibq); 7789 break; 7790 case CHELSIO_T7: 7791 default: 7792 if (i == 0) { 7793 qname = &t7_ibq[0]; 7794 qcount = nitems(t7_ibq); 7795 } else { 7796 qname = &t7_ibq_sec[0]; 7797 qcount = nitems(t7_ibq_sec); 7798 } 7799 break; 7800 } 7801 MPASS(qcount <= sc->chip_params->cim_num_ibq); 7802 for (j = 0; j < qcount; j++) { 7803 if (qname[j] == NULL) 7804 continue; 7805 SYSCTL_ADD_PROC(ctx, children1, OID_AUTO, qname[j], 7806 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 7807 (i << 16) | j, sysctl_cim_ibq, "A", NULL); 7808 } 7809 7810 /* OBQs */ 7811 switch (chip_id(sc)) { 7812 case CHELSIO_T4: 7813 qname = t4_obq; 7814 qcount = CIM_NUM_OBQ; 7815 break; 7816 case CHELSIO_T5: 7817 case CHELSIO_T6: 7818 qname = t4_obq; 7819 qcount = nitems(t4_obq); 7820 break; 7821 case CHELSIO_T7: 7822 default: 7823 if (i == 0) { 7824 qname = t7_obq; 7825 qcount = nitems(t7_obq); 7826 } else { 7827 qname = t7_obq_sec; 7828 qcount = nitems(t7_obq_sec); 7829 } 7830 break; 7831 } 7832 MPASS(qcount <= sc->chip_params->cim_num_obq); 7833 for (j = 0; j < qcount; j++) { 7834 if (qname[j] == NULL) 7835 continue; 7836 SYSCTL_ADD_PROC(ctx, children1, OID_AUTO, qname[j], 7837 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 7838 (i << 16) | j, sysctl_cim_obq, "A", NULL); 7839 } 7840 } 7841 } 7842 7843 /* 7844 * Should match fw_caps_config_<foo> enums in t4fw_interface.h 7845 */ 7846 static char *caps_decoder[] = { 7847 "\20\001IPMI\002NCSI", /* 0: NBM */ 7848 "\20\001PPP\002QFC\003DCBX", /* 1: link */ 7849 "\20\001INGRESS\002EGRESS", /* 2: switch */ 7850 "\20\001NIC\002VM\003IDS\004UM\005UM_ISGL" /* 3: NIC */ 7851 "\006HASHFILTER\007ETHOFLD", 7852 "\20\001TOE\002SENDPATH", /* 4: TOE */ 7853 "\20\001RDDP\002RDMAC\003ROCEv2", /* 5: RDMA */ 7854 "\20\001INITIATOR_PDU\002TARGET_PDU" /* 6: iSCSI */ 7855 "\003INITIATOR_CNXOFLD\004TARGET_CNXOFLD" 7856 "\005INITIATOR_SSNOFLD\006TARGET_SSNOFLD" 7857 "\007T10DIF" 7858 "\010INITIATOR_CMDOFLD\011TARGET_CMDOFLD", 7859 "\20\001LOOKASIDE\002TLSKEYS\003IPSEC_INLINE" /* 7: Crypto */ 7860 "\004TLS_HW,\005TOE_IPSEC", 7861 "\20\001INITIATOR\002TARGET\003CTRL_OFLD" /* 8: FCoE */ 7862 "\004PO_INITIATOR\005PO_TARGET", 7863 "\20\001NVMe_TCP", /* 9: NVMe */ 7864 }; 7865 7866 void 7867 t4_sysctls(struct adapter *sc) 7868 { 7869 struct sysctl_ctx_list *ctx = &sc->ctx; 7870 struct sysctl_oid *oid; 7871 struct sysctl_oid_list *children, *c0; 7872 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 7873 7874 /* 7875 * dev.t4nex.X. 7876 */ 7877 oid = device_get_sysctl_tree(sc->dev); 7878 c0 = children = SYSCTL_CHILDREN(oid); 7879 7880 sc->sc_do_rxcopy = 1; 7881 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, 7882 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); 7883 7884 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 7885 sc->params.nports, "# of ports"); 7886 7887 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 7888 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, doorbells, 7889 (uintptr_t)&sc->doorbells, sysctl_bitfield_8b, "A", 7890 "available doorbells"); 7891 7892 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 7893 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 7894 7895 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 7896 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, 7897 sc->params.sge.timer_val, sizeof(sc->params.sge.timer_val), 7898 sysctl_int_array, "A", "interrupt holdoff timer values (us)"); 7899 7900 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 7901 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, 7902 sc->params.sge.counter_val, sizeof(sc->params.sge.counter_val), 7903 sysctl_int_array, "A", "interrupt holdoff packet counter values"); 7904 7905 t4_sge_sysctls(sc, ctx, children); 7906 7907 sc->lro_timeout = 100; 7908 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, 7909 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); 7910 7911 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dflags", CTLFLAG_RW, 7912 &sc->debug_flags, 0, "flags to enable runtime debugging"); 7913 7914 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "iflags", CTLFLAG_RW, 7915 &sc->intr_flags, 0, "flags for the slow interrupt handler"); 7916 7917 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "tp_version", 7918 CTLFLAG_RD, sc->tp_version, 0, "TP microcode version"); 7919 7920 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 7921 CTLFLAG_RD, sc->fw_version, 0, "firmware version"); 7922 7923 if (sc->flags & IS_VF) 7924 return; 7925 7926 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 7927 NULL, chip_rev(sc), "chip hardware revision"); 7928 7929 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "sn", 7930 CTLFLAG_RD, sc->params.vpd.sn, 0, "serial number"); 7931 7932 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "pn", 7933 CTLFLAG_RD, sc->params.vpd.pn, 0, "part number"); 7934 7935 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "ec", 7936 CTLFLAG_RD, sc->params.vpd.ec, 0, "engineering change"); 7937 7938 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "md_version", 7939 CTLFLAG_RD, sc->params.vpd.md, 0, "manufacturing diags version"); 7940 7941 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "na", 7942 CTLFLAG_RD, sc->params.vpd.na, 0, "network address"); 7943 7944 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "er_version", CTLFLAG_RD, 7945 sc->er_version, 0, "expansion ROM version"); 7946 7947 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "bs_version", CTLFLAG_RD, 7948 sc->bs_version, 0, "bootstrap firmware version"); 7949 7950 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "scfg_version", CTLFLAG_RD, 7951 NULL, sc->params.scfg_vers, "serial config version"); 7952 7953 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "vpd_version", CTLFLAG_RD, 7954 NULL, sc->params.vpd_vers, "VPD version"); 7955 7956 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 7957 CTLFLAG_RD, sc->cfg_file, 0, "configuration file"); 7958 7959 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 7960 sc->cfcsum, "config file checksum"); 7961 7962 #define SYSCTL_CAP(name, n, text) \ 7963 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, #name, \ 7964 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, caps_decoder[n], \ 7965 (uintptr_t)&sc->name, sysctl_bitfield_16b, "A", \ 7966 "available " text " capabilities") 7967 7968 SYSCTL_CAP(nbmcaps, 0, "NBM"); 7969 SYSCTL_CAP(linkcaps, 1, "link"); 7970 SYSCTL_CAP(switchcaps, 2, "switch"); 7971 SYSCTL_CAP(nvmecaps, 9, "NVMe"); 7972 SYSCTL_CAP(niccaps, 3, "NIC"); 7973 SYSCTL_CAP(toecaps, 4, "TCP offload"); 7974 SYSCTL_CAP(rdmacaps, 5, "RDMA"); 7975 SYSCTL_CAP(iscsicaps, 6, "iSCSI"); 7976 SYSCTL_CAP(cryptocaps, 7, "crypto"); 7977 SYSCTL_CAP(fcoecaps, 8, "FCoE"); 7978 #undef SYSCTL_CAP 7979 7980 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, 7981 NULL, sc->tids.nftids, "number of filters"); 7982 7983 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 7984 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 7985 sysctl_temperature, "I", "chip temperature (in Celsius)"); 7986 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "reset_sensor", 7987 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 7988 sysctl_reset_sensor, "I", "reset the chip's temperature sensor."); 7989 7990 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "core_vdd", 7991 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_vdd, 7992 "I", "core Vdd (in mV)"); 7993 7994 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "local_cpus", 7995 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, LOCAL_CPUS, 7996 sysctl_cpus, "A", "local CPUs"); 7997 7998 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "intr_cpus", 7999 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, INTR_CPUS, 8000 sysctl_cpus, "A", "preferred CPUs for interrupts"); 8001 8002 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "swintr", CTLFLAG_RW, 8003 &sc->swintr, 0, "software triggered interrupts"); 8004 8005 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "reset", 8006 CTLTYPE_INT | CTLFLAG_RW, sc, 0, sysctl_reset, "I", 8007 "1 = reset adapter, 0 = zero reset counter"); 8008 8009 /* 8010 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 8011 */ 8012 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 8013 CTLFLAG_RD | CTLFLAG_SKIP | CTLFLAG_MPSAFE, NULL, 8014 "logs and miscellaneous information"); 8015 children = SYSCTL_CHILDREN(oid); 8016 8017 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 8018 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 8019 sysctl_cctrl, "A", "congestion control"); 8020 8021 cim_sysctls(sc, ctx, children); 8022 8023 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 8024 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 8025 sysctl_cpl_stats, "A", "CPL statistics"); 8026 8027 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 8028 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 8029 sysctl_ddp_stats, "A", "non-TCP DDP statistics"); 8030 8031 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tid_stats", 8032 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 8033 sysctl_tid_stats, "A", "tid stats"); 8034 8035 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 8036 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, -1, 8037 sysctl_devlog, "A", "firmware's device log (all cores)"); 8038 8039 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 8040 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 8041 sysctl_fcoe_stats, "A", "FCoE statistics"); 8042 8043 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 8044 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 8045 sysctl_hw_sched, "A", "hardware scheduler "); 8046 8047 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 8048 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 8049 sysctl_l2t, "A", "hardware L2 table"); 8050 8051 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "smt", 8052 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 8053 sysctl_smt, "A", "hardware source MAC table"); 8054 8055 #ifdef INET6 8056 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "clip", 8057 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 8058 sysctl_clip, "A", "active CLIP table entries"); 8059 #endif 8060 8061 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 8062 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 8063 sysctl_lb_stats, "A", "loopback statistics"); 8064 8065 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 8066 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 8067 sysctl_meminfo, "A", "memory regions"); 8068 8069 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", 8070 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 8071 chip_id(sc) >= CHELSIO_T7 ? sysctl_mps_tcam_t7 : 8072 (chip_id(sc) >= CHELSIO_T6 ? sysctl_mps_tcam_t6 : sysctl_mps_tcam), 8073 "A", "MPS TCAM entries"); 8074 8075 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 8076 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 8077 sysctl_path_mtus, "A", "path MTUs"); 8078 8079 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 8080 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 8081 sysctl_pm_stats, "A", "PM statistics"); 8082 8083 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 8084 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 8085 sysctl_rdma_stats, "A", "RDMA statistics"); 8086 8087 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 8088 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 8089 sysctl_tcp_stats, "A", "TCP statistics"); 8090 8091 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 8092 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 8093 sysctl_tids, "A", "TID information"); 8094 8095 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 8096 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 8097 sysctl_tp_err_stats, "A", "TP error statistics"); 8098 8099 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tnl_stats", 8100 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 8101 sysctl_tnl_stats, "A", "TP tunnel statistics"); 8102 8103 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la_mask", 8104 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 8105 sysctl_tp_la_mask, "I", "TP logic analyzer event capture mask"); 8106 8107 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", 8108 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 8109 sysctl_tp_la, "A", "TP logic analyzer"); 8110 8111 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 8112 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 8113 sysctl_tx_rate, "A", "Tx rate"); 8114 8115 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", 8116 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 8117 sysctl_ulprx_la, "A", "ULPRX logic analyzer"); 8118 8119 if (chip_id(sc) >= CHELSIO_T5) { 8120 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 8121 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 8122 sysctl_wcwr_stats, "A", "write combined work requests"); 8123 } 8124 8125 if (chip_id(sc) >= CHELSIO_T7) { 8126 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcb_cache", 8127 CTLTYPE_INT | CTLFLAG_RW, sc, 0, sysctl_tcb_cache, "I", 8128 "1 = enabled (default), 0 = disabled (for debug only)"); 8129 } 8130 8131 #ifdef KERN_TLS 8132 if (is_ktls(sc)) { 8133 /* 8134 * dev.t4nex.0.tls. 8135 */ 8136 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "tls", 8137 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "KERN_TLS parameters"); 8138 children = SYSCTL_CHILDREN(oid); 8139 8140 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "inline_keys", 8141 CTLFLAG_RW, &sc->tlst.inline_keys, 0, "Always pass TLS " 8142 "keys in work requests (1) or attempt to store TLS keys " 8143 "in card memory."); 8144 8145 if (is_t6(sc)) 8146 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "combo_wrs", 8147 CTLFLAG_RW, &sc->tlst.combo_wrs, 0, "Attempt to " 8148 "combine TCB field updates with TLS record work " 8149 "requests."); 8150 else { 8151 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "short_records", 8152 CTLFLAG_RW, &sc->tlst.short_records, 0, 8153 "Use cipher-only mode for short records."); 8154 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "partial_ghash", 8155 CTLFLAG_RW, &sc->tlst.partial_ghash, 0, 8156 "Use partial GHASH for AES-GCM records."); 8157 } 8158 } 8159 #endif 8160 8161 #ifdef TCP_OFFLOAD 8162 if (is_offload(sc)) { 8163 int i; 8164 char s[4]; 8165 8166 /* 8167 * dev.t4nex.X.toe. 8168 */ 8169 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", 8170 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "TOE parameters"); 8171 children = SYSCTL_CHILDREN(oid); 8172 8173 sc->tt.cong_algorithm = -1; 8174 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "cong_algorithm", 8175 CTLFLAG_RW, &sc->tt.cong_algorithm, 0, "congestion control " 8176 "(-1 = default, 0 = reno, 1 = tahoe, 2 = newreno, " 8177 "3 = highspeed)"); 8178 8179 sc->tt.sndbuf = -1; 8180 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 8181 &sc->tt.sndbuf, 0, "hardware send buffer"); 8182 8183 sc->tt.ddp = 0; 8184 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", 8185 CTLFLAG_RW | CTLFLAG_SKIP, &sc->tt.ddp, 0, ""); 8186 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_zcopy", CTLFLAG_RW, 8187 &sc->tt.ddp, 0, "Enable zero-copy aio_read(2)"); 8188 8189 sc->tt.rx_coalesce = -1; 8190 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", 8191 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); 8192 8193 sc->tt.tls = 1; 8194 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tls", CTLTYPE_INT | 8195 CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, sysctl_tls, "I", 8196 "Inline TLS allowed"); 8197 8198 sc->tt.tx_align = -1; 8199 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align", 8200 CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload"); 8201 8202 sc->tt.tx_zcopy = 0; 8203 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_zcopy", 8204 CTLFLAG_RW, &sc->tt.tx_zcopy, 0, 8205 "Enable zero-copy aio_write(2)"); 8206 8207 sc->tt.cop_managed_offloading = !!t4_cop_managed_offloading; 8208 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 8209 "cop_managed_offloading", CTLFLAG_RW, 8210 &sc->tt.cop_managed_offloading, 0, 8211 "COP (Connection Offload Policy) controls all TOE offload"); 8212 8213 sc->tt.autorcvbuf_inc = 16 * 1024; 8214 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "autorcvbuf_inc", 8215 CTLFLAG_RW, &sc->tt.autorcvbuf_inc, 0, 8216 "autorcvbuf increment"); 8217 8218 sc->tt.update_hc_on_pmtu_change = 1; 8219 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 8220 "update_hc_on_pmtu_change", CTLFLAG_RW, 8221 &sc->tt.update_hc_on_pmtu_change, 0, 8222 "Update hostcache entry if the PMTU changes"); 8223 8224 sc->tt.iso = 1; 8225 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "iso", CTLFLAG_RW, 8226 &sc->tt.iso, 0, "Enable iSCSI segmentation offload"); 8227 8228 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "timer_tick", 8229 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 8230 sysctl_tp_tick, "A", "TP timer tick (us)"); 8231 8232 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "timestamp_tick", 8233 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 1, 8234 sysctl_tp_tick, "A", "TCP timestamp tick (us)"); 8235 8236 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dack_tick", 8237 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 2, 8238 sysctl_tp_tick, "A", "DACK tick (us)"); 8239 8240 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dack_timer", 8241 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 8242 sysctl_tp_dack_timer, "IU", "DACK timer (us)"); 8243 8244 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rexmt_min", 8245 CTLTYPE_ULONG | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 8246 A_TP_RXT_MIN, sysctl_tp_timer, "LU", 8247 "Minimum retransmit interval (us)"); 8248 8249 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rexmt_max", 8250 CTLTYPE_ULONG | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 8251 A_TP_RXT_MAX, sysctl_tp_timer, "LU", 8252 "Maximum retransmit interval (us)"); 8253 8254 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "persist_min", 8255 CTLTYPE_ULONG | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 8256 A_TP_PERS_MIN, sysctl_tp_timer, "LU", 8257 "Persist timer min (us)"); 8258 8259 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "persist_max", 8260 CTLTYPE_ULONG | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 8261 A_TP_PERS_MAX, sysctl_tp_timer, "LU", 8262 "Persist timer max (us)"); 8263 8264 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "keepalive_idle", 8265 CTLTYPE_ULONG | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 8266 A_TP_KEEP_IDLE, sysctl_tp_timer, "LU", 8267 "Keepalive idle timer (us)"); 8268 8269 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "keepalive_interval", 8270 CTLTYPE_ULONG | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 8271 A_TP_KEEP_INTVL, sysctl_tp_timer, "LU", 8272 "Keepalive interval timer (us)"); 8273 8274 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "initial_srtt", 8275 CTLTYPE_ULONG | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 8276 A_TP_INIT_SRTT, sysctl_tp_timer, "LU", "Initial SRTT (us)"); 8277 8278 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "finwait2_timer", 8279 CTLTYPE_ULONG | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 8280 A_TP_FINWAIT2_TIMER, sysctl_tp_timer, "LU", 8281 "FINWAIT2 timer (us)"); 8282 8283 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "syn_rexmt_count", 8284 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 8285 S_SYNSHIFTMAX, sysctl_tp_shift_cnt, "IU", 8286 "Number of SYN retransmissions before abort"); 8287 8288 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rexmt_count", 8289 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 8290 S_RXTSHIFTMAXR2, sysctl_tp_shift_cnt, "IU", 8291 "Number of retransmissions before abort"); 8292 8293 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "keepalive_count", 8294 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 8295 S_KEEPALIVEMAXR2, sysctl_tp_shift_cnt, "IU", 8296 "Number of keepalive probes before abort"); 8297 8298 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "rexmt_backoff", 8299 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 8300 "TOE retransmit backoffs"); 8301 children = SYSCTL_CHILDREN(oid); 8302 for (i = 0; i < 16; i++) { 8303 snprintf(s, sizeof(s), "%u", i); 8304 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, s, 8305 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 8306 i, sysctl_tp_backoff, "IU", 8307 "TOE retransmit backoff"); 8308 } 8309 } 8310 #endif 8311 } 8312 8313 void 8314 vi_sysctls(struct vi_info *vi) 8315 { 8316 struct sysctl_ctx_list *ctx = &vi->ctx; 8317 struct sysctl_oid *oid; 8318 struct sysctl_oid_list *children; 8319 8320 /* 8321 * dev.v?(cxgbe|cxl).X. 8322 */ 8323 oid = device_get_sysctl_tree(vi->dev); 8324 children = SYSCTL_CHILDREN(oid); 8325 8326 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "viid", CTLFLAG_RD, NULL, 8327 vi->viid, "VI identifer"); 8328 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 8329 &vi->nrxq, 0, "# of rx queues"); 8330 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 8331 &vi->ntxq, 0, "# of tx queues"); 8332 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 8333 &vi->first_rxq, 0, "index of first rx queue"); 8334 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 8335 &vi->first_txq, 0, "index of first tx queue"); 8336 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "rss_base", CTLFLAG_RD, NULL, 8337 vi->rss_base, "start of RSS indirection table"); 8338 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "rss_size", CTLFLAG_RD, NULL, 8339 vi->rss_size, "size of RSS indirection table"); 8340 8341 if (IS_MAIN_VI(vi)) { 8342 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", 8343 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, vi, 0, 8344 sysctl_noflowq, "IU", 8345 "Reserve queue 0 for non-flowid packets"); 8346 } 8347 8348 if (vi->adapter->flags & IS_VF) { 8349 MPASS(vi->flags & TX_USES_VM_WR); 8350 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_vm_wr", CTLFLAG_RD, 8351 NULL, 1, "use VM work requests for transmit"); 8352 } else { 8353 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_vm_wr", 8354 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, vi, 0, 8355 sysctl_tx_vm_wr, "I", "use VM work requestes for transmit"); 8356 } 8357 8358 #ifdef TCP_OFFLOAD 8359 if (vi->nofldrxq != 0) { 8360 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 8361 &vi->nofldrxq, 0, 8362 "# of rx queues for offloaded TCP connections"); 8363 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 8364 CTLFLAG_RD, &vi->first_ofld_rxq, 0, 8365 "index of first TOE rx queue"); 8366 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx_ofld", 8367 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, vi, 0, 8368 sysctl_holdoff_tmr_idx_ofld, "I", 8369 "holdoff timer index for TOE queues"); 8370 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx_ofld", 8371 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, vi, 0, 8372 sysctl_holdoff_pktc_idx_ofld, "I", 8373 "holdoff packet counter index for TOE queues"); 8374 } 8375 #endif 8376 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 8377 if (vi->nofldtxq != 0) { 8378 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 8379 &vi->nofldtxq, 0, 8380 "# of tx queues for TOE/ETHOFLD"); 8381 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 8382 CTLFLAG_RD, &vi->first_ofld_txq, 0, 8383 "index of first TOE/ETHOFLD tx queue"); 8384 } 8385 #endif 8386 #ifdef DEV_NETMAP 8387 if (vi->nnmrxq != 0) { 8388 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD, 8389 &vi->nnmrxq, 0, "# of netmap rx queues"); 8390 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD, 8391 &vi->nnmtxq, 0, "# of netmap tx queues"); 8392 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq", 8393 CTLFLAG_RD, &vi->first_nm_rxq, 0, 8394 "index of first netmap rx queue"); 8395 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq", 8396 CTLFLAG_RD, &vi->first_nm_txq, 0, 8397 "index of first netmap tx queue"); 8398 } 8399 #endif 8400 8401 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 8402 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, vi, 0, 8403 sysctl_holdoff_tmr_idx, "I", "holdoff timer index"); 8404 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 8405 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, vi, 0, 8406 sysctl_holdoff_pktc_idx, "I", "holdoff packet counter index"); 8407 8408 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 8409 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, vi, 0, 8410 sysctl_qsize_rxq, "I", "rx queue size"); 8411 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 8412 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, vi, 0, 8413 sysctl_qsize_txq, "I", "tx queue size"); 8414 } 8415 8416 static void 8417 cxgbe_sysctls(struct port_info *pi) 8418 { 8419 struct sysctl_ctx_list *ctx = &pi->ctx; 8420 struct sysctl_oid *oid; 8421 struct sysctl_oid_list *children, *children2; 8422 struct adapter *sc = pi->adapter; 8423 int i; 8424 char name[16]; 8425 static char *tc_flags = {"\20\1USER"}; 8426 8427 /* 8428 * dev.cxgbe.X. 8429 */ 8430 oid = device_get_sysctl_tree(pi->dev); 8431 children = SYSCTL_CHILDREN(oid); 8432 8433 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", 8434 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, pi, 0, 8435 sysctl_linkdnrc, "A", "reason why link is down"); 8436 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { 8437 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 8438 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, pi, 0, 8439 sysctl_btphy, "I", "PHY temperature (in Celsius)"); 8440 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", 8441 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, pi, 1, 8442 sysctl_btphy, "I", "PHY firmware version"); 8443 } 8444 8445 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings", 8446 CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, pi, 0, 8447 sysctl_pause_settings, "A", 8448 "PAUSE settings (bit 0 = rx_pause, 1 = tx_pause, 2 = pause_autoneg)"); 8449 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "link_fec", 8450 CTLTYPE_STRING | CTLFLAG_MPSAFE, pi, 0, sysctl_link_fec, "A", 8451 "FEC in use on the link"); 8452 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "requested_fec", 8453 CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, pi, 0, 8454 sysctl_requested_fec, "A", 8455 "FECs to use (bit 0 = RS, 1 = FC, 2 = none, 5 = auto, 6 = module)"); 8456 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "module_fec", 8457 CTLTYPE_STRING | CTLFLAG_MPSAFE, pi, 0, sysctl_module_fec, "A", 8458 "FEC recommended by the cable/transceiver"); 8459 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "autoneg", 8460 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, pi, 0, 8461 sysctl_autoneg, "I", 8462 "autonegotiation (-1 = not supported)"); 8463 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "force_fec", 8464 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, pi, 0, 8465 sysctl_force_fec, "I", "when to use FORCE_FEC bit for link config"); 8466 8467 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rcaps", CTLFLAG_RD, 8468 &pi->link_cfg.requested_caps, 0, "L1 config requested by driver"); 8469 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "pcaps", CTLFLAG_RD, 8470 &pi->link_cfg.pcaps, 0, "port capabilities"); 8471 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "acaps", CTLFLAG_RD, 8472 &pi->link_cfg.acaps, 0, "advertised capabilities"); 8473 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lpacaps", CTLFLAG_RD, 8474 &pi->link_cfg.lpacaps, 0, "link partner advertised capabilities"); 8475 8476 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "max_speed", CTLFLAG_RD, NULL, 8477 port_top_speed(pi), "max speed (in Gbps)"); 8478 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "mps_bg_map", CTLFLAG_RD, NULL, 8479 pi->mps_bg_map, "MPS buffer group map"); 8480 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_e_chan_map", CTLFLAG_RD, 8481 NULL, pi->rx_e_chan_map, "TP rx e-channel map"); 8482 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_chan", CTLFLAG_RD, NULL, 8483 pi->tx_chan, "TP tx c-channel"); 8484 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_chan", CTLFLAG_RD, NULL, 8485 pi->rx_chan, "TP rx c-channel"); 8486 8487 if (sc->flags & IS_VF) 8488 return; 8489 8490 /* 8491 * dev.(cxgbe|cxl).X.tc. 8492 */ 8493 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "tc", 8494 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 8495 "Tx scheduler traffic classes (cl_rl)"); 8496 children2 = SYSCTL_CHILDREN(oid); 8497 SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "pktsize", 8498 CTLFLAG_RW, &pi->sched_params->pktsize, 0, 8499 "pktsize for per-flow cl-rl (0 means up to the driver )"); 8500 SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "burstsize", 8501 CTLFLAG_RW, &pi->sched_params->burstsize, 0, 8502 "burstsize for per-flow cl-rl (0 means up to the driver)"); 8503 for (i = 0; i < sc->params.nsched_cls; i++) { 8504 struct tx_cl_rl_params *tc = &pi->sched_params->cl_rl[i]; 8505 8506 snprintf(name, sizeof(name), "%d", i); 8507 children2 = SYSCTL_CHILDREN(SYSCTL_ADD_NODE(ctx, 8508 SYSCTL_CHILDREN(oid), OID_AUTO, name, 8509 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "traffic class")); 8510 SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "state", 8511 CTLFLAG_RD, &tc->state, 0, "current state"); 8512 SYSCTL_ADD_PROC(ctx, children2, OID_AUTO, "flags", 8513 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, tc_flags, 8514 (uintptr_t)&tc->flags, sysctl_bitfield_8b, "A", "flags"); 8515 SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "refcount", 8516 CTLFLAG_RD, &tc->refcount, 0, "references to this class"); 8517 SYSCTL_ADD_PROC(ctx, children2, OID_AUTO, "params", 8518 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 8519 (pi->port_id << 16) | i, sysctl_tc_params, "A", 8520 "traffic class parameters"); 8521 } 8522 8523 /* 8524 * dev.cxgbe.X.stats. 8525 */ 8526 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", 8527 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "port statistics"); 8528 children = SYSCTL_CHILDREN(oid); 8529 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD, 8530 &pi->tx_parse_error, 0, 8531 "# of tx packets with invalid length or # of segments"); 8532 8533 #define T4_LBSTAT(name, stat, desc) do { \ 8534 if (sc->params.tp.lb_mode) { \ 8535 SYSCTL_ADD_OID(ctx, children, OID_AUTO, #name, \ 8536 CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, pi, \ 8537 A_MPS_PORT_STAT_##stat##_L, \ 8538 sysctl_handle_t4_portstat64, "QU", desc); \ 8539 } else { \ 8540 SYSCTL_ADD_OID(ctx, children, OID_AUTO, #name, \ 8541 CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, \ 8542 t4_port_reg(sc, pi->tx_chan, A_MPS_PORT_STAT_##stat##_L), \ 8543 sysctl_handle_t4_reg64, "QU", desc); \ 8544 } \ 8545 } while (0) 8546 8547 T4_LBSTAT(tx_octets, TX_PORT_BYTES, "# of octets in good frames"); 8548 T4_LBSTAT(tx_frames, TX_PORT_FRAMES, "total # of good frames"); 8549 T4_LBSTAT(tx_bcast_frames, TX_PORT_BCAST, "# of broadcast frames"); 8550 T4_LBSTAT(tx_mcast_frames, TX_PORT_MCAST, "# of multicast frames"); 8551 T4_LBSTAT(tx_ucast_frames, TX_PORT_UCAST, "# of unicast frames"); 8552 T4_LBSTAT(tx_error_frames, TX_PORT_ERROR, "# of error frames"); 8553 T4_LBSTAT(tx_frames_64, TX_PORT_64B, "# of tx frames in this range"); 8554 T4_LBSTAT(tx_frames_65_127, TX_PORT_65B_127B, "# of tx frames in this range"); 8555 T4_LBSTAT(tx_frames_128_255, TX_PORT_128B_255B, "# of tx frames in this range"); 8556 T4_LBSTAT(tx_frames_256_511, TX_PORT_256B_511B, "# of tx frames in this range"); 8557 T4_LBSTAT(tx_frames_512_1023, TX_PORT_512B_1023B, "# of tx frames in this range"); 8558 T4_LBSTAT(tx_frames_1024_1518, TX_PORT_1024B_1518B, "# of tx frames in this range"); 8559 T4_LBSTAT(tx_frames_1519_max, TX_PORT_1519B_MAX, "# of tx frames in this range"); 8560 T4_LBSTAT(tx_drop, TX_PORT_DROP, "# of dropped tx frames"); 8561 T4_LBSTAT(tx_pause, TX_PORT_PAUSE, "# of pause frames transmitted"); 8562 T4_LBSTAT(tx_ppp0, TX_PORT_PPP0, "# of PPP prio 0 frames transmitted"); 8563 T4_LBSTAT(tx_ppp1, TX_PORT_PPP1, "# of PPP prio 1 frames transmitted"); 8564 T4_LBSTAT(tx_ppp2, TX_PORT_PPP2, "# of PPP prio 2 frames transmitted"); 8565 T4_LBSTAT(tx_ppp3, TX_PORT_PPP3, "# of PPP prio 3 frames transmitted"); 8566 T4_LBSTAT(tx_ppp4, TX_PORT_PPP4, "# of PPP prio 4 frames transmitted"); 8567 T4_LBSTAT(tx_ppp5, TX_PORT_PPP5, "# of PPP prio 5 frames transmitted"); 8568 T4_LBSTAT(tx_ppp6, TX_PORT_PPP6, "# of PPP prio 6 frames transmitted"); 8569 T4_LBSTAT(tx_ppp7, TX_PORT_PPP7, "# of PPP prio 7 frames transmitted"); 8570 8571 T4_LBSTAT(rx_octets, RX_PORT_BYTES, "# of octets in good frames"); 8572 T4_LBSTAT(rx_frames, RX_PORT_FRAMES, "total # of good frames"); 8573 T4_LBSTAT(rx_bcast_frames, RX_PORT_BCAST, "# of broadcast frames"); 8574 T4_LBSTAT(rx_mcast_frames, RX_PORT_MCAST, "# of multicast frames"); 8575 T4_LBSTAT(rx_ucast_frames, RX_PORT_UCAST, "# of unicast frames"); 8576 T4_LBSTAT(rx_too_long, RX_PORT_MTU_ERROR, "# of frames exceeding MTU"); 8577 T4_LBSTAT(rx_jabber, RX_PORT_MTU_CRC_ERROR, "# of jabber frames"); 8578 if (is_t6(sc)) { 8579 /* Read from port_stats and may be stale by up to 1s */ 8580 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "rx_fcs_err", 8581 CTLFLAG_RD, &pi->stats.rx_fcs_err, 8582 "# of frames received with bad FCS since last link up"); 8583 } else { 8584 T4_LBSTAT(rx_fcs_err, RX_PORT_CRC_ERROR, 8585 "# of frames received with bad FCS"); 8586 } 8587 T4_LBSTAT(rx_len_err, RX_PORT_LEN_ERROR, "# of frames received with length error"); 8588 T4_LBSTAT(rx_symbol_err, RX_PORT_SYM_ERROR, "symbol errors"); 8589 T4_LBSTAT(rx_runt, RX_PORT_LESS_64B, "# of short frames received"); 8590 T4_LBSTAT(rx_frames_64, RX_PORT_64B, "# of rx frames in this range"); 8591 T4_LBSTAT(rx_frames_65_127, RX_PORT_65B_127B, "# of rx frames in this range"); 8592 T4_LBSTAT(rx_frames_128_255, RX_PORT_128B_255B, "# of rx frames in this range"); 8593 T4_LBSTAT(rx_frames_256_511, RX_PORT_256B_511B, "# of rx frames in this range"); 8594 T4_LBSTAT(rx_frames_512_1023, RX_PORT_512B_1023B, "# of rx frames in this range"); 8595 T4_LBSTAT(rx_frames_1024_1518, RX_PORT_1024B_1518B, "# of rx frames in this range"); 8596 T4_LBSTAT(rx_frames_1519_max, RX_PORT_1519B_MAX, "# of rx frames in this range"); 8597 T4_LBSTAT(rx_pause, RX_PORT_PAUSE, "# of pause frames received"); 8598 T4_LBSTAT(rx_ppp0, RX_PORT_PPP0, "# of PPP prio 0 frames received"); 8599 T4_LBSTAT(rx_ppp1, RX_PORT_PPP1, "# of PPP prio 1 frames received"); 8600 T4_LBSTAT(rx_ppp2, RX_PORT_PPP2, "# of PPP prio 2 frames received"); 8601 T4_LBSTAT(rx_ppp3, RX_PORT_PPP3, "# of PPP prio 3 frames received"); 8602 T4_LBSTAT(rx_ppp4, RX_PORT_PPP4, "# of PPP prio 4 frames received"); 8603 T4_LBSTAT(rx_ppp5, RX_PORT_PPP5, "# of PPP prio 5 frames received"); 8604 T4_LBSTAT(rx_ppp6, RX_PORT_PPP6, "# of PPP prio 6 frames received"); 8605 T4_LBSTAT(rx_ppp7, RX_PORT_PPP7, "# of PPP prio 7 frames received"); 8606 #undef T4_LBSTAT 8607 8608 #define T4_REGSTAT(name, stat, desc) do { \ 8609 SYSCTL_ADD_OID(ctx, children, OID_AUTO, #name, \ 8610 CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, \ 8611 A_MPS_STAT_##stat##_L, sysctl_handle_t4_reg64, "QU", desc); \ 8612 } while (0) 8613 8614 if (pi->mps_bg_map & 1) { 8615 T4_REGSTAT(rx_ovflow0, RX_BG_0_MAC_DROP_FRAME, 8616 "# drops due to buffer-group 0 overflows"); 8617 T4_REGSTAT(rx_trunc0, RX_BG_0_MAC_TRUNC_FRAME, 8618 "# of buffer-group 0 truncated packets"); 8619 } 8620 if (pi->mps_bg_map & 2) { 8621 T4_REGSTAT(rx_ovflow1, RX_BG_1_MAC_DROP_FRAME, 8622 "# drops due to buffer-group 1 overflows"); 8623 T4_REGSTAT(rx_trunc1, RX_BG_1_MAC_TRUNC_FRAME, 8624 "# of buffer-group 1 truncated packets"); 8625 } 8626 if (pi->mps_bg_map & 4) { 8627 T4_REGSTAT(rx_ovflow2, RX_BG_2_MAC_DROP_FRAME, 8628 "# drops due to buffer-group 2 overflows"); 8629 T4_REGSTAT(rx_trunc2, RX_BG_2_MAC_TRUNC_FRAME, 8630 "# of buffer-group 2 truncated packets"); 8631 } 8632 if (pi->mps_bg_map & 8) { 8633 T4_REGSTAT(rx_ovflow3, RX_BG_3_MAC_DROP_FRAME, 8634 "# drops due to buffer-group 3 overflows"); 8635 T4_REGSTAT(rx_trunc3, RX_BG_3_MAC_TRUNC_FRAME, 8636 "# of buffer-group 3 truncated packets"); 8637 } 8638 #undef T4_REGSTAT 8639 } 8640 8641 static int 8642 sysctl_int_array(SYSCTL_HANDLER_ARGS) 8643 { 8644 int rc, *i, space = 0; 8645 struct sbuf sb; 8646 8647 sbuf_new_for_sysctl(&sb, NULL, 64, req); 8648 for (i = arg1; arg2; arg2 -= sizeof(int), i++) { 8649 if (space) 8650 sbuf_printf(&sb, " "); 8651 sbuf_printf(&sb, "%d", *i); 8652 space = 1; 8653 } 8654 rc = sbuf_finish(&sb); 8655 sbuf_delete(&sb); 8656 return (rc); 8657 } 8658 8659 static int 8660 sysctl_bitfield_8b(SYSCTL_HANDLER_ARGS) 8661 { 8662 int rc; 8663 struct sbuf *sb; 8664 8665 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 8666 if (sb == NULL) 8667 return (ENOMEM); 8668 8669 sbuf_printf(sb, "%b", *(uint8_t *)(uintptr_t)arg2, (char *)arg1); 8670 rc = sbuf_finish(sb); 8671 sbuf_delete(sb); 8672 8673 return (rc); 8674 } 8675 8676 static int 8677 sysctl_bitfield_16b(SYSCTL_HANDLER_ARGS) 8678 { 8679 int rc; 8680 struct sbuf *sb; 8681 8682 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 8683 if (sb == NULL) 8684 return (ENOMEM); 8685 8686 sbuf_printf(sb, "%b", *(uint16_t *)(uintptr_t)arg2, (char *)arg1); 8687 rc = sbuf_finish(sb); 8688 sbuf_delete(sb); 8689 8690 return (rc); 8691 } 8692 8693 static int 8694 sysctl_btphy(SYSCTL_HANDLER_ARGS) 8695 { 8696 struct port_info *pi = arg1; 8697 int op = arg2; 8698 struct adapter *sc = pi->adapter; 8699 u_int v; 8700 int rc; 8701 8702 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, "t4btt"); 8703 if (rc) 8704 return (rc); 8705 if (!hw_all_ok(sc)) 8706 rc = ENXIO; 8707 else { 8708 /* XXX: magic numbers */ 8709 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, 8710 op ? 0x20 : 0xc820, &v); 8711 } 8712 end_synchronized_op(sc, 0); 8713 if (rc) 8714 return (rc); 8715 if (op == 0) 8716 v /= 256; 8717 8718 rc = sysctl_handle_int(oidp, &v, 0, req); 8719 return (rc); 8720 } 8721 8722 static int 8723 sysctl_noflowq(SYSCTL_HANDLER_ARGS) 8724 { 8725 struct vi_info *vi = arg1; 8726 int rc, val; 8727 8728 val = vi->rsrv_noflowq; 8729 rc = sysctl_handle_int(oidp, &val, 0, req); 8730 if (rc != 0 || req->newptr == NULL) 8731 return (rc); 8732 8733 if ((val >= 1) && (vi->ntxq > 1)) 8734 vi->rsrv_noflowq = 1; 8735 else 8736 vi->rsrv_noflowq = 0; 8737 8738 return (rc); 8739 } 8740 8741 static int 8742 sysctl_tx_vm_wr(SYSCTL_HANDLER_ARGS) 8743 { 8744 struct vi_info *vi = arg1; 8745 struct adapter *sc = vi->adapter; 8746 int rc, val, i; 8747 8748 MPASS(!(sc->flags & IS_VF)); 8749 8750 val = vi->flags & TX_USES_VM_WR ? 1 : 0; 8751 rc = sysctl_handle_int(oidp, &val, 0, req); 8752 if (rc != 0 || req->newptr == NULL) 8753 return (rc); 8754 8755 if (val != 0 && val != 1) 8756 return (EINVAL); 8757 8758 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 8759 "t4txvm"); 8760 if (rc) 8761 return (rc); 8762 if (!hw_all_ok(sc)) 8763 rc = ENXIO; 8764 else if (if_getdrvflags(vi->ifp) & IFF_DRV_RUNNING) { 8765 /* 8766 * We don't want parse_pkt to run with one setting (VF or PF) 8767 * and then eth_tx to see a different setting but still use 8768 * stale information calculated by parse_pkt. 8769 */ 8770 rc = EBUSY; 8771 } else { 8772 struct port_info *pi = vi->pi; 8773 struct sge_txq *txq; 8774 uint32_t ctrl0; 8775 uint8_t npkt = sc->params.max_pkts_per_eth_tx_pkts_wr; 8776 8777 if (val) { 8778 vi->flags |= TX_USES_VM_WR; 8779 if_sethwtsomaxsegcount(vi->ifp, TX_SGL_SEGS_VM_TSO); 8780 ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) | 8781 V_TXPKT_INTF(pi->hw_port)); 8782 if (!(sc->flags & IS_VF)) 8783 npkt--; 8784 } else { 8785 vi->flags &= ~TX_USES_VM_WR; 8786 if_sethwtsomaxsegcount(vi->ifp, TX_SGL_SEGS_TSO); 8787 ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) | 8788 V_TXPKT_INTF(pi->hw_port) | V_TXPKT_PF(sc->pf) | 8789 V_TXPKT_VF(vi->vin) | V_TXPKT_VF_VLD(vi->vfvld)); 8790 } 8791 for_each_txq(vi, i, txq) { 8792 txq->cpl_ctrl0 = ctrl0; 8793 txq->txp.max_npkt = npkt; 8794 } 8795 } 8796 end_synchronized_op(sc, LOCK_HELD); 8797 return (rc); 8798 } 8799 8800 static int 8801 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 8802 { 8803 struct vi_info *vi = arg1; 8804 struct adapter *sc = vi->adapter; 8805 int idx, rc, i; 8806 struct sge_rxq *rxq; 8807 uint8_t v; 8808 8809 idx = vi->tmr_idx; 8810 8811 rc = sysctl_handle_int(oidp, &idx, 0, req); 8812 if (rc != 0 || req->newptr == NULL) 8813 return (rc); 8814 8815 if (idx < 0 || idx >= SGE_NTIMERS) 8816 return (EINVAL); 8817 8818 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 8819 "t4tmr"); 8820 if (rc) 8821 return (rc); 8822 8823 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(vi->pktc_idx != -1); 8824 for_each_rxq(vi, i, rxq) { 8825 #ifdef atomic_store_rel_8 8826 atomic_store_rel_8(&rxq->iq.intr_params, v); 8827 #else 8828 rxq->iq.intr_params = v; 8829 #endif 8830 } 8831 vi->tmr_idx = idx; 8832 8833 end_synchronized_op(sc, LOCK_HELD); 8834 return (0); 8835 } 8836 8837 static int 8838 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 8839 { 8840 struct vi_info *vi = arg1; 8841 struct adapter *sc = vi->adapter; 8842 int idx, rc; 8843 8844 idx = vi->pktc_idx; 8845 8846 rc = sysctl_handle_int(oidp, &idx, 0, req); 8847 if (rc != 0 || req->newptr == NULL) 8848 return (rc); 8849 8850 if (idx < -1 || idx >= SGE_NCOUNTERS) 8851 return (EINVAL); 8852 8853 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 8854 "t4pktc"); 8855 if (rc) 8856 return (rc); 8857 8858 if (vi->flags & VI_INIT_DONE) 8859 rc = EBUSY; /* cannot be changed once the queues are created */ 8860 else 8861 vi->pktc_idx = idx; 8862 8863 end_synchronized_op(sc, LOCK_HELD); 8864 return (rc); 8865 } 8866 8867 static int 8868 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 8869 { 8870 struct vi_info *vi = arg1; 8871 struct adapter *sc = vi->adapter; 8872 int qsize, rc; 8873 8874 qsize = vi->qsize_rxq; 8875 8876 rc = sysctl_handle_int(oidp, &qsize, 0, req); 8877 if (rc != 0 || req->newptr == NULL) 8878 return (rc); 8879 8880 if (qsize < 128 || (qsize & 7)) 8881 return (EINVAL); 8882 8883 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 8884 "t4rxqs"); 8885 if (rc) 8886 return (rc); 8887 8888 if (vi->flags & VI_INIT_DONE) 8889 rc = EBUSY; /* cannot be changed once the queues are created */ 8890 else 8891 vi->qsize_rxq = qsize; 8892 8893 end_synchronized_op(sc, LOCK_HELD); 8894 return (rc); 8895 } 8896 8897 static int 8898 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 8899 { 8900 struct vi_info *vi = arg1; 8901 struct adapter *sc = vi->adapter; 8902 int qsize, rc; 8903 8904 qsize = vi->qsize_txq; 8905 8906 rc = sysctl_handle_int(oidp, &qsize, 0, req); 8907 if (rc != 0 || req->newptr == NULL) 8908 return (rc); 8909 8910 if (qsize < 128 || qsize > 65536) 8911 return (EINVAL); 8912 8913 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 8914 "t4txqs"); 8915 if (rc) 8916 return (rc); 8917 8918 if (vi->flags & VI_INIT_DONE) 8919 rc = EBUSY; /* cannot be changed once the queues are created */ 8920 else 8921 vi->qsize_txq = qsize; 8922 8923 end_synchronized_op(sc, LOCK_HELD); 8924 return (rc); 8925 } 8926 8927 static int 8928 sysctl_pause_settings(SYSCTL_HANDLER_ARGS) 8929 { 8930 struct port_info *pi = arg1; 8931 struct adapter *sc = pi->adapter; 8932 struct link_config *lc = &pi->link_cfg; 8933 int rc; 8934 8935 if (req->newptr == NULL) { 8936 struct sbuf *sb; 8937 static char *bits = "\20\1RX\2TX\3AUTO"; 8938 8939 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 8940 if (sb == NULL) 8941 return (ENOMEM); 8942 8943 if (lc->link_ok) { 8944 sbuf_printf(sb, "%b", (lc->fc & (PAUSE_TX | PAUSE_RX)) | 8945 (lc->requested_fc & PAUSE_AUTONEG), bits); 8946 } else { 8947 sbuf_printf(sb, "%b", lc->requested_fc & (PAUSE_TX | 8948 PAUSE_RX | PAUSE_AUTONEG), bits); 8949 } 8950 rc = sbuf_finish(sb); 8951 sbuf_delete(sb); 8952 } else { 8953 char s[2]; 8954 int n; 8955 8956 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX | 8957 PAUSE_AUTONEG)); 8958 s[1] = 0; 8959 8960 rc = sysctl_handle_string(oidp, s, sizeof(s), req); 8961 if (rc != 0) 8962 return(rc); 8963 8964 if (s[1] != 0) 8965 return (EINVAL); 8966 if (s[0] < '0' || s[0] > '9') 8967 return (EINVAL); /* not a number */ 8968 n = s[0] - '0'; 8969 if (n & ~(PAUSE_TX | PAUSE_RX | PAUSE_AUTONEG)) 8970 return (EINVAL); /* some other bit is set too */ 8971 8972 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, 8973 "t4PAUSE"); 8974 if (rc) 8975 return (rc); 8976 if (hw_all_ok(sc)) { 8977 PORT_LOCK(pi); 8978 lc->requested_fc = n; 8979 fixup_link_config(pi); 8980 if (pi->up_vis > 0) 8981 rc = apply_link_config(pi); 8982 set_current_media(pi); 8983 PORT_UNLOCK(pi); 8984 } 8985 end_synchronized_op(sc, 0); 8986 } 8987 8988 return (rc); 8989 } 8990 8991 static int 8992 sysctl_link_fec(SYSCTL_HANDLER_ARGS) 8993 { 8994 struct port_info *pi = arg1; 8995 struct link_config *lc = &pi->link_cfg; 8996 int rc; 8997 struct sbuf *sb; 8998 8999 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 9000 if (sb == NULL) 9001 return (ENOMEM); 9002 if (lc->link_ok) 9003 sbuf_printf(sb, "%b", lc->fec, t4_fec_bits); 9004 else 9005 sbuf_printf(sb, "no link"); 9006 rc = sbuf_finish(sb); 9007 sbuf_delete(sb); 9008 9009 return (rc); 9010 } 9011 9012 static int 9013 sysctl_requested_fec(SYSCTL_HANDLER_ARGS) 9014 { 9015 struct port_info *pi = arg1; 9016 struct adapter *sc = pi->adapter; 9017 struct link_config *lc = &pi->link_cfg; 9018 int rc; 9019 int8_t old = lc->requested_fec; 9020 9021 if (req->newptr == NULL) { 9022 struct sbuf *sb; 9023 9024 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 9025 if (sb == NULL) 9026 return (ENOMEM); 9027 9028 sbuf_printf(sb, "%b", old, t4_fec_bits); 9029 rc = sbuf_finish(sb); 9030 sbuf_delete(sb); 9031 } else { 9032 char s[8]; 9033 int n; 9034 9035 snprintf(s, sizeof(s), "%d", old == FEC_AUTO ? -1 : 9036 old & (M_FW_PORT_CAP32_FEC | FEC_MODULE)); 9037 9038 rc = sysctl_handle_string(oidp, s, sizeof(s), req); 9039 if (rc != 0) 9040 return(rc); 9041 9042 n = strtol(&s[0], NULL, 0); 9043 if (n < 0 || n & FEC_AUTO) 9044 n = FEC_AUTO; 9045 else if (n & ~(M_FW_PORT_CAP32_FEC | FEC_MODULE)) 9046 return (EINVAL);/* some other bit is set too */ 9047 9048 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, 9049 "t4reqf"); 9050 if (rc) 9051 return (rc); 9052 PORT_LOCK(pi); 9053 if (lc->requested_fec != old) { 9054 rc = EBUSY; 9055 goto done; 9056 } 9057 if (n == FEC_AUTO) 9058 lc->requested_fec = FEC_AUTO; 9059 else if (n == 0 || n == FEC_NONE) 9060 lc->requested_fec = FEC_NONE; 9061 else { 9062 if ((lc->pcaps | 9063 V_FW_PORT_CAP32_FEC(n & M_FW_PORT_CAP32_FEC)) != 9064 lc->pcaps) { 9065 rc = ENOTSUP; 9066 goto done; 9067 } 9068 lc->requested_fec = n & (M_FW_PORT_CAP32_FEC | 9069 FEC_MODULE); 9070 } 9071 if (hw_all_ok(sc)) { 9072 fixup_link_config(pi); 9073 if (pi->up_vis > 0) { 9074 rc = apply_link_config(pi); 9075 if (rc != 0) { 9076 lc->requested_fec = old; 9077 if (rc == FW_EPROTO) 9078 rc = ENOTSUP; 9079 } 9080 } 9081 } 9082 done: 9083 PORT_UNLOCK(pi); 9084 end_synchronized_op(sc, 0); 9085 } 9086 9087 return (rc); 9088 } 9089 9090 static int 9091 sysctl_module_fec(SYSCTL_HANDLER_ARGS) 9092 { 9093 struct port_info *pi = arg1; 9094 struct adapter *sc = pi->adapter; 9095 struct link_config *lc = &pi->link_cfg; 9096 int rc; 9097 int8_t fec; 9098 struct sbuf *sb; 9099 9100 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 9101 if (sb == NULL) 9102 return (ENOMEM); 9103 9104 if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4mfec") != 0) { 9105 rc = EBUSY; 9106 goto done; 9107 } 9108 if (!hw_all_ok(sc)) { 9109 rc = ENXIO; 9110 goto done; 9111 } 9112 PORT_LOCK(pi); 9113 if (pi->up_vis == 0) { 9114 /* 9115 * If all the interfaces are administratively down the firmware 9116 * does not report transceiver changes. Refresh port info here. 9117 * This is the only reason we have a synchronized op in this 9118 * function. Just PORT_LOCK would have been enough otherwise. 9119 */ 9120 t4_update_port_info(pi); 9121 } 9122 9123 fec = lc->fec_hint; 9124 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE || 9125 !fec_supported(lc->pcaps)) { 9126 PORT_UNLOCK(pi); 9127 sbuf_printf(sb, "n/a"); 9128 } else { 9129 if (fec == 0) 9130 fec = FEC_NONE; 9131 PORT_UNLOCK(pi); 9132 sbuf_printf(sb, "%b", fec & M_FW_PORT_CAP32_FEC, t4_fec_bits); 9133 } 9134 rc = sbuf_finish(sb); 9135 done: 9136 sbuf_delete(sb); 9137 end_synchronized_op(sc, 0); 9138 9139 return (rc); 9140 } 9141 9142 static int 9143 sysctl_autoneg(SYSCTL_HANDLER_ARGS) 9144 { 9145 struct port_info *pi = arg1; 9146 struct adapter *sc = pi->adapter; 9147 struct link_config *lc = &pi->link_cfg; 9148 int rc, val; 9149 9150 if (lc->pcaps & FW_PORT_CAP32_ANEG) 9151 val = lc->requested_aneg == AUTONEG_DISABLE ? 0 : 1; 9152 else 9153 val = -1; 9154 rc = sysctl_handle_int(oidp, &val, 0, req); 9155 if (rc != 0 || req->newptr == NULL) 9156 return (rc); 9157 if (val == 0) 9158 val = AUTONEG_DISABLE; 9159 else if (val == 1) 9160 val = AUTONEG_ENABLE; 9161 else 9162 val = AUTONEG_AUTO; 9163 9164 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, 9165 "t4aneg"); 9166 if (rc) 9167 return (rc); 9168 PORT_LOCK(pi); 9169 if (val == AUTONEG_ENABLE && !(lc->pcaps & FW_PORT_CAP32_ANEG)) { 9170 rc = ENOTSUP; 9171 goto done; 9172 } 9173 lc->requested_aneg = val; 9174 if (hw_all_ok(sc)) { 9175 fixup_link_config(pi); 9176 if (pi->up_vis > 0) 9177 rc = apply_link_config(pi); 9178 set_current_media(pi); 9179 } 9180 done: 9181 PORT_UNLOCK(pi); 9182 end_synchronized_op(sc, 0); 9183 return (rc); 9184 } 9185 9186 static int 9187 sysctl_force_fec(SYSCTL_HANDLER_ARGS) 9188 { 9189 struct port_info *pi = arg1; 9190 struct adapter *sc = pi->adapter; 9191 struct link_config *lc = &pi->link_cfg; 9192 int rc, val; 9193 9194 val = lc->force_fec; 9195 MPASS(val >= -1 && val <= 1); 9196 rc = sysctl_handle_int(oidp, &val, 0, req); 9197 if (rc != 0 || req->newptr == NULL) 9198 return (rc); 9199 if (!(lc->pcaps & FW_PORT_CAP32_FORCE_FEC)) 9200 return (ENOTSUP); 9201 if (val < -1 || val > 1) 9202 return (EINVAL); 9203 9204 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, "t4ff"); 9205 if (rc) 9206 return (rc); 9207 PORT_LOCK(pi); 9208 lc->force_fec = val; 9209 if (hw_all_ok(sc)) { 9210 fixup_link_config(pi); 9211 if (pi->up_vis > 0) 9212 rc = apply_link_config(pi); 9213 } 9214 PORT_UNLOCK(pi); 9215 end_synchronized_op(sc, 0); 9216 return (rc); 9217 } 9218 9219 static int 9220 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 9221 { 9222 struct adapter *sc = arg1; 9223 int rc, reg = arg2; 9224 uint64_t val; 9225 9226 mtx_lock(&sc->reg_lock); 9227 if (hw_off_limits(sc)) 9228 rc = ENXIO; 9229 else { 9230 rc = 0; 9231 val = t4_read_reg64(sc, reg); 9232 } 9233 mtx_unlock(&sc->reg_lock); 9234 if (rc == 0) 9235 rc = sysctl_handle_64(oidp, &val, 0, req); 9236 return (rc); 9237 } 9238 9239 static int 9240 sysctl_handle_t4_portstat64(SYSCTL_HANDLER_ARGS) 9241 { 9242 struct port_info *pi = arg1; 9243 struct adapter *sc = pi->adapter; 9244 int rc, i, reg = arg2; 9245 uint64_t val; 9246 9247 mtx_lock(&sc->reg_lock); 9248 if (hw_off_limits(sc)) 9249 rc = ENXIO; 9250 else { 9251 val = 0; 9252 for (i = 0; i < sc->params.tp.lb_nchan; i++) { 9253 val += t4_read_reg64(sc, 9254 t4_port_reg(sc, pi->tx_chan + i, reg)); 9255 } 9256 rc = 0; 9257 } 9258 mtx_unlock(&sc->reg_lock); 9259 if (rc == 0) 9260 rc = sysctl_handle_64(oidp, &val, 0, req); 9261 return (rc); 9262 } 9263 9264 static int 9265 sysctl_temperature(SYSCTL_HANDLER_ARGS) 9266 { 9267 struct adapter *sc = arg1; 9268 int rc, t; 9269 uint32_t param, val; 9270 9271 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); 9272 if (rc) 9273 return (rc); 9274 if (!hw_all_ok(sc)) 9275 rc = ENXIO; 9276 else { 9277 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 9278 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 9279 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); 9280 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 9281 } 9282 end_synchronized_op(sc, 0); 9283 if (rc) 9284 return (rc); 9285 9286 /* unknown is returned as 0 but we display -1 in that case */ 9287 t = val == 0 ? -1 : val; 9288 9289 rc = sysctl_handle_int(oidp, &t, 0, req); 9290 return (rc); 9291 } 9292 9293 static int 9294 sysctl_vdd(SYSCTL_HANDLER_ARGS) 9295 { 9296 struct adapter *sc = arg1; 9297 int rc; 9298 uint32_t param, val; 9299 9300 if (sc->params.core_vdd == 0) { 9301 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 9302 "t4vdd"); 9303 if (rc) 9304 return (rc); 9305 if (!hw_all_ok(sc)) 9306 rc = ENXIO; 9307 else { 9308 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 9309 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 9310 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_VDD); 9311 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, 9312 ¶m, &val); 9313 } 9314 end_synchronized_op(sc, 0); 9315 if (rc) 9316 return (rc); 9317 sc->params.core_vdd = val; 9318 } 9319 9320 return (sysctl_handle_int(oidp, &sc->params.core_vdd, 0, req)); 9321 } 9322 9323 static int 9324 sysctl_reset_sensor(SYSCTL_HANDLER_ARGS) 9325 { 9326 struct adapter *sc = arg1; 9327 int rc, v; 9328 uint32_t param, val; 9329 9330 v = sc->sensor_resets; 9331 rc = sysctl_handle_int(oidp, &v, 0, req); 9332 if (rc != 0 || req->newptr == NULL || v <= 0) 9333 return (rc); 9334 9335 if (sc->params.fw_vers < FW_VERSION32(1, 24, 7, 0) || 9336 chip_id(sc) < CHELSIO_T5) 9337 return (ENOTSUP); 9338 9339 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4srst"); 9340 if (rc) 9341 return (rc); 9342 if (!hw_all_ok(sc)) 9343 rc = ENXIO; 9344 else { 9345 param = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 9346 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 9347 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_RESET_TMP_SENSOR)); 9348 val = 1; 9349 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 9350 } 9351 end_synchronized_op(sc, 0); 9352 if (rc == 0) 9353 sc->sensor_resets++; 9354 return (rc); 9355 } 9356 9357 static int 9358 sysctl_loadavg(SYSCTL_HANDLER_ARGS) 9359 { 9360 struct adapter *sc = arg1; 9361 struct sbuf *sb; 9362 int rc; 9363 uint32_t param, val; 9364 uint8_t coreid = (uint8_t)arg2; 9365 9366 KASSERT(coreid < sc->params.ncores, 9367 ("%s: bad coreid %u\n", __func__, coreid)); 9368 9369 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4lavg"); 9370 if (rc) 9371 return (rc); 9372 if (!hw_all_ok(sc)) 9373 rc = ENXIO; 9374 else { 9375 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 9376 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_LOAD) | 9377 V_FW_PARAMS_PARAM_Y(coreid); 9378 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 9379 } 9380 end_synchronized_op(sc, 0); 9381 if (rc) 9382 return (rc); 9383 9384 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 9385 if (sb == NULL) 9386 return (ENOMEM); 9387 9388 if (val == 0xffffffff) { 9389 /* Only debug and custom firmwares report load averages. */ 9390 sbuf_printf(sb, "not available"); 9391 } else { 9392 sbuf_printf(sb, "%d %d %d", val & 0xff, (val >> 8) & 0xff, 9393 (val >> 16) & 0xff); 9394 } 9395 rc = sbuf_finish(sb); 9396 sbuf_delete(sb); 9397 9398 return (rc); 9399 } 9400 9401 static int 9402 sysctl_cctrl(SYSCTL_HANDLER_ARGS) 9403 { 9404 struct adapter *sc = arg1; 9405 struct sbuf *sb; 9406 int rc, i; 9407 uint16_t incr[NMTUS][NCCTRL_WIN]; 9408 static const char *dec_fac[] = { 9409 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 9410 "0.9375" 9411 }; 9412 9413 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 9414 if (sb == NULL) 9415 return (ENOMEM); 9416 9417 rc = 0; 9418 mtx_lock(&sc->reg_lock); 9419 if (hw_off_limits(sc)) 9420 rc = ENXIO; 9421 else 9422 t4_read_cong_tbl(sc, incr); 9423 mtx_unlock(&sc->reg_lock); 9424 if (rc) 9425 goto done; 9426 9427 for (i = 0; i < NCCTRL_WIN; ++i) { 9428 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 9429 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 9430 incr[5][i], incr[6][i], incr[7][i]); 9431 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 9432 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 9433 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 9434 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 9435 } 9436 9437 rc = sbuf_finish(sb); 9438 done: 9439 sbuf_delete(sb); 9440 return (rc); 9441 } 9442 9443 static int 9444 sysctl_cim_ibq(SYSCTL_HANDLER_ARGS) 9445 { 9446 struct adapter *sc = arg1; 9447 struct sbuf *sb; 9448 int rc, i, n, qid, coreid; 9449 uint32_t *buf, *p; 9450 9451 qid = arg2 & 0xffff; 9452 coreid = arg2 >> 16; 9453 9454 KASSERT(qid >= 0 && qid < sc->chip_params->cim_num_ibq, 9455 ("%s: bad ibq qid %d\n", __func__, qid)); 9456 KASSERT(coreid >= 0 && coreid < sc->params.ncores, 9457 ("%s: bad coreid %d\n", __func__, coreid)); 9458 9459 n = 4 * CIM_IBQ_SIZE; 9460 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 9461 mtx_lock(&sc->reg_lock); 9462 if (hw_off_limits(sc)) 9463 rc = -ENXIO; 9464 else 9465 rc = t4_read_cim_ibq_core(sc, coreid, qid, buf, n); 9466 mtx_unlock(&sc->reg_lock); 9467 if (rc < 0) { 9468 rc = -rc; 9469 goto done; 9470 } 9471 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 9472 9473 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 9474 if (sb == NULL) { 9475 rc = ENOMEM; 9476 goto done; 9477 } 9478 for (i = 0, p = buf; i < n; i += 16, p += 4) 9479 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 9480 p[2], p[3]); 9481 rc = sbuf_finish(sb); 9482 sbuf_delete(sb); 9483 done: 9484 free(buf, M_CXGBE); 9485 return (rc); 9486 } 9487 9488 static int 9489 sysctl_cim_obq(SYSCTL_HANDLER_ARGS) 9490 { 9491 struct adapter *sc = arg1; 9492 struct sbuf *sb; 9493 int rc, i, n, qid, coreid; 9494 uint32_t *buf, *p; 9495 9496 qid = arg2 & 0xffff; 9497 coreid = arg2 >> 16; 9498 9499 KASSERT(qid >= 0 && qid < sc->chip_params->cim_num_obq, 9500 ("%s: bad obq qid %d\n", __func__, qid)); 9501 KASSERT(coreid >= 0 && coreid < sc->params.ncores, 9502 ("%s: bad coreid %d\n", __func__, coreid)); 9503 9504 n = 6 * CIM_OBQ_SIZE * 4; 9505 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 9506 mtx_lock(&sc->reg_lock); 9507 if (hw_off_limits(sc)) 9508 rc = -ENXIO; 9509 else 9510 rc = t4_read_cim_obq_core(sc, coreid, qid, buf, n); 9511 mtx_unlock(&sc->reg_lock); 9512 if (rc < 0) { 9513 rc = -rc; 9514 goto done; 9515 } 9516 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 9517 9518 rc = sysctl_wire_old_buffer(req, 0); 9519 if (rc != 0) 9520 goto done; 9521 9522 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 9523 if (sb == NULL) { 9524 rc = ENOMEM; 9525 goto done; 9526 } 9527 for (i = 0, p = buf; i < n; i += 16, p += 4) 9528 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 9529 p[2], p[3]); 9530 rc = sbuf_finish(sb); 9531 sbuf_delete(sb); 9532 done: 9533 free(buf, M_CXGBE); 9534 return (rc); 9535 } 9536 9537 static void 9538 sbuf_cim_la4(struct adapter *sc, struct sbuf *sb, uint32_t *buf, uint32_t cfg) 9539 { 9540 uint32_t *p; 9541 9542 sbuf_printf(sb, "Status Data PC%s", 9543 cfg & F_UPDBGLACAPTPCONLY ? "" : 9544 " LS0Stat LS0Addr LS0Data"); 9545 9546 for (p = buf; p <= &buf[sc->params.cim_la_size - 8]; p += 8) { 9547 if (cfg & F_UPDBGLACAPTPCONLY) { 9548 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 9549 p[6], p[7]); 9550 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 9551 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 9552 p[4] & 0xff, p[5] >> 8); 9553 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 9554 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 9555 p[1] & 0xf, p[2] >> 4); 9556 } else { 9557 sbuf_printf(sb, 9558 "\n %02x %x%07x %x%07x %08x %08x " 9559 "%08x%08x%08x%08x", 9560 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 9561 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 9562 p[6], p[7]); 9563 } 9564 } 9565 } 9566 9567 static void 9568 sbuf_cim_la6(struct adapter *sc, struct sbuf *sb, uint32_t *buf, uint32_t cfg) 9569 { 9570 uint32_t *p; 9571 9572 sbuf_printf(sb, "Status Inst Data PC%s", 9573 cfg & F_UPDBGLACAPTPCONLY ? "" : 9574 " LS0Stat LS0Addr LS0Data LS1Stat LS1Addr LS1Data"); 9575 9576 for (p = buf; p <= &buf[sc->params.cim_la_size - 10]; p += 10) { 9577 if (cfg & F_UPDBGLACAPTPCONLY) { 9578 sbuf_printf(sb, "\n %02x %08x %08x %08x", 9579 p[3] & 0xff, p[2], p[1], p[0]); 9580 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x %02x%06x", 9581 (p[6] >> 8) & 0xff, p[6] & 0xff, p[5] >> 8, 9582 p[5] & 0xff, p[4] >> 8, p[4] & 0xff, p[3] >> 8); 9583 sbuf_printf(sb, "\n %02x %04x%04x %04x%04x %04x%04x", 9584 (p[9] >> 16) & 0xff, p[9] & 0xffff, p[8] >> 16, 9585 p[8] & 0xffff, p[7] >> 16, p[7] & 0xffff, 9586 p[6] >> 16); 9587 } else { 9588 sbuf_printf(sb, "\n %02x %04x%04x %04x%04x %04x%04x " 9589 "%08x %08x %08x %08x %08x %08x", 9590 (p[9] >> 16) & 0xff, 9591 p[9] & 0xffff, p[8] >> 16, 9592 p[8] & 0xffff, p[7] >> 16, 9593 p[7] & 0xffff, p[6] >> 16, 9594 p[2], p[1], p[0], p[5], p[4], p[3]); 9595 } 9596 } 9597 } 9598 9599 static int 9600 sbuf_cim_la(struct adapter *sc, int coreid, struct sbuf *sb, int flags) 9601 { 9602 uint32_t cfg, *buf; 9603 int rc; 9604 9605 MPASS(flags == M_WAITOK || flags == M_NOWAIT); 9606 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 9607 M_ZERO | flags); 9608 if (buf == NULL) 9609 return (ENOMEM); 9610 9611 mtx_lock(&sc->reg_lock); 9612 if (hw_off_limits(sc)) 9613 rc = ENXIO; 9614 else { 9615 rc = -t4_cim_read_core(sc, 1, coreid, A_UP_UP_DBG_LA_CFG, 1, 9616 &cfg); 9617 if (rc == 0) 9618 rc = -t4_cim_read_la_core(sc, coreid, buf, NULL); 9619 } 9620 mtx_unlock(&sc->reg_lock); 9621 if (rc == 0) { 9622 if (chip_id(sc) < CHELSIO_T6) 9623 sbuf_cim_la4(sc, sb, buf, cfg); 9624 else 9625 sbuf_cim_la6(sc, sb, buf, cfg); 9626 } 9627 free(buf, M_CXGBE); 9628 return (rc); 9629 } 9630 9631 static int 9632 sysctl_cim_la(SYSCTL_HANDLER_ARGS) 9633 { 9634 struct adapter *sc = arg1; 9635 int coreid = arg2; 9636 struct sbuf *sb; 9637 int rc; 9638 9639 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 9640 if (sb == NULL) 9641 return (ENOMEM); 9642 9643 rc = sbuf_cim_la(sc, coreid, sb, M_WAITOK); 9644 if (rc == 0) 9645 rc = sbuf_finish(sb); 9646 sbuf_delete(sb); 9647 return (rc); 9648 } 9649 9650 static void 9651 dump_cim_regs(struct adapter *sc) 9652 { 9653 log(LOG_DEBUG, "%s: CIM debug regs1 %08x %08x %08x %08x %08x\n", 9654 device_get_nameunit(sc->dev), 9655 t4_read_reg(sc, A_EDC_H_BIST_USER_WDATA0), 9656 t4_read_reg(sc, A_EDC_H_BIST_USER_WDATA1), 9657 t4_read_reg(sc, A_EDC_H_BIST_USER_WDATA2), 9658 t4_read_reg(sc, A_EDC_H_BIST_DATA_PATTERN), 9659 t4_read_reg(sc, A_EDC_H_BIST_STATUS_RDATA)); 9660 log(LOG_DEBUG, "%s: CIM debug regs2 %08x %08x %08x %08x %08x\n", 9661 device_get_nameunit(sc->dev), 9662 t4_read_reg(sc, A_EDC_H_BIST_USER_WDATA0), 9663 t4_read_reg(sc, A_EDC_H_BIST_USER_WDATA1), 9664 t4_read_reg(sc, A_EDC_H_BIST_USER_WDATA0 + 0x800), 9665 t4_read_reg(sc, A_EDC_H_BIST_USER_WDATA1 + 0x800), 9666 t4_read_reg(sc, A_EDC_H_BIST_CMD_LEN)); 9667 } 9668 9669 static void 9670 dump_cimla(struct adapter *sc) 9671 { 9672 struct sbuf sb; 9673 int rc; 9674 9675 if (sbuf_new(&sb, NULL, 4096, SBUF_AUTOEXTEND) != &sb) { 9676 log(LOG_DEBUG, "%s: failed to generate CIM LA dump.\n", 9677 device_get_nameunit(sc->dev)); 9678 return; 9679 } 9680 rc = sbuf_cim_la(sc, 0, &sb, M_WAITOK); 9681 if (rc == 0) { 9682 rc = sbuf_finish(&sb); 9683 if (rc == 0) { 9684 log(LOG_DEBUG, "%s: CIM LA dump follows.\n%s\n", 9685 device_get_nameunit(sc->dev), sbuf_data(&sb)); 9686 } 9687 } 9688 sbuf_delete(&sb); 9689 } 9690 9691 void 9692 t4_os_cim_err(struct adapter *sc) 9693 { 9694 atomic_set_int(&sc->error_flags, ADAP_CIM_ERR); 9695 } 9696 9697 static int 9698 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) 9699 { 9700 struct adapter *sc = arg1; 9701 u_int i; 9702 struct sbuf *sb; 9703 uint32_t *buf, *p; 9704 int rc; 9705 9706 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 9707 if (sb == NULL) 9708 return (ENOMEM); 9709 9710 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, 9711 M_ZERO | M_WAITOK); 9712 9713 rc = 0; 9714 mtx_lock(&sc->reg_lock); 9715 if (hw_off_limits(sc)) 9716 rc = ENXIO; 9717 else 9718 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); 9719 mtx_unlock(&sc->reg_lock); 9720 if (rc) 9721 goto done; 9722 9723 p = buf; 9724 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 9725 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], 9726 p[1], p[0]); 9727 } 9728 9729 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); 9730 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 9731 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", 9732 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, 9733 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, 9734 (p[1] >> 2) | ((p[2] & 3) << 30), 9735 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, 9736 p[0] & 1); 9737 } 9738 rc = sbuf_finish(sb); 9739 done: 9740 sbuf_delete(sb); 9741 free(buf, M_CXGBE); 9742 return (rc); 9743 } 9744 9745 static int 9746 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) 9747 { 9748 struct adapter *sc = arg1; 9749 u_int i; 9750 struct sbuf *sb; 9751 uint32_t *buf, *p; 9752 int rc; 9753 9754 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 9755 if (sb == NULL) 9756 return (ENOMEM); 9757 9758 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, 9759 M_ZERO | M_WAITOK); 9760 9761 rc = 0; 9762 mtx_lock(&sc->reg_lock); 9763 if (hw_off_limits(sc)) 9764 rc = ENXIO; 9765 else 9766 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); 9767 mtx_unlock(&sc->reg_lock); 9768 if (rc) 9769 goto done; 9770 9771 p = buf; 9772 sbuf_printf(sb, "Cntl ID DataBE Addr Data"); 9773 for (i = 0; i < CIM_PIFLA_SIZE; i++, p += 6) { 9774 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", 9775 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, 9776 p[4], p[3], p[2], p[1], p[0]); 9777 } 9778 9779 sbuf_printf(sb, "\n\nCntl ID Data"); 9780 for (i = 0; i < CIM_PIFLA_SIZE; i++, p += 6) { 9781 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", 9782 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); 9783 } 9784 9785 rc = sbuf_finish(sb); 9786 done: 9787 sbuf_delete(sb); 9788 free(buf, M_CXGBE); 9789 return (rc); 9790 } 9791 9792 static int 9793 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 9794 { 9795 struct adapter *sc = arg1; 9796 struct sbuf *sb; 9797 int rc, i; 9798 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 9799 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 9800 uint16_t thres[CIM_NUM_IBQ]; 9801 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 9802 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 9803 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 9804 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 9805 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 9806 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 9807 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 9808 }; 9809 9810 MPASS(chip_id(sc) < CHELSIO_T7); 9811 9812 cim_num_obq = sc->chip_params->cim_num_obq; 9813 if (is_t4(sc)) { 9814 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 9815 obq_rdaddr = A_UP_OBQ_0_REALADDR; 9816 } else { 9817 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 9818 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 9819 } 9820 nq = CIM_NUM_IBQ + cim_num_obq; 9821 9822 mtx_lock(&sc->reg_lock); 9823 if (hw_off_limits(sc)) 9824 rc = ENXIO; 9825 else { 9826 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 9827 if (rc == 0) { 9828 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, 9829 obq_wr); 9830 if (rc == 0) 9831 t4_read_cimq_cfg(sc, base, size, thres); 9832 } 9833 } 9834 mtx_unlock(&sc->reg_lock); 9835 if (rc) 9836 return (rc); 9837 9838 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 9839 if (sb == NULL) 9840 return (ENOMEM); 9841 9842 sbuf_printf(sb, 9843 " Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 9844 9845 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 9846 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 9847 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 9848 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 9849 G_QUEREMFLITS(p[2]) * 16); 9850 for ( ; i < nq; i++, p += 4, wr += 2) 9851 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 9852 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 9853 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 9854 G_QUEREMFLITS(p[2]) * 16); 9855 9856 rc = sbuf_finish(sb); 9857 sbuf_delete(sb); 9858 9859 return (rc); 9860 } 9861 9862 static int 9863 sysctl_cim_qcfg_t7(SYSCTL_HANDLER_ARGS) 9864 { 9865 struct adapter *sc = arg1; 9866 u_int coreid = arg2; 9867 struct sbuf *sb; 9868 int rc, i; 9869 u_int addr; 9870 uint16_t base[CIM_NUM_IBQ_T7 + CIM_NUM_OBQ_T7]; 9871 uint16_t size[CIM_NUM_IBQ_T7 + CIM_NUM_OBQ_T7]; 9872 uint16_t thres[CIM_NUM_IBQ_T7]; 9873 uint32_t obq_wr[2 * CIM_NUM_OBQ_T7], *wr = obq_wr; 9874 uint32_t stat[4 * (CIM_NUM_IBQ_T7 + CIM_NUM_OBQ_T7)], *p = stat; 9875 static const char * const qname_ibq_t7[] = { 9876 "TP0", "TP1", "TP2", "TP3", "ULP", "SGE0", "SGE1", "NC-SI", 9877 "RSVD", "IPC1", "IPC2", "IPC3", "IPC4", "IPC5", "IPC6", "IPC7", 9878 }; 9879 static const char * const qname_obq_t7[] = { 9880 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", "SGE0-RX", 9881 "RSVD", "RSVD", "IPC1", "IPC2", "IPC3", "IPC4", "IPC5", 9882 "IPC6", "IPC7" 9883 }; 9884 static const char * const qname_ibq_sec_t7[] = { 9885 "TP0", "TP1", "TP2", "TP3", "ULP", "SGE0", "RSVD", "RSVD", 9886 "RSVD", "IPC0", "RSVD", "RSVD", "RSVD", "RSVD", "RSVD", "RSVD", 9887 }; 9888 static const char * const qname_obq_sec_t7[] = { 9889 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "RSVD", "SGE0-RX", 9890 "RSVD", "RSVD", "IPC0", "RSVD", "RSVD", "RSVD", "RSVD", 9891 "RSVD", "RSVD", 9892 }; 9893 9894 MPASS(chip_id(sc) >= CHELSIO_T7); 9895 9896 mtx_lock(&sc->reg_lock); 9897 if (hw_off_limits(sc)) 9898 rc = ENXIO; 9899 else { 9900 rc = -t4_cim_read_core(sc, 1, coreid, 9901 A_T7_UP_IBQ_0_SHADOW_RDADDR, 4 * CIM_NUM_IBQ_T7, stat); 9902 if (rc != 0) 9903 goto unlock; 9904 9905 rc = -t4_cim_read_core(sc, 1, coreid, 9906 A_T7_UP_OBQ_0_SHADOW_RDADDR, 4 * CIM_NUM_OBQ_T7, 9907 &stat[4 * CIM_NUM_IBQ_T7]); 9908 if (rc != 0) 9909 goto unlock; 9910 9911 addr = A_T7_UP_OBQ_0_SHADOW_REALADDR; 9912 for (i = 0; i < CIM_NUM_OBQ_T7 * 2; i++, addr += 8) { 9913 rc = -t4_cim_read_core(sc, 1, coreid, addr, 1, 9914 &obq_wr[i]); 9915 if (rc != 0) 9916 goto unlock; 9917 } 9918 t4_read_cimq_cfg_core(sc, coreid, base, size, thres); 9919 } 9920 unlock: 9921 mtx_unlock(&sc->reg_lock); 9922 if (rc) 9923 return (rc); 9924 9925 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 9926 if (sb == NULL) 9927 return (ENOMEM); 9928 9929 sbuf_printf(sb, 9930 " Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 9931 9932 for (i = 0; i < CIM_NUM_IBQ_T7; i++, p += 4) { 9933 if (!size[i]) 9934 continue; 9935 9936 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 9937 coreid == 0 ? qname_ibq_t7[i] : qname_ibq_sec_t7[i], 9938 base[i], size[i], thres[i], G_IBQRDADDR(p[0]) & 0xfff, 9939 G_IBQWRADDR(p[1]) & 0xfff, G_QUESOPCNT(p[3]), 9940 G_QUEEOPCNT(p[3]), G_T7_QUEREMFLITS(p[2]) * 16); 9941 } 9942 9943 for ( ; i < CIM_NUM_IBQ_T7 + CIM_NUM_OBQ_T7; i++, p += 4, wr += 2) { 9944 if (!size[i]) 9945 continue; 9946 9947 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", 9948 coreid == 0 ? qname_obq_t7[i - CIM_NUM_IBQ_T7] : 9949 qname_obq_sec_t7[i - CIM_NUM_IBQ_T7], 9950 base[i], size[i], G_QUERDADDR(p[0]) & 0xfff, 9951 wr[0] << 1, G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 9952 G_T7_QUEREMFLITS(p[2]) * 16); 9953 } 9954 9955 rc = sbuf_finish(sb); 9956 sbuf_delete(sb); 9957 return (rc); 9958 } 9959 9960 static int 9961 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 9962 { 9963 struct adapter *sc = arg1; 9964 struct sbuf *sb; 9965 int rc; 9966 struct tp_cpl_stats stats; 9967 9968 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 9969 if (sb == NULL) 9970 return (ENOMEM); 9971 9972 rc = 0; 9973 mtx_lock(&sc->reg_lock); 9974 if (hw_off_limits(sc)) 9975 rc = ENXIO; 9976 else 9977 t4_tp_get_cpl_stats(sc, &stats, 0); 9978 mtx_unlock(&sc->reg_lock); 9979 if (rc) 9980 goto done; 9981 9982 if (sc->chip_params->nchan > 2) { 9983 sbuf_printf(sb, " channel 0 channel 1" 9984 " channel 2 channel 3"); 9985 sbuf_printf(sb, "\nCPL requests: %10u %10u %10u %10u", 9986 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 9987 sbuf_printf(sb, "\nCPL responses: %10u %10u %10u %10u", 9988 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 9989 } else { 9990 sbuf_printf(sb, " channel 0 channel 1"); 9991 sbuf_printf(sb, "\nCPL requests: %10u %10u", 9992 stats.req[0], stats.req[1]); 9993 sbuf_printf(sb, "\nCPL responses: %10u %10u", 9994 stats.rsp[0], stats.rsp[1]); 9995 } 9996 9997 rc = sbuf_finish(sb); 9998 done: 9999 sbuf_delete(sb); 10000 return (rc); 10001 } 10002 10003 static int 10004 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 10005 { 10006 struct adapter *sc = arg1; 10007 struct sbuf *sb; 10008 int rc; 10009 struct tp_usm_stats stats; 10010 10011 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 10012 if (sb == NULL) 10013 return (ENOMEM); 10014 10015 rc = 0; 10016 mtx_lock(&sc->reg_lock); 10017 if (hw_off_limits(sc)) 10018 rc = ENXIO; 10019 else 10020 t4_get_usm_stats(sc, &stats, 1); 10021 mtx_unlock(&sc->reg_lock); 10022 if (rc == 0) { 10023 sbuf_printf(sb, "Frames: %u\n", stats.frames); 10024 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 10025 sbuf_printf(sb, "Drops: %u", stats.drops); 10026 rc = sbuf_finish(sb); 10027 } 10028 sbuf_delete(sb); 10029 10030 return (rc); 10031 } 10032 10033 static int 10034 sysctl_tid_stats(SYSCTL_HANDLER_ARGS) 10035 { 10036 struct adapter *sc = arg1; 10037 struct sbuf *sb; 10038 int rc; 10039 struct tp_tid_stats stats; 10040 10041 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 10042 if (sb == NULL) 10043 return (ENOMEM); 10044 10045 rc = 0; 10046 mtx_lock(&sc->reg_lock); 10047 if (hw_off_limits(sc)) 10048 rc = ENXIO; 10049 else 10050 t4_tp_get_tid_stats(sc, &stats, 1); 10051 mtx_unlock(&sc->reg_lock); 10052 if (rc == 0) { 10053 sbuf_printf(sb, "Delete: %u\n", stats.del); 10054 sbuf_printf(sb, "Invalidate: %u\n", stats.inv); 10055 sbuf_printf(sb, "Active: %u\n", stats.act); 10056 sbuf_printf(sb, "Passive: %u", stats.pas); 10057 rc = sbuf_finish(sb); 10058 } 10059 sbuf_delete(sb); 10060 10061 return (rc); 10062 } 10063 10064 static const char * const devlog_level_strings[] = { 10065 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 10066 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 10067 [FW_DEVLOG_LEVEL_ERR] = "ERR", 10068 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 10069 [FW_DEVLOG_LEVEL_INFO] = "INFO", 10070 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 10071 }; 10072 10073 static const char * const devlog_facility_strings[] = { 10074 [FW_DEVLOG_FACILITY_CORE] = "CORE", 10075 [FW_DEVLOG_FACILITY_CF] = "CF", 10076 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 10077 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 10078 [FW_DEVLOG_FACILITY_RES] = "RES", 10079 [FW_DEVLOG_FACILITY_HW] = "HW", 10080 [FW_DEVLOG_FACILITY_FLR] = "FLR", 10081 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 10082 [FW_DEVLOG_FACILITY_PHY] = "PHY", 10083 [FW_DEVLOG_FACILITY_MAC] = "MAC", 10084 [FW_DEVLOG_FACILITY_PORT] = "PORT", 10085 [FW_DEVLOG_FACILITY_VI] = "VI", 10086 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 10087 [FW_DEVLOG_FACILITY_ACL] = "ACL", 10088 [FW_DEVLOG_FACILITY_TM] = "TM", 10089 [FW_DEVLOG_FACILITY_QFC] = "QFC", 10090 [FW_DEVLOG_FACILITY_DCB] = "DCB", 10091 [FW_DEVLOG_FACILITY_ETH] = "ETH", 10092 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 10093 [FW_DEVLOG_FACILITY_RI] = "RI", 10094 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 10095 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 10096 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 10097 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE", 10098 [FW_DEVLOG_FACILITY_CHNET] = "CHNET", 10099 }; 10100 10101 static int 10102 sbuf_devlog(struct adapter *sc, int coreid, struct sbuf *sb, int flags) 10103 { 10104 int i, j, rc, nentries, first = 0; 10105 struct devlog_params *dparams = &sc->params.devlog; 10106 struct fw_devlog_e *buf, *e; 10107 uint32_t addr, size; 10108 uint64_t ftstamp = UINT64_MAX; 10109 10110 KASSERT(coreid >= 0 && coreid < sc->params.ncores, 10111 ("%s: bad coreid %d\n", __func__, coreid)); 10112 10113 if (dparams->addr == 0) 10114 return (ENXIO); 10115 10116 size = dparams->size / sc->params.ncores; 10117 addr = dparams->addr + coreid * size; 10118 10119 MPASS(flags == M_WAITOK || flags == M_NOWAIT); 10120 buf = malloc(size, M_CXGBE, M_ZERO | flags); 10121 if (buf == NULL) 10122 return (ENOMEM); 10123 10124 mtx_lock(&sc->reg_lock); 10125 if (hw_off_limits(sc)) 10126 rc = ENXIO; 10127 else 10128 rc = read_via_memwin(sc, 1, addr, (void *)buf, size); 10129 mtx_unlock(&sc->reg_lock); 10130 if (rc != 0) 10131 goto done; 10132 10133 nentries = size / sizeof(struct fw_devlog_e); 10134 for (i = 0; i < nentries; i++) { 10135 e = &buf[i]; 10136 10137 if (e->timestamp == 0) 10138 break; /* end */ 10139 10140 e->timestamp = be64toh(e->timestamp); 10141 e->seqno = be32toh(e->seqno); 10142 for (j = 0; j < 8; j++) 10143 e->params[j] = be32toh(e->params[j]); 10144 10145 if (e->timestamp < ftstamp) { 10146 ftstamp = e->timestamp; 10147 first = i; 10148 } 10149 } 10150 10151 if (buf[first].timestamp == 0) 10152 goto done; /* nothing in the log */ 10153 10154 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 10155 "Seq#", "Tstamp", "Level", "Facility", "Message"); 10156 10157 i = first; 10158 do { 10159 e = &buf[i]; 10160 if (e->timestamp == 0) 10161 break; /* end */ 10162 10163 sbuf_printf(sb, "%10d %15ju %8s %8s ", 10164 e->seqno, e->timestamp, 10165 (e->level < nitems(devlog_level_strings) ? 10166 devlog_level_strings[e->level] : "UNKNOWN"), 10167 (e->facility < nitems(devlog_facility_strings) ? 10168 devlog_facility_strings[e->facility] : "UNKNOWN")); 10169 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 10170 e->params[2], e->params[3], e->params[4], 10171 e->params[5], e->params[6], e->params[7]); 10172 10173 if (++i == nentries) 10174 i = 0; 10175 } while (i != first); 10176 done: 10177 free(buf, M_CXGBE); 10178 return (rc); 10179 } 10180 10181 static int 10182 sysctl_devlog(SYSCTL_HANDLER_ARGS) 10183 { 10184 struct adapter *sc = arg1; 10185 int rc, i, coreid = arg2; 10186 struct sbuf *sb; 10187 10188 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 10189 if (sb == NULL) 10190 return (ENOMEM); 10191 if (coreid == -1) { 10192 /* -1 means all cores */ 10193 for (i = rc = 0; i < sc->params.ncores && rc == 0; i++) { 10194 if (sc->params.ncores > 0) 10195 sbuf_printf(sb, "=== CIM core %u ===\n", i); 10196 rc = sbuf_devlog(sc, i, sb, M_WAITOK); 10197 } 10198 } else { 10199 KASSERT(coreid >= 0 && coreid < sc->params.ncores, 10200 ("%s: bad coreid %d\n", __func__, coreid)); 10201 rc = sbuf_devlog(sc, coreid, sb, M_WAITOK); 10202 } 10203 if (rc == 0) 10204 rc = sbuf_finish(sb); 10205 sbuf_delete(sb); 10206 return (rc); 10207 } 10208 10209 static void 10210 dump_devlog(struct adapter *sc) 10211 { 10212 int rc, i; 10213 struct sbuf sb; 10214 10215 if (sbuf_new(&sb, NULL, 4096, SBUF_AUTOEXTEND) != &sb) { 10216 log(LOG_DEBUG, "%s: failed to generate devlog dump.\n", 10217 device_get_nameunit(sc->dev)); 10218 return; 10219 } 10220 for (i = rc = 0; i < sc->params.ncores && rc == 0; i++) { 10221 if (sc->params.ncores > 0) 10222 sbuf_printf(&sb, "=== CIM core %u ===\n", i); 10223 rc = sbuf_devlog(sc, i, &sb, M_WAITOK); 10224 } 10225 if (rc == 0) { 10226 sbuf_finish(&sb); 10227 log(LOG_DEBUG, "%s: device log follows.\n%s", 10228 device_get_nameunit(sc->dev), sbuf_data(&sb)); 10229 } 10230 sbuf_delete(&sb); 10231 } 10232 10233 static int 10234 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 10235 { 10236 struct adapter *sc = arg1; 10237 struct sbuf *sb; 10238 int rc; 10239 struct tp_fcoe_stats stats[MAX_NCHAN]; 10240 int i, nchan = sc->chip_params->nchan; 10241 10242 rc = 0; 10243 mtx_lock(&sc->reg_lock); 10244 if (hw_off_limits(sc)) 10245 rc = ENXIO; 10246 else { 10247 for (i = 0; i < nchan; i++) 10248 t4_get_fcoe_stats(sc, i, &stats[i], 1); 10249 } 10250 mtx_unlock(&sc->reg_lock); 10251 if (rc != 0) 10252 return (rc); 10253 10254 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 10255 if (sb == NULL) 10256 return (ENOMEM); 10257 10258 if (nchan > 2) { 10259 sbuf_printf(sb, " channel 0 channel 1" 10260 " channel 2 channel 3"); 10261 sbuf_printf(sb, "\noctetsDDP: %16ju %16ju %16ju %16ju", 10262 stats[0].octets_ddp, stats[1].octets_ddp, 10263 stats[2].octets_ddp, stats[3].octets_ddp); 10264 sbuf_printf(sb, "\nframesDDP: %16u %16u %16u %16u", 10265 stats[0].frames_ddp, stats[1].frames_ddp, 10266 stats[2].frames_ddp, stats[3].frames_ddp); 10267 sbuf_printf(sb, "\nframesDrop: %16u %16u %16u %16u", 10268 stats[0].frames_drop, stats[1].frames_drop, 10269 stats[2].frames_drop, stats[3].frames_drop); 10270 } else { 10271 sbuf_printf(sb, " channel 0 channel 1"); 10272 sbuf_printf(sb, "\noctetsDDP: %16ju %16ju", 10273 stats[0].octets_ddp, stats[1].octets_ddp); 10274 sbuf_printf(sb, "\nframesDDP: %16u %16u", 10275 stats[0].frames_ddp, stats[1].frames_ddp); 10276 sbuf_printf(sb, "\nframesDrop: %16u %16u", 10277 stats[0].frames_drop, stats[1].frames_drop); 10278 } 10279 10280 rc = sbuf_finish(sb); 10281 sbuf_delete(sb); 10282 10283 return (rc); 10284 } 10285 10286 static int 10287 sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 10288 { 10289 struct adapter *sc = arg1; 10290 struct sbuf *sb; 10291 int rc, i; 10292 unsigned int map, kbps, ipg, mode; 10293 unsigned int pace_tab[NTX_SCHED]; 10294 10295 sb = sbuf_new_for_sysctl(NULL, NULL, 512, req); 10296 if (sb == NULL) 10297 return (ENOMEM); 10298 10299 mtx_lock(&sc->reg_lock); 10300 if (hw_off_limits(sc)) { 10301 mtx_unlock(&sc->reg_lock); 10302 rc = ENXIO; 10303 goto done; 10304 } 10305 10306 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 10307 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 10308 t4_read_pace_tbl(sc, pace_tab); 10309 mtx_unlock(&sc->reg_lock); 10310 10311 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 10312 "Class IPG (0.1 ns) Flow IPG (us)"); 10313 10314 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 10315 t4_get_tx_sched(sc, i, &kbps, &ipg, 1); 10316 sbuf_printf(sb, "\n %u %-5s %u ", i, 10317 (mode & (1 << i)) ? "flow" : "class", map & 3); 10318 if (kbps) 10319 sbuf_printf(sb, "%9u ", kbps); 10320 else 10321 sbuf_printf(sb, " disabled "); 10322 10323 if (ipg) 10324 sbuf_printf(sb, "%13u ", ipg); 10325 else 10326 sbuf_printf(sb, " disabled "); 10327 10328 if (pace_tab[i]) 10329 sbuf_printf(sb, "%10u", pace_tab[i]); 10330 else 10331 sbuf_printf(sb, " disabled"); 10332 } 10333 rc = sbuf_finish(sb); 10334 done: 10335 sbuf_delete(sb); 10336 return (rc); 10337 } 10338 10339 static int 10340 sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 10341 { 10342 struct adapter *sc = arg1; 10343 struct sbuf *sb; 10344 int rc, i, j; 10345 uint64_t *p0, *p1; 10346 struct lb_port_stats s[2]; 10347 static const char *stat_name[] = { 10348 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 10349 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 10350 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 10351 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 10352 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 10353 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 10354 "BG2FramesTrunc:", "BG3FramesTrunc:" 10355 }; 10356 10357 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 10358 if (sb == NULL) 10359 return (ENOMEM); 10360 10361 memset(s, 0, sizeof(s)); 10362 10363 rc = 0; 10364 for (i = 0; i < sc->chip_params->nchan; i += 2) { 10365 mtx_lock(&sc->reg_lock); 10366 if (hw_off_limits(sc)) 10367 rc = ENXIO; 10368 else { 10369 t4_get_lb_stats(sc, i, &s[0]); 10370 t4_get_lb_stats(sc, i + 1, &s[1]); 10371 } 10372 mtx_unlock(&sc->reg_lock); 10373 if (rc != 0) 10374 break; 10375 10376 p0 = &s[0].octets; 10377 p1 = &s[1].octets; 10378 sbuf_printf(sb, "%s Loopback %u" 10379 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 10380 10381 for (j = 0; j < nitems(stat_name); j++) 10382 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 10383 *p0++, *p1++); 10384 } 10385 10386 if (rc == 0) 10387 rc = sbuf_finish(sb); 10388 sbuf_delete(sb); 10389 10390 return (rc); 10391 } 10392 10393 static int 10394 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) 10395 { 10396 int rc = 0; 10397 struct port_info *pi = arg1; 10398 struct link_config *lc = &pi->link_cfg; 10399 struct sbuf *sb; 10400 10401 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); 10402 if (sb == NULL) 10403 return (ENOMEM); 10404 10405 if (lc->link_ok || lc->link_down_rc == 255) 10406 sbuf_printf(sb, "n/a"); 10407 else 10408 sbuf_printf(sb, "%s", t4_link_down_rc_str(lc->link_down_rc)); 10409 10410 rc = sbuf_finish(sb); 10411 sbuf_delete(sb); 10412 10413 return (rc); 10414 } 10415 10416 struct mem_desc { 10417 uint64_t base; 10418 uint64_t limit; 10419 u_int idx; 10420 }; 10421 10422 static int 10423 mem_desc_cmp(const void *a, const void *b) 10424 { 10425 const uint64_t v1 = ((const struct mem_desc *)a)->base; 10426 const uint64_t v2 = ((const struct mem_desc *)b)->base; 10427 10428 if (v1 < v2) 10429 return (-1); 10430 else if (v1 > v2) 10431 return (1); 10432 10433 return (0); 10434 } 10435 10436 static void 10437 mem_region_show(struct sbuf *sb, const char *name, uint64_t from, uint64_t to) 10438 { 10439 uintmax_t size; 10440 10441 if (from == to) 10442 return; 10443 10444 size = to - from + 1; 10445 if (size == 0) 10446 return; 10447 10448 if (from > UINT32_MAX || to > UINT32_MAX) 10449 sbuf_printf(sb, "%-18s 0x%012jx-0x%012jx [%ju]\n", name, 10450 (uintmax_t)from, (uintmax_t)to, size); 10451 else 10452 sbuf_printf(sb, "%-18s 0x%08jx-0x%08jx [%ju]\n", name, 10453 (uintmax_t)from, (uintmax_t)to, size); 10454 } 10455 10456 static int 10457 sysctl_meminfo(SYSCTL_HANDLER_ARGS) 10458 { 10459 struct adapter *sc = arg1; 10460 struct sbuf *sb; 10461 int rc, i, n, nchan; 10462 uint32_t lo, hi, used, free, alloc; 10463 static const char *memory[] = { 10464 "EDC0:", "EDC1:", "MC:", "MC0:", "MC1:", "HMA:" 10465 }; 10466 static const char *region[] = { 10467 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 10468 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 10469 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 10470 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 10471 "RQUDP region:", "PBL region:", "TXPBL region:", 10472 "TLSKey region:", "RRQ region:", "NVMe STAG region:", 10473 "NVMe RQ region:", "NVMe RXPBL region:", "NVMe TPT region:", 10474 "NVMe TXPBL region:", "DBVFIFO region:", "ULPRX state:", 10475 "ULPTX state:", "RoCE RRQ region:", "On-chip queues:", 10476 }; 10477 struct mem_desc avail[4]; 10478 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 10479 struct mem_desc *md; 10480 10481 rc = sysctl_wire_old_buffer(req, 0); 10482 if (rc != 0) 10483 return (rc); 10484 10485 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 10486 if (sb == NULL) 10487 return (ENOMEM); 10488 10489 for (i = 0; i < nitems(mem); i++) { 10490 mem[i].limit = 0; 10491 mem[i].idx = i; 10492 } 10493 10494 mtx_lock(&sc->reg_lock); 10495 if (hw_off_limits(sc)) { 10496 rc = ENXIO; 10497 goto done; 10498 } 10499 10500 /* Find and sort the populated memory ranges */ 10501 i = 0; 10502 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 10503 if (lo & F_EDRAM0_ENABLE) { 10504 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 10505 if (chip_id(sc) >= CHELSIO_T7) { 10506 avail[i].base = (uint64_t)G_T7_EDRAM0_BASE(hi) << 20; 10507 avail[i].limit = avail[i].base + 10508 (G_T7_EDRAM0_SIZE(hi) << 20); 10509 } else { 10510 avail[i].base = (uint64_t)G_EDRAM0_BASE(hi) << 20; 10511 avail[i].limit = avail[i].base + 10512 (G_EDRAM0_SIZE(hi) << 20); 10513 } 10514 avail[i].idx = 0; 10515 i++; 10516 } 10517 if (lo & F_EDRAM1_ENABLE) { 10518 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 10519 if (chip_id(sc) >= CHELSIO_T7) { 10520 avail[i].base = (uint64_t)G_T7_EDRAM1_BASE(hi) << 20; 10521 avail[i].limit = avail[i].base + 10522 (G_T7_EDRAM1_SIZE(hi) << 20); 10523 } else { 10524 avail[i].base = (uint64_t)G_EDRAM1_BASE(hi) << 20; 10525 avail[i].limit = avail[i].base + 10526 (G_EDRAM1_SIZE(hi) << 20); 10527 } 10528 avail[i].idx = 1; 10529 i++; 10530 } 10531 if (lo & F_EXT_MEM_ENABLE) { 10532 switch (chip_id(sc)) { 10533 case CHELSIO_T4: 10534 case CHELSIO_T6: 10535 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 10536 avail[i].base = (uint64_t)G_EXT_MEM_BASE(hi) << 20; 10537 avail[i].limit = avail[i].base + 10538 (G_EXT_MEM_SIZE(hi) << 20); 10539 avail[i].idx = 2; 10540 break; 10541 case CHELSIO_T5: 10542 hi = t4_read_reg(sc, A_MA_EXT_MEMORY0_BAR); 10543 avail[i].base = (uint64_t)G_EXT_MEM0_BASE(hi) << 20; 10544 avail[i].limit = avail[i].base + 10545 (G_EXT_MEM0_SIZE(hi) << 20); 10546 avail[i].idx = 3; /* Call it MC0 for T5 */ 10547 break; 10548 default: 10549 hi = t4_read_reg(sc, A_MA_EXT_MEMORY0_BAR); 10550 avail[i].base = (uint64_t)G_T7_EXT_MEM0_BASE(hi) << 20; 10551 avail[i].limit = avail[i].base + 10552 (G_T7_EXT_MEM0_SIZE(hi) << 20); 10553 avail[i].idx = 3; /* Call it MC0 for T7+ */ 10554 break; 10555 } 10556 i++; 10557 } 10558 if (lo & F_EXT_MEM1_ENABLE && !(lo & F_MC_SPLIT)) { 10559 /* Only T5 and T7+ have 2 MCs. */ 10560 MPASS(is_t5(sc) || chip_id(sc) >= CHELSIO_T7); 10561 10562 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 10563 if (chip_id(sc) >= CHELSIO_T7) { 10564 avail[i].base = (uint64_t)G_T7_EXT_MEM1_BASE(hi) << 20; 10565 avail[i].limit = avail[i].base + 10566 (G_T7_EXT_MEM1_SIZE(hi) << 20); 10567 } else { 10568 avail[i].base = (uint64_t)G_EXT_MEM1_BASE(hi) << 20; 10569 avail[i].limit = avail[i].base + 10570 (G_EXT_MEM1_SIZE(hi) << 20); 10571 } 10572 avail[i].idx = 4; 10573 i++; 10574 } 10575 if (lo & F_HMA_MUX) { 10576 /* Only T6+ have HMA. */ 10577 MPASS(chip_id(sc) >= CHELSIO_T6); 10578 10579 if (chip_id(sc) >= CHELSIO_T7) { 10580 hi = t4_read_reg(sc, A_MA_HOST_MEMORY_BAR); 10581 avail[i].base = (uint64_t)G_HMATARGETBASE(hi) << 20; 10582 avail[i].limit = avail[i].base + 10583 (G_T7_HMA_SIZE(hi) << 20); 10584 } else { 10585 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 10586 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 10587 avail[i].limit = avail[i].base + 10588 (G_EXT_MEM1_SIZE(hi) << 20); 10589 } 10590 avail[i].idx = 5; 10591 i++; 10592 } 10593 MPASS(i <= nitems(avail)); 10594 if (!i) /* no memory available */ 10595 goto done; 10596 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 10597 10598 md = &mem[0]; 10599 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 10600 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 10601 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 10602 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 10603 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 10604 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 10605 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 10606 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 10607 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 10608 10609 /* the next few have explicit upper bounds */ 10610 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 10611 md->limit = md->base - 1 + 10612 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 10613 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 10614 md++; 10615 10616 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 10617 md->limit = md->base - 1 + 10618 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 10619 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 10620 md++; 10621 10622 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 10623 if (chip_id(sc) <= CHELSIO_T5) 10624 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 10625 else 10626 md->base = t4_read_reg(sc, A_LE_DB_HASH_TBL_BASE_ADDR); 10627 md->limit = 0; 10628 } else { 10629 md->base = 0; 10630 md->idx = nitems(region); /* hide it */ 10631 } 10632 md++; 10633 10634 #define ulp_region(reg) do {\ 10635 const u_int shift = chip_id(sc) >= CHELSIO_T7 ? 4 : 0; \ 10636 md->base = (uint64_t)t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT) << shift; \ 10637 md->limit = (uint64_t)t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) << shift; \ 10638 md->limit += (1 << shift) - 1; \ 10639 md++; \ 10640 } while (0) 10641 10642 #define hide_ulp_region() do { \ 10643 md->base = 0; \ 10644 md->idx = nitems(region); \ 10645 md++; \ 10646 } while (0) 10647 10648 ulp_region(RX_ISCSI); 10649 ulp_region(RX_TDDP); 10650 ulp_region(TX_TPT); 10651 ulp_region(RX_STAG); 10652 ulp_region(RX_RQ); 10653 if (chip_id(sc) < CHELSIO_T7) 10654 ulp_region(RX_RQUDP); 10655 else 10656 hide_ulp_region(); 10657 ulp_region(RX_PBL); 10658 ulp_region(TX_PBL); 10659 if (chip_id(sc) >= CHELSIO_T6) 10660 ulp_region(RX_TLS_KEY); 10661 else 10662 hide_ulp_region(); 10663 if (chip_id(sc) >= CHELSIO_T7) { 10664 ulp_region(RX_RRQ); 10665 ulp_region(RX_NVME_TCP_STAG); 10666 ulp_region(RX_NVME_TCP_RQ); 10667 ulp_region(RX_NVME_TCP_PBL); 10668 ulp_region(TX_NVME_TCP_TPT); 10669 ulp_region(TX_NVME_TCP_PBL); 10670 } else { 10671 hide_ulp_region(); 10672 hide_ulp_region(); 10673 hide_ulp_region(); 10674 hide_ulp_region(); 10675 hide_ulp_region(); 10676 hide_ulp_region(); 10677 } 10678 #undef ulp_region 10679 #undef hide_ulp_region 10680 10681 md->base = 0; 10682 if (is_t4(sc)) 10683 md->idx = nitems(region); 10684 else { 10685 uint32_t size = 0; 10686 uint32_t sge_ctrl = t4_read_reg(sc, A_SGE_CONTROL2); 10687 uint32_t fifo_size = t4_read_reg(sc, A_SGE_DBVFIFO_SIZE); 10688 10689 if (is_t5(sc)) { 10690 if (sge_ctrl & F_VFIFO_ENABLE) 10691 size = fifo_size << 2; 10692 } else 10693 size = G_T6_DBVFIFO_SIZE(fifo_size) << 6; 10694 10695 if (size) { 10696 md->base = t4_read_reg(sc, A_SGE_DBVFIFO_BADDR); 10697 md->limit = md->base + size - 1; 10698 } else 10699 md->idx = nitems(region); 10700 } 10701 md++; 10702 10703 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 10704 md->limit = 0; 10705 md++; 10706 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 10707 md->limit = 0; 10708 md++; 10709 10710 if (chip_id(sc) >= CHELSIO_T7) { 10711 t4_tp_pio_read(sc, &lo, 1, A_TP_ROCE_RRQ_BASE, false); 10712 md->base = lo; 10713 } else { 10714 md->base = 0; 10715 md->idx = nitems(region); 10716 } 10717 md++; 10718 10719 md->base = sc->vres.ocq.start; 10720 if (sc->vres.ocq.size) 10721 md->limit = md->base + sc->vres.ocq.size - 1; 10722 else 10723 md->idx = nitems(region); /* hide it */ 10724 md++; 10725 10726 /* add any address-space holes, there can be up to 3 */ 10727 for (n = 0; n < i - 1; n++) 10728 if (avail[n].limit < avail[n + 1].base) 10729 (md++)->base = avail[n].limit; 10730 if (avail[n].limit) 10731 (md++)->base = avail[n].limit; 10732 10733 n = md - mem; 10734 MPASS(n <= nitems(mem)); 10735 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 10736 10737 for (lo = 0; lo < i; lo++) 10738 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 10739 avail[lo].limit - 1); 10740 10741 sbuf_printf(sb, "\n"); 10742 for (i = 0; i < n; i++) { 10743 if (mem[i].idx >= nitems(region)) 10744 continue; /* skip holes */ 10745 if (!mem[i].limit) 10746 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 10747 mem_region_show(sb, region[mem[i].idx], mem[i].base, 10748 mem[i].limit); 10749 } 10750 10751 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 10752 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 10753 if (hi != lo - 1) { 10754 sbuf_printf(sb, "\n"); 10755 mem_region_show(sb, "uP RAM:", lo, hi); 10756 } 10757 10758 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 10759 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 10760 if (hi != lo - 1) 10761 mem_region_show(sb, "uP Extmem2:", lo, hi); 10762 10763 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 10764 if (chip_id(sc) >= CHELSIO_T7) 10765 nchan = 1 << G_T7_PMRXNUMCHN(lo); 10766 else 10767 nchan = lo & F_PMRXNUMCHN ? 2 : 1; 10768 for (i = 0, free = 0; i < nchan; i++) 10769 free += G_FREERXPAGECOUNT(t4_read_reg(sc, A_TP_FLM_FREE_RX_CNT)); 10770 sbuf_printf(sb, "\n%u Rx pages (%u free) of size %uKiB for %u channels\n", 10771 G_PMRXMAXPAGE(lo), free, 10772 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, nchan); 10773 10774 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 10775 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 10776 if (chip_id(sc) >= CHELSIO_T7) 10777 nchan = 1 << G_T7_PMTXNUMCHN(lo); 10778 else 10779 nchan = 1 << G_PMTXNUMCHN(lo); 10780 for (i = 0, free = 0; i < nchan; i++) 10781 free += G_FREETXPAGECOUNT(t4_read_reg(sc, A_TP_FLM_FREE_TX_CNT)); 10782 sbuf_printf(sb, "%u Tx pages (%u free) of size %u%ciB for %u channels\n", 10783 G_PMTXMAXPAGE(lo), free, 10784 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 10785 hi >= (1 << 20) ? 'M' : 'K', nchan); 10786 sbuf_printf(sb, "%u p-structs (%u free)\n", 10787 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT), 10788 G_FREEPSTRUCTCOUNT(t4_read_reg(sc, A_TP_FLM_FREE_PS_CNT))); 10789 10790 for (i = 0; i < 4; i++) { 10791 if (chip_id(sc) > CHELSIO_T5) 10792 lo = t4_read_reg(sc, A_MPS_RX_MAC_BG_PG_CNT0 + i * 4); 10793 else 10794 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 10795 if (is_t5(sc)) { 10796 used = G_T5_USED(lo); 10797 alloc = G_T5_ALLOC(lo); 10798 } else { 10799 used = G_USED(lo); 10800 alloc = G_ALLOC(lo); 10801 } 10802 /* For T6+ these are MAC buffer groups */ 10803 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 10804 i, used, alloc); 10805 } 10806 for (i = 0; i < sc->chip_params->nchan; i++) { 10807 if (chip_id(sc) > CHELSIO_T5) 10808 lo = t4_read_reg(sc, A_MPS_RX_LPBK_BG_PG_CNT0 + i * 4); 10809 else 10810 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 10811 if (is_t5(sc)) { 10812 used = G_T5_USED(lo); 10813 alloc = G_T5_ALLOC(lo); 10814 } else { 10815 used = G_USED(lo); 10816 alloc = G_ALLOC(lo); 10817 } 10818 /* For T6+ these are MAC buffer groups */ 10819 sbuf_printf(sb, 10820 "\nLoopback %d using %u pages out of %u allocated", 10821 i, used, alloc); 10822 } 10823 done: 10824 mtx_unlock(&sc->reg_lock); 10825 if (rc == 0) 10826 rc = sbuf_finish(sb); 10827 sbuf_delete(sb); 10828 return (rc); 10829 } 10830 10831 static inline void 10832 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) 10833 { 10834 *mask = x | y; 10835 y = htobe64(y); 10836 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); 10837 } 10838 10839 static int 10840 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) 10841 { 10842 struct adapter *sc = arg1; 10843 struct sbuf *sb; 10844 int rc, i; 10845 10846 MPASS(chip_id(sc) <= CHELSIO_T5); 10847 10848 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 10849 if (sb == NULL) 10850 return (ENOMEM); 10851 10852 sbuf_printf(sb, 10853 "Idx Ethernet address Mask Vld Ports PF" 10854 " VF Replication P0 P1 P2 P3 ML"); 10855 rc = 0; 10856 for (i = 0; i < sc->chip_params->mps_tcam_size; i++) { 10857 uint64_t tcamx, tcamy, mask; 10858 uint32_t cls_lo, cls_hi; 10859 uint8_t addr[ETHER_ADDR_LEN]; 10860 10861 mtx_lock(&sc->reg_lock); 10862 if (hw_off_limits(sc)) 10863 rc = ENXIO; 10864 else { 10865 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); 10866 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); 10867 } 10868 mtx_unlock(&sc->reg_lock); 10869 if (rc != 0) 10870 break; 10871 if (tcamx & tcamy) 10872 continue; 10873 tcamxy2valmask(tcamx, tcamy, addr, &mask); 10874 mtx_lock(&sc->reg_lock); 10875 if (hw_off_limits(sc)) 10876 rc = ENXIO; 10877 else { 10878 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 10879 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 10880 } 10881 mtx_unlock(&sc->reg_lock); 10882 if (rc != 0) 10883 break; 10884 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" 10885 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], 10886 addr[3], addr[4], addr[5], (uintmax_t)mask, 10887 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', 10888 G_PORTMAP(cls_hi), G_PF(cls_lo), 10889 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); 10890 10891 if (cls_lo & F_REPLICATE) { 10892 struct fw_ldst_cmd ldst_cmd; 10893 10894 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 10895 ldst_cmd.op_to_addrspace = 10896 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 10897 F_FW_CMD_REQUEST | F_FW_CMD_READ | 10898 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 10899 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 10900 ldst_cmd.u.mps.rplc.fid_idx = 10901 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 10902 V_FW_LDST_CMD_IDX(i)); 10903 10904 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 10905 "t4mps"); 10906 if (rc) 10907 break; 10908 if (hw_off_limits(sc)) 10909 rc = ENXIO; 10910 else 10911 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 10912 sizeof(ldst_cmd), &ldst_cmd); 10913 end_synchronized_op(sc, 0); 10914 if (rc != 0) 10915 break; 10916 else { 10917 sbuf_printf(sb, " %08x %08x %08x %08x", 10918 be32toh(ldst_cmd.u.mps.rplc.rplc127_96), 10919 be32toh(ldst_cmd.u.mps.rplc.rplc95_64), 10920 be32toh(ldst_cmd.u.mps.rplc.rplc63_32), 10921 be32toh(ldst_cmd.u.mps.rplc.rplc31_0)); 10922 } 10923 } else 10924 sbuf_printf(sb, "%36s", ""); 10925 10926 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), 10927 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), 10928 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); 10929 } 10930 10931 if (rc) 10932 (void) sbuf_finish(sb); 10933 else 10934 rc = sbuf_finish(sb); 10935 sbuf_delete(sb); 10936 10937 return (rc); 10938 } 10939 10940 static int 10941 sysctl_mps_tcam_t6(SYSCTL_HANDLER_ARGS) 10942 { 10943 struct adapter *sc = arg1; 10944 struct sbuf *sb; 10945 int rc, i; 10946 10947 MPASS(chip_id(sc) == CHELSIO_T6); 10948 10949 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 10950 if (sb == NULL) 10951 return (ENOMEM); 10952 10953 sbuf_printf(sb, "Idx Ethernet address Mask VNI Mask" 10954 " IVLAN Vld DIP_Hit Lookup Port Vld Ports PF VF" 10955 " Replication" 10956 " P0 P1 P2 P3 ML"); 10957 10958 rc = 0; 10959 for (i = 0; i < sc->chip_params->mps_tcam_size; i++) { 10960 uint8_t dip_hit, vlan_vld, lookup_type, port_num; 10961 uint16_t ivlan; 10962 uint64_t tcamx, tcamy, val, mask; 10963 uint32_t cls_lo, cls_hi, ctl, data2, vnix, vniy; 10964 uint8_t addr[ETHER_ADDR_LEN]; 10965 10966 ctl = V_CTLREQID(1) | V_CTLCMDTYPE(0) | V_CTLXYBITSEL(0); 10967 if (i < 256) 10968 ctl |= V_CTLTCAMINDEX(i) | V_CTLTCAMSEL(0); 10969 else 10970 ctl |= V_CTLTCAMINDEX(i - 256) | V_CTLTCAMSEL(1); 10971 mtx_lock(&sc->reg_lock); 10972 if (hw_off_limits(sc)) 10973 rc = ENXIO; 10974 else { 10975 t4_write_reg(sc, A_MPS_CLS_TCAM_DATA2_CTL, ctl); 10976 val = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA1_REQ_ID1); 10977 tcamy = G_DMACH(val) << 32; 10978 tcamy |= t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA0_REQ_ID1); 10979 data2 = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA2_REQ_ID1); 10980 } 10981 mtx_unlock(&sc->reg_lock); 10982 if (rc != 0) 10983 break; 10984 10985 lookup_type = G_DATALKPTYPE(data2); 10986 port_num = G_DATAPORTNUM(data2); 10987 if (lookup_type && lookup_type != M_DATALKPTYPE) { 10988 /* Inner header VNI */ 10989 vniy = ((data2 & F_DATAVIDH2) << 23) | 10990 (G_DATAVIDH1(data2) << 16) | G_VIDL(val); 10991 dip_hit = data2 & F_DATADIPHIT; 10992 vlan_vld = 0; 10993 } else { 10994 vniy = 0; 10995 dip_hit = 0; 10996 vlan_vld = data2 & F_DATAVIDH2; 10997 ivlan = G_VIDL(val); 10998 } 10999 11000 ctl |= V_CTLXYBITSEL(1); 11001 mtx_lock(&sc->reg_lock); 11002 if (hw_off_limits(sc)) 11003 rc = ENXIO; 11004 else { 11005 t4_write_reg(sc, A_MPS_CLS_TCAM_DATA2_CTL, ctl); 11006 val = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA1_REQ_ID1); 11007 tcamx = G_DMACH(val) << 32; 11008 tcamx |= t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA0_REQ_ID1); 11009 data2 = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA2_REQ_ID1); 11010 } 11011 mtx_unlock(&sc->reg_lock); 11012 if (rc != 0) 11013 break; 11014 11015 if (lookup_type && lookup_type != M_DATALKPTYPE) { 11016 /* Inner header VNI mask */ 11017 vnix = ((data2 & F_DATAVIDH2) << 23) | 11018 (G_DATAVIDH1(data2) << 16) | G_VIDL(val); 11019 } else 11020 vnix = 0; 11021 11022 if (tcamx & tcamy) 11023 continue; 11024 tcamxy2valmask(tcamx, tcamy, addr, &mask); 11025 11026 mtx_lock(&sc->reg_lock); 11027 if (hw_off_limits(sc)) 11028 rc = ENXIO; 11029 else { 11030 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 11031 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 11032 } 11033 mtx_unlock(&sc->reg_lock); 11034 if (rc != 0) 11035 break; 11036 11037 if (lookup_type && lookup_type != M_DATALKPTYPE) { 11038 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x " 11039 "%012jx %06x %06x - - %3c" 11040 " I %4x %3c %#x%4u%4d", i, addr[0], 11041 addr[1], addr[2], addr[3], addr[4], addr[5], 11042 (uintmax_t)mask, vniy, vnix, dip_hit ? 'Y' : 'N', 11043 port_num, cls_lo & F_T6_SRAM_VLD ? 'Y' : 'N', 11044 G_PORTMAP(cls_hi), G_T6_PF(cls_lo), 11045 cls_lo & F_T6_VF_VALID ? G_T6_VF(cls_lo) : -1); 11046 } else { 11047 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x " 11048 "%012jx - - ", i, addr[0], addr[1], 11049 addr[2], addr[3], addr[4], addr[5], 11050 (uintmax_t)mask); 11051 11052 if (vlan_vld) 11053 sbuf_printf(sb, "%4u Y ", ivlan); 11054 else 11055 sbuf_printf(sb, " - N "); 11056 11057 sbuf_printf(sb, "- %3c %4x %3c %#x%4u%4d", 11058 lookup_type ? 'I' : 'O', port_num, 11059 cls_lo & F_T6_SRAM_VLD ? 'Y' : 'N', 11060 G_PORTMAP(cls_hi), G_T6_PF(cls_lo), 11061 cls_lo & F_T6_VF_VALID ? G_T6_VF(cls_lo) : -1); 11062 } 11063 11064 11065 if (cls_lo & F_T6_REPLICATE) { 11066 struct fw_ldst_cmd ldst_cmd; 11067 11068 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 11069 ldst_cmd.op_to_addrspace = 11070 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 11071 F_FW_CMD_REQUEST | F_FW_CMD_READ | 11072 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 11073 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 11074 ldst_cmd.u.mps.rplc.fid_idx = 11075 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 11076 V_FW_LDST_CMD_IDX(i)); 11077 11078 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 11079 "t6mps"); 11080 if (rc) 11081 break; 11082 if (hw_off_limits(sc)) 11083 rc = ENXIO; 11084 else 11085 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 11086 sizeof(ldst_cmd), &ldst_cmd); 11087 end_synchronized_op(sc, 0); 11088 if (rc != 0) 11089 break; 11090 else { 11091 sbuf_printf(sb, " %08x %08x %08x %08x" 11092 " %08x %08x %08x %08x", 11093 be32toh(ldst_cmd.u.mps.rplc.rplc255_224), 11094 be32toh(ldst_cmd.u.mps.rplc.rplc223_192), 11095 be32toh(ldst_cmd.u.mps.rplc.rplc191_160), 11096 be32toh(ldst_cmd.u.mps.rplc.rplc159_128), 11097 be32toh(ldst_cmd.u.mps.rplc.rplc127_96), 11098 be32toh(ldst_cmd.u.mps.rplc.rplc95_64), 11099 be32toh(ldst_cmd.u.mps.rplc.rplc63_32), 11100 be32toh(ldst_cmd.u.mps.rplc.rplc31_0)); 11101 } 11102 } else 11103 sbuf_printf(sb, "%72s", ""); 11104 11105 sbuf_printf(sb, "%4u%3u%3u%3u %#x", 11106 G_T6_SRAM_PRIO0(cls_lo), G_T6_SRAM_PRIO1(cls_lo), 11107 G_T6_SRAM_PRIO2(cls_lo), G_T6_SRAM_PRIO3(cls_lo), 11108 (cls_lo >> S_T6_MULTILISTEN0) & 0xf); 11109 } 11110 11111 if (rc) 11112 (void) sbuf_finish(sb); 11113 else 11114 rc = sbuf_finish(sb); 11115 sbuf_delete(sb); 11116 11117 return (rc); 11118 } 11119 11120 static int 11121 sysctl_mps_tcam_t7(SYSCTL_HANDLER_ARGS) 11122 { 11123 struct adapter *sc = arg1; 11124 struct sbuf *sb; 11125 int rc, i; 11126 11127 MPASS(chip_id(sc) >= CHELSIO_T7); 11128 11129 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 11130 if (sb == NULL) 11131 return (ENOMEM); 11132 11133 sbuf_printf(sb, "Idx Ethernet address Mask VNI Mask" 11134 " IVLAN Vld DIP_Hit Lookup Port Vld Ports PF VF" 11135 " Replication" 11136 " P0 P1 P2 P3 ML"); 11137 11138 rc = 0; 11139 for (i = 0; i < sc->chip_params->mps_tcam_size; i++) { 11140 uint8_t dip_hit, vlan_vld, lookup_type, port_num; 11141 uint16_t ivlan; 11142 uint64_t tcamx, tcamy, val, mask; 11143 uint32_t cls_lo, cls_hi, ctl, data2, vnix, vniy; 11144 uint8_t addr[ETHER_ADDR_LEN]; 11145 11146 /* Read tcamy */ 11147 ctl = (V_CTLREQID(1) | V_CTLCMDTYPE(0) | V_CTLXYBITSEL(0)); 11148 if (chip_rev(sc) == 0) { 11149 if (i < 256) 11150 ctl |= V_CTLTCAMINDEX(i) | V_T7_CTLTCAMSEL(0); 11151 else 11152 ctl |= V_CTLTCAMINDEX(i - 256) | V_T7_CTLTCAMSEL(1); 11153 } else { 11154 #if 0 11155 ctl = (V_CTLREQID(1) | V_CTLCMDTYPE(0) | V_CTLXYBITSEL(0)); 11156 #endif 11157 if (i < 512) 11158 ctl |= V_CTLTCAMINDEX(i) | V_T7_CTLTCAMSEL(0); 11159 else if (i < 1024) 11160 ctl |= V_CTLTCAMINDEX(i - 512) | V_T7_CTLTCAMSEL(1); 11161 else 11162 ctl |= V_CTLTCAMINDEX(i - 1024) | V_T7_CTLTCAMSEL(2); 11163 } 11164 11165 mtx_lock(&sc->reg_lock); 11166 if (hw_off_limits(sc)) 11167 rc = ENXIO; 11168 else { 11169 t4_write_reg(sc, A_MPS_CLS_TCAM_DATA2_CTL, ctl); 11170 val = t4_read_reg(sc, A_MPS_CLS_TCAM0_RDATA1_REQ_ID1); 11171 tcamy = G_DMACH(val) << 32; 11172 tcamy |= t4_read_reg(sc, A_MPS_CLS_TCAM0_RDATA0_REQ_ID1); 11173 data2 = t4_read_reg(sc, A_MPS_CLS_TCAM0_RDATA2_REQ_ID1); 11174 } 11175 mtx_unlock(&sc->reg_lock); 11176 if (rc != 0) 11177 break; 11178 11179 lookup_type = G_DATALKPTYPE(data2); 11180 port_num = G_DATAPORTNUM(data2); 11181 if (lookup_type && lookup_type != M_DATALKPTYPE) { 11182 /* Inner header VNI */ 11183 vniy = (((data2 & F_DATAVIDH2) | 11184 G_DATAVIDH1(data2)) << 16) | G_VIDL(val); 11185 dip_hit = data2 & F_DATADIPHIT; 11186 vlan_vld = 0; 11187 } else { 11188 vniy = 0; 11189 dip_hit = 0; 11190 vlan_vld = data2 & F_DATAVIDH2; 11191 ivlan = G_VIDL(val); 11192 } 11193 11194 ctl |= V_CTLXYBITSEL(1); 11195 mtx_lock(&sc->reg_lock); 11196 if (hw_off_limits(sc)) 11197 rc = ENXIO; 11198 else { 11199 t4_write_reg(sc, A_MPS_CLS_TCAM_DATA2_CTL, ctl); 11200 val = t4_read_reg(sc, A_MPS_CLS_TCAM0_RDATA1_REQ_ID1); 11201 tcamx = G_DMACH(val) << 32; 11202 tcamx |= t4_read_reg(sc, A_MPS_CLS_TCAM0_RDATA0_REQ_ID1); 11203 data2 = t4_read_reg(sc, A_MPS_CLS_TCAM0_RDATA2_REQ_ID1); 11204 } 11205 mtx_unlock(&sc->reg_lock); 11206 if (rc != 0) 11207 break; 11208 11209 if (lookup_type && lookup_type != M_DATALKPTYPE) { 11210 /* Inner header VNI mask */ 11211 vnix = (((data2 & F_DATAVIDH2) | 11212 G_DATAVIDH1(data2)) << 16) | G_VIDL(val); 11213 } else 11214 vnix = 0; 11215 11216 if (tcamx & tcamy) 11217 continue; 11218 tcamxy2valmask(tcamx, tcamy, addr, &mask); 11219 11220 mtx_lock(&sc->reg_lock); 11221 if (hw_off_limits(sc)) 11222 rc = ENXIO; 11223 else { 11224 if (chip_rev(sc) == 0) { 11225 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 11226 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 11227 } else { 11228 t4_write_reg(sc, A_MPS_CLS_SRAM_H, 11229 V_SRAMWRN(0) | V_SRAMINDEX(i)); 11230 cls_lo = t4_read_reg(sc, A_MPS_CLS_SRAM_L); 11231 cls_hi = t4_read_reg(sc, A_MPS_CLS_SRAM_H); 11232 } 11233 } 11234 mtx_unlock(&sc->reg_lock); 11235 if (rc != 0) 11236 break; 11237 11238 if (lookup_type && lookup_type != M_DATALKPTYPE) { 11239 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x " 11240 "%012jx %06x %06x - - %3c" 11241 " I %4x %3c %#x%4u%4d", i, addr[0], 11242 addr[1], addr[2], addr[3], addr[4], addr[5], 11243 (uintmax_t)mask, vniy, vnix, dip_hit ? 'Y' : 'N', 11244 port_num, cls_lo & F_T6_SRAM_VLD ? 'Y' : 'N', 11245 G_PORTMAP(cls_hi), G_T6_PF(cls_lo), 11246 cls_lo & F_T6_VF_VALID ? G_T6_VF(cls_lo) : -1); 11247 } else { 11248 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x " 11249 "%012jx - - ", i, addr[0], addr[1], 11250 addr[2], addr[3], addr[4], addr[5], 11251 (uintmax_t)mask); 11252 11253 if (vlan_vld) 11254 sbuf_printf(sb, "%4u Y ", ivlan); 11255 else 11256 sbuf_printf(sb, " - N "); 11257 11258 sbuf_printf(sb, "- %3c %4x %3c %#x%4u%4d", 11259 lookup_type ? 'I' : 'O', port_num, 11260 cls_lo & F_T6_SRAM_VLD ? 'Y' : 'N', 11261 G_PORTMAP(cls_hi), G_T6_PF(cls_lo), 11262 cls_lo & F_T6_VF_VALID ? G_T6_VF(cls_lo) : -1); 11263 } 11264 11265 if (cls_lo & F_T6_REPLICATE) { 11266 struct fw_ldst_cmd ldst_cmd; 11267 11268 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 11269 ldst_cmd.op_to_addrspace = 11270 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 11271 F_FW_CMD_REQUEST | F_FW_CMD_READ | 11272 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 11273 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 11274 ldst_cmd.u.mps.rplc.fid_idx = 11275 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 11276 V_FW_LDST_CMD_IDX(i)); 11277 11278 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 11279 "t6mps"); 11280 if (rc) 11281 break; 11282 if (hw_off_limits(sc)) 11283 rc = ENXIO; 11284 else 11285 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 11286 sizeof(ldst_cmd), &ldst_cmd); 11287 end_synchronized_op(sc, 0); 11288 if (rc != 0) 11289 break; 11290 else { 11291 sbuf_printf(sb, " %08x %08x %08x %08x" 11292 " %08x %08x %08x %08x", 11293 be32toh(ldst_cmd.u.mps.rplc.rplc255_224), 11294 be32toh(ldst_cmd.u.mps.rplc.rplc223_192), 11295 be32toh(ldst_cmd.u.mps.rplc.rplc191_160), 11296 be32toh(ldst_cmd.u.mps.rplc.rplc159_128), 11297 be32toh(ldst_cmd.u.mps.rplc.rplc127_96), 11298 be32toh(ldst_cmd.u.mps.rplc.rplc95_64), 11299 be32toh(ldst_cmd.u.mps.rplc.rplc63_32), 11300 be32toh(ldst_cmd.u.mps.rplc.rplc31_0)); 11301 } 11302 } else 11303 sbuf_printf(sb, "%72s", ""); 11304 11305 sbuf_printf(sb, "%4u%3u%3u%3u %#x", 11306 G_T6_SRAM_PRIO0(cls_lo), G_T6_SRAM_PRIO1(cls_lo), 11307 G_T6_SRAM_PRIO2(cls_lo), G_T6_SRAM_PRIO3(cls_lo), 11308 (cls_lo >> S_T6_MULTILISTEN0) & 0xf); 11309 } 11310 11311 if (rc) 11312 (void) sbuf_finish(sb); 11313 else 11314 rc = sbuf_finish(sb); 11315 sbuf_delete(sb); 11316 11317 return (rc); 11318 } 11319 11320 static int 11321 sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 11322 { 11323 struct adapter *sc = arg1; 11324 struct sbuf *sb; 11325 int rc; 11326 uint16_t mtus[NMTUS]; 11327 11328 rc = 0; 11329 mtx_lock(&sc->reg_lock); 11330 if (hw_off_limits(sc)) 11331 rc = ENXIO; 11332 else 11333 t4_read_mtu_tbl(sc, mtus, NULL); 11334 mtx_unlock(&sc->reg_lock); 11335 if (rc != 0) 11336 return (rc); 11337 11338 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 11339 if (sb == NULL) 11340 return (ENOMEM); 11341 11342 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 11343 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 11344 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 11345 mtus[14], mtus[15]); 11346 11347 rc = sbuf_finish(sb); 11348 sbuf_delete(sb); 11349 11350 return (rc); 11351 } 11352 11353 static int 11354 sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 11355 { 11356 struct adapter *sc = arg1; 11357 struct sbuf *sb; 11358 int rc, i; 11359 uint32_t tx_cnt[MAX_PM_NSTATS], rx_cnt[MAX_PM_NSTATS]; 11360 uint64_t tx_cyc[MAX_PM_NSTATS], rx_cyc[MAX_PM_NSTATS]; 11361 uint32_t stats[T7_PM_RX_CACHE_NSTATS]; 11362 static const char *tx_stats[MAX_PM_NSTATS] = { 11363 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:", 11364 "Tx FIFO wait", NULL, "Tx latency" 11365 }; 11366 static const char *rx_stats[MAX_PM_NSTATS] = { 11367 "Read:", "Write bypass:", "Write mem:", "Flush:", 11368 "Rx FIFO wait", NULL, "Rx latency" 11369 }; 11370 11371 rc = 0; 11372 mtx_lock(&sc->reg_lock); 11373 if (hw_off_limits(sc)) 11374 rc = ENXIO; 11375 else { 11376 t4_pmtx_get_stats(sc, tx_cnt, tx_cyc); 11377 t4_pmrx_get_stats(sc, rx_cnt, rx_cyc); 11378 if (chip_id(sc) >= CHELSIO_T7) 11379 t4_pmrx_cache_get_stats(sc, stats); 11380 } 11381 mtx_unlock(&sc->reg_lock); 11382 if (rc != 0) 11383 return (rc); 11384 11385 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 11386 if (sb == NULL) 11387 return (ENOMEM); 11388 11389 sbuf_printf(sb, " Tx pcmds Tx bytes"); 11390 for (i = 0; i < 4; i++) { 11391 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i], 11392 tx_cyc[i]); 11393 } 11394 11395 sbuf_printf(sb, "\n Rx pcmds Rx bytes"); 11396 for (i = 0; i < 4; i++) { 11397 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i], 11398 rx_cyc[i]); 11399 } 11400 11401 if (chip_id(sc) > CHELSIO_T5) { 11402 sbuf_printf(sb, 11403 "\n Total wait Total occupancy"); 11404 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i], 11405 tx_cyc[i]); 11406 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i], 11407 rx_cyc[i]); 11408 11409 i += 2; 11410 MPASS(i < nitems(tx_stats)); 11411 11412 sbuf_printf(sb, 11413 "\n Reads Total wait"); 11414 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i], 11415 tx_cyc[i]); 11416 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i], 11417 rx_cyc[i]); 11418 } 11419 11420 if (chip_id(sc) >= CHELSIO_T7) { 11421 i = 0; 11422 sbuf_printf(sb, "\n\nPM RX Cache Stats\n"); 11423 sbuf_printf(sb, "%-40s %u\n", "ReqWrite", stats[i++]); 11424 sbuf_printf(sb, "%-40s %u\n", "ReqReadInv", stats[i++]); 11425 sbuf_printf(sb, "%-40s %u\n", "ReqReadNoInv", stats[i++]); 11426 sbuf_printf(sb, "%-40s %u\n", "Write Split Request", 11427 stats[i++]); 11428 sbuf_printf(sb, "%-40s %u\n", 11429 "Normal Read Split (Read Invalidate)", stats[i++]); 11430 sbuf_printf(sb, "%-40s %u\n", 11431 "Feedback Read Split (Read NoInvalidate)", 11432 stats[i++]); 11433 sbuf_printf(sb, "%-40s %u\n", "Write Hit", stats[i++]); 11434 sbuf_printf(sb, "%-40s %u\n", "Normal Read Hit", 11435 stats[i++]); 11436 sbuf_printf(sb, "%-40s %u\n", "Feedback Read Hit", 11437 stats[i++]); 11438 sbuf_printf(sb, "%-40s %u\n", "Normal Read Hit Full Avail", 11439 stats[i++]); 11440 sbuf_printf(sb, "%-40s %u\n", "Normal Read Hit Full UnAvail", 11441 stats[i++]); 11442 sbuf_printf(sb, "%-40s %u\n", 11443 "Normal Read Hit Partial Avail", 11444 stats[i++]); 11445 sbuf_printf(sb, "%-40s %u\n", "FB Read Hit Full Avail", 11446 stats[i++]); 11447 sbuf_printf(sb, "%-40s %u\n", "FB Read Hit Full UnAvail", 11448 stats[i++]); 11449 sbuf_printf(sb, "%-40s %u\n", "FB Read Hit Partial Avail", 11450 stats[i++]); 11451 sbuf_printf(sb, "%-40s %u\n", "Normal Read Full Free", 11452 stats[i++]); 11453 sbuf_printf(sb, "%-40s %u\n", 11454 "Normal Read Part-avail Mul-Regions", 11455 stats[i++]); 11456 sbuf_printf(sb, "%-40s %u\n", 11457 "FB Read Part-avail Mul-Regions", 11458 stats[i++]); 11459 sbuf_printf(sb, "%-40s %u\n", "Write Miss FL Used", 11460 stats[i++]); 11461 sbuf_printf(sb, "%-40s %u\n", "Write Miss LRU Used", 11462 stats[i++]); 11463 sbuf_printf(sb, "%-40s %u\n", 11464 "Write Miss LRU-Multiple Evict", stats[i++]); 11465 sbuf_printf(sb, "%-40s %u\n", 11466 "Write Hit Increasing Islands", stats[i++]); 11467 sbuf_printf(sb, "%-40s %u\n", 11468 "Normal Read Island Read split", stats[i++]); 11469 sbuf_printf(sb, "%-40s %u\n", "Write Overflow Eviction", 11470 stats[i++]); 11471 sbuf_printf(sb, "%-40s %u", "Read Overflow Eviction", 11472 stats[i++]); 11473 } 11474 11475 rc = sbuf_finish(sb); 11476 sbuf_delete(sb); 11477 11478 return (rc); 11479 } 11480 11481 static int 11482 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 11483 { 11484 struct adapter *sc = arg1; 11485 struct sbuf *sb; 11486 int rc; 11487 struct tp_rdma_stats stats; 11488 11489 rc = 0; 11490 mtx_lock(&sc->reg_lock); 11491 if (hw_off_limits(sc)) 11492 rc = ENXIO; 11493 else 11494 t4_tp_get_rdma_stats(sc, &stats, 0); 11495 mtx_unlock(&sc->reg_lock); 11496 if (rc != 0) 11497 return (rc); 11498 11499 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 11500 if (sb == NULL) 11501 return (ENOMEM); 11502 11503 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 11504 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 11505 11506 rc = sbuf_finish(sb); 11507 sbuf_delete(sb); 11508 11509 return (rc); 11510 } 11511 11512 static int 11513 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 11514 { 11515 struct adapter *sc = arg1; 11516 struct sbuf *sb; 11517 int rc; 11518 struct tp_tcp_stats v4, v6; 11519 11520 rc = 0; 11521 mtx_lock(&sc->reg_lock); 11522 if (hw_off_limits(sc)) 11523 rc = ENXIO; 11524 else 11525 t4_tp_get_tcp_stats(sc, &v4, &v6, 0); 11526 mtx_unlock(&sc->reg_lock); 11527 if (rc != 0) 11528 return (rc); 11529 11530 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 11531 if (sb == NULL) 11532 return (ENOMEM); 11533 11534 sbuf_printf(sb, 11535 " IP IPv6\n"); 11536 sbuf_printf(sb, "OutRsts: %20u %20u\n", 11537 v4.tcp_out_rsts, v6.tcp_out_rsts); 11538 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 11539 v4.tcp_in_segs, v6.tcp_in_segs); 11540 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 11541 v4.tcp_out_segs, v6.tcp_out_segs); 11542 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 11543 v4.tcp_retrans_segs, v6.tcp_retrans_segs); 11544 11545 rc = sbuf_finish(sb); 11546 sbuf_delete(sb); 11547 11548 return (rc); 11549 } 11550 11551 static int 11552 sysctl_tids(SYSCTL_HANDLER_ARGS) 11553 { 11554 struct adapter *sc = arg1; 11555 struct sbuf *sb; 11556 int rc; 11557 uint32_t x, y; 11558 struct tid_info *t = &sc->tids; 11559 11560 rc = 0; 11561 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 11562 if (sb == NULL) 11563 return (ENOMEM); 11564 11565 if (t->natids) { 11566 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 11567 t->atids_in_use); 11568 } 11569 11570 if (t->nhpftids) { 11571 sbuf_printf(sb, "HPFTID range: %u-%u, in use: %u\n", 11572 t->hpftid_base, t->hpftid_end, t->hpftids_in_use); 11573 } 11574 11575 if (t->ntids) { 11576 bool hashen = false; 11577 11578 mtx_lock(&sc->reg_lock); 11579 if (hw_off_limits(sc)) 11580 rc = ENXIO; 11581 else if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 11582 hashen = true; 11583 if (chip_id(sc) <= CHELSIO_T5) { 11584 x = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 11585 y = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 11586 } else { 11587 x = t4_read_reg(sc, A_LE_DB_SRVR_START_INDEX); 11588 y = t4_read_reg(sc, A_T6_LE_DB_HASH_TID_BASE); 11589 } 11590 } 11591 mtx_unlock(&sc->reg_lock); 11592 if (rc != 0) 11593 goto done; 11594 11595 sbuf_printf(sb, "TID range: "); 11596 if (hashen) { 11597 if (x) 11598 sbuf_printf(sb, "%u-%u, ", t->tid_base, x - 1); 11599 sbuf_printf(sb, "%u-%u", y, t->tid_base + t->ntids - 1); 11600 } else { 11601 sbuf_printf(sb, "%u-%u", t->tid_base, t->tid_base + 11602 t->ntids - 1); 11603 } 11604 sbuf_printf(sb, ", in use: %u\n", 11605 atomic_load_acq_int(&t->tids_in_use)); 11606 } 11607 11608 if (t->nstids) { 11609 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 11610 t->stid_base + t->nstids - 1, t->stids_in_use); 11611 } 11612 11613 if (t->nftids) { 11614 sbuf_printf(sb, "FTID range: %u-%u, in use: %u\n", t->ftid_base, 11615 t->ftid_end, t->ftids_in_use); 11616 } 11617 11618 if (t->netids) { 11619 sbuf_printf(sb, "ETID range: %u-%u, in use: %u\n", t->etid_base, 11620 t->etid_base + t->netids - 1, t->etids_in_use); 11621 } 11622 11623 mtx_lock(&sc->reg_lock); 11624 if (hw_off_limits(sc)) 11625 rc = ENXIO; 11626 else { 11627 x = t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4); 11628 y = t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6); 11629 } 11630 mtx_unlock(&sc->reg_lock); 11631 if (rc != 0) 11632 goto done; 11633 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", x, y); 11634 done: 11635 if (rc == 0) 11636 rc = sbuf_finish(sb); 11637 else 11638 (void)sbuf_finish(sb); 11639 sbuf_delete(sb); 11640 11641 return (rc); 11642 } 11643 11644 static int 11645 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 11646 { 11647 struct adapter *sc = arg1; 11648 struct sbuf *sb; 11649 int rc; 11650 struct tp_err_stats stats; 11651 11652 rc = 0; 11653 mtx_lock(&sc->reg_lock); 11654 if (hw_off_limits(sc)) 11655 rc = ENXIO; 11656 else 11657 t4_tp_get_err_stats(sc, &stats, 0); 11658 mtx_unlock(&sc->reg_lock); 11659 if (rc != 0) 11660 return (rc); 11661 11662 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 11663 if (sb == NULL) 11664 return (ENOMEM); 11665 11666 if (sc->chip_params->nchan > 2) { 11667 sbuf_printf(sb, " channel 0 channel 1" 11668 " channel 2 channel 3\n"); 11669 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 11670 stats.mac_in_errs[0], stats.mac_in_errs[1], 11671 stats.mac_in_errs[2], stats.mac_in_errs[3]); 11672 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 11673 stats.hdr_in_errs[0], stats.hdr_in_errs[1], 11674 stats.hdr_in_errs[2], stats.hdr_in_errs[3]); 11675 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 11676 stats.tcp_in_errs[0], stats.tcp_in_errs[1], 11677 stats.tcp_in_errs[2], stats.tcp_in_errs[3]); 11678 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 11679 stats.tcp6_in_errs[0], stats.tcp6_in_errs[1], 11680 stats.tcp6_in_errs[2], stats.tcp6_in_errs[3]); 11681 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 11682 stats.tnl_cong_drops[0], stats.tnl_cong_drops[1], 11683 stats.tnl_cong_drops[2], stats.tnl_cong_drops[3]); 11684 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 11685 stats.tnl_tx_drops[0], stats.tnl_tx_drops[1], 11686 stats.tnl_tx_drops[2], stats.tnl_tx_drops[3]); 11687 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 11688 stats.ofld_vlan_drops[0], stats.ofld_vlan_drops[1], 11689 stats.ofld_vlan_drops[2], stats.ofld_vlan_drops[3]); 11690 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 11691 stats.ofld_chan_drops[0], stats.ofld_chan_drops[1], 11692 stats.ofld_chan_drops[2], stats.ofld_chan_drops[3]); 11693 } else { 11694 sbuf_printf(sb, " channel 0 channel 1\n"); 11695 sbuf_printf(sb, "macInErrs: %10u %10u\n", 11696 stats.mac_in_errs[0], stats.mac_in_errs[1]); 11697 sbuf_printf(sb, "hdrInErrs: %10u %10u\n", 11698 stats.hdr_in_errs[0], stats.hdr_in_errs[1]); 11699 sbuf_printf(sb, "tcpInErrs: %10u %10u\n", 11700 stats.tcp_in_errs[0], stats.tcp_in_errs[1]); 11701 sbuf_printf(sb, "tcp6InErrs: %10u %10u\n", 11702 stats.tcp6_in_errs[0], stats.tcp6_in_errs[1]); 11703 sbuf_printf(sb, "tnlCongDrops: %10u %10u\n", 11704 stats.tnl_cong_drops[0], stats.tnl_cong_drops[1]); 11705 sbuf_printf(sb, "tnlTxDrops: %10u %10u\n", 11706 stats.tnl_tx_drops[0], stats.tnl_tx_drops[1]); 11707 sbuf_printf(sb, "ofldVlanDrops: %10u %10u\n", 11708 stats.ofld_vlan_drops[0], stats.ofld_vlan_drops[1]); 11709 sbuf_printf(sb, "ofldChanDrops: %10u %10u\n\n", 11710 stats.ofld_chan_drops[0], stats.ofld_chan_drops[1]); 11711 } 11712 11713 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 11714 stats.ofld_no_neigh, stats.ofld_cong_defer); 11715 11716 rc = sbuf_finish(sb); 11717 sbuf_delete(sb); 11718 11719 return (rc); 11720 } 11721 11722 static int 11723 sysctl_tnl_stats(SYSCTL_HANDLER_ARGS) 11724 { 11725 struct adapter *sc = arg1; 11726 struct sbuf *sb; 11727 int rc; 11728 struct tp_tnl_stats stats; 11729 11730 rc = 0; 11731 mtx_lock(&sc->reg_lock); 11732 if (hw_off_limits(sc)) 11733 rc = ENXIO; 11734 else 11735 t4_tp_get_tnl_stats(sc, &stats, 1); 11736 mtx_unlock(&sc->reg_lock); 11737 if (rc != 0) 11738 return (rc); 11739 11740 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 11741 if (sb == NULL) 11742 return (ENOMEM); 11743 11744 if (sc->chip_params->nchan > 2) { 11745 sbuf_printf(sb, " channel 0 channel 1" 11746 " channel 2 channel 3\n"); 11747 sbuf_printf(sb, "OutPkts: %10u %10u %10u %10u\n", 11748 stats.out_pkt[0], stats.out_pkt[1], 11749 stats.out_pkt[2], stats.out_pkt[3]); 11750 sbuf_printf(sb, "InPkts: %10u %10u %10u %10u", 11751 stats.in_pkt[0], stats.in_pkt[1], 11752 stats.in_pkt[2], stats.in_pkt[3]); 11753 } else { 11754 sbuf_printf(sb, " channel 0 channel 1\n"); 11755 sbuf_printf(sb, "OutPkts: %10u %10u\n", 11756 stats.out_pkt[0], stats.out_pkt[1]); 11757 sbuf_printf(sb, "InPkts: %10u %10u", 11758 stats.in_pkt[0], stats.in_pkt[1]); 11759 } 11760 11761 rc = sbuf_finish(sb); 11762 sbuf_delete(sb); 11763 11764 return (rc); 11765 } 11766 11767 static int 11768 sysctl_tp_la_mask(SYSCTL_HANDLER_ARGS) 11769 { 11770 struct adapter *sc = arg1; 11771 struct tp_params *tpp = &sc->params.tp; 11772 u_int mask; 11773 int rc; 11774 11775 mask = tpp->la_mask >> 16; 11776 rc = sysctl_handle_int(oidp, &mask, 0, req); 11777 if (rc != 0 || req->newptr == NULL) 11778 return (rc); 11779 if (mask > 0xffff) 11780 return (EINVAL); 11781 mtx_lock(&sc->reg_lock); 11782 if (hw_off_limits(sc)) 11783 rc = ENXIO; 11784 else { 11785 tpp->la_mask = mask << 16; 11786 t4_set_reg_field(sc, A_TP_DBG_LA_CONFIG, 0xffff0000U, 11787 tpp->la_mask); 11788 } 11789 mtx_unlock(&sc->reg_lock); 11790 11791 return (rc); 11792 } 11793 11794 struct field_desc { 11795 const char *name; 11796 u_int start; 11797 u_int width; 11798 }; 11799 11800 static void 11801 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) 11802 { 11803 char buf[32]; 11804 int line_size = 0; 11805 11806 while (f->name) { 11807 uint64_t mask = (1ULL << f->width) - 1; 11808 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, 11809 ((uintmax_t)v >> f->start) & mask); 11810 11811 if (line_size + len >= 79) { 11812 line_size = 8; 11813 sbuf_printf(sb, "\n "); 11814 } 11815 sbuf_printf(sb, "%s ", buf); 11816 line_size += len + 1; 11817 f++; 11818 } 11819 sbuf_printf(sb, "\n"); 11820 } 11821 11822 static const struct field_desc tp_la0[] = { 11823 { "RcfOpCodeOut", 60, 4 }, 11824 { "State", 56, 4 }, 11825 { "WcfState", 52, 4 }, 11826 { "RcfOpcSrcOut", 50, 2 }, 11827 { "CRxError", 49, 1 }, 11828 { "ERxError", 48, 1 }, 11829 { "SanityFailed", 47, 1 }, 11830 { "SpuriousMsg", 46, 1 }, 11831 { "FlushInputMsg", 45, 1 }, 11832 { "FlushInputCpl", 44, 1 }, 11833 { "RssUpBit", 43, 1 }, 11834 { "RssFilterHit", 42, 1 }, 11835 { "Tid", 32, 10 }, 11836 { "InitTcb", 31, 1 }, 11837 { "LineNumber", 24, 7 }, 11838 { "Emsg", 23, 1 }, 11839 { "EdataOut", 22, 1 }, 11840 { "Cmsg", 21, 1 }, 11841 { "CdataOut", 20, 1 }, 11842 { "EreadPdu", 19, 1 }, 11843 { "CreadPdu", 18, 1 }, 11844 { "TunnelPkt", 17, 1 }, 11845 { "RcfPeerFin", 16, 1 }, 11846 { "RcfReasonOut", 12, 4 }, 11847 { "TxCchannel", 10, 2 }, 11848 { "RcfTxChannel", 8, 2 }, 11849 { "RxEchannel", 6, 2 }, 11850 { "RcfRxChannel", 5, 1 }, 11851 { "RcfDataOutSrdy", 4, 1 }, 11852 { "RxDvld", 3, 1 }, 11853 { "RxOoDvld", 2, 1 }, 11854 { "RxCongestion", 1, 1 }, 11855 { "TxCongestion", 0, 1 }, 11856 { NULL } 11857 }; 11858 11859 static const struct field_desc tp_la1[] = { 11860 { "CplCmdIn", 56, 8 }, 11861 { "CplCmdOut", 48, 8 }, 11862 { "ESynOut", 47, 1 }, 11863 { "EAckOut", 46, 1 }, 11864 { "EFinOut", 45, 1 }, 11865 { "ERstOut", 44, 1 }, 11866 { "SynIn", 43, 1 }, 11867 { "AckIn", 42, 1 }, 11868 { "FinIn", 41, 1 }, 11869 { "RstIn", 40, 1 }, 11870 { "DataIn", 39, 1 }, 11871 { "DataInVld", 38, 1 }, 11872 { "PadIn", 37, 1 }, 11873 { "RxBufEmpty", 36, 1 }, 11874 { "RxDdp", 35, 1 }, 11875 { "RxFbCongestion", 34, 1 }, 11876 { "TxFbCongestion", 33, 1 }, 11877 { "TxPktSumSrdy", 32, 1 }, 11878 { "RcfUlpType", 28, 4 }, 11879 { "Eread", 27, 1 }, 11880 { "Ebypass", 26, 1 }, 11881 { "Esave", 25, 1 }, 11882 { "Static0", 24, 1 }, 11883 { "Cread", 23, 1 }, 11884 { "Cbypass", 22, 1 }, 11885 { "Csave", 21, 1 }, 11886 { "CPktOut", 20, 1 }, 11887 { "RxPagePoolFull", 18, 2 }, 11888 { "RxLpbkPkt", 17, 1 }, 11889 { "TxLpbkPkt", 16, 1 }, 11890 { "RxVfValid", 15, 1 }, 11891 { "SynLearned", 14, 1 }, 11892 { "SetDelEntry", 13, 1 }, 11893 { "SetInvEntry", 12, 1 }, 11894 { "CpcmdDvld", 11, 1 }, 11895 { "CpcmdSave", 10, 1 }, 11896 { "RxPstructsFull", 8, 2 }, 11897 { "EpcmdDvld", 7, 1 }, 11898 { "EpcmdFlush", 6, 1 }, 11899 { "EpcmdTrimPrefix", 5, 1 }, 11900 { "EpcmdTrimPostfix", 4, 1 }, 11901 { "ERssIp4Pkt", 3, 1 }, 11902 { "ERssIp6Pkt", 2, 1 }, 11903 { "ERssTcpUdpPkt", 1, 1 }, 11904 { "ERssFceFipPkt", 0, 1 }, 11905 { NULL } 11906 }; 11907 11908 static const struct field_desc tp_la2[] = { 11909 { "CplCmdIn", 56, 8 }, 11910 { "MpsVfVld", 55, 1 }, 11911 { "MpsPf", 52, 3 }, 11912 { "MpsVf", 44, 8 }, 11913 { "SynIn", 43, 1 }, 11914 { "AckIn", 42, 1 }, 11915 { "FinIn", 41, 1 }, 11916 { "RstIn", 40, 1 }, 11917 { "DataIn", 39, 1 }, 11918 { "DataInVld", 38, 1 }, 11919 { "PadIn", 37, 1 }, 11920 { "RxBufEmpty", 36, 1 }, 11921 { "RxDdp", 35, 1 }, 11922 { "RxFbCongestion", 34, 1 }, 11923 { "TxFbCongestion", 33, 1 }, 11924 { "TxPktSumSrdy", 32, 1 }, 11925 { "RcfUlpType", 28, 4 }, 11926 { "Eread", 27, 1 }, 11927 { "Ebypass", 26, 1 }, 11928 { "Esave", 25, 1 }, 11929 { "Static0", 24, 1 }, 11930 { "Cread", 23, 1 }, 11931 { "Cbypass", 22, 1 }, 11932 { "Csave", 21, 1 }, 11933 { "CPktOut", 20, 1 }, 11934 { "RxPagePoolFull", 18, 2 }, 11935 { "RxLpbkPkt", 17, 1 }, 11936 { "TxLpbkPkt", 16, 1 }, 11937 { "RxVfValid", 15, 1 }, 11938 { "SynLearned", 14, 1 }, 11939 { "SetDelEntry", 13, 1 }, 11940 { "SetInvEntry", 12, 1 }, 11941 { "CpcmdDvld", 11, 1 }, 11942 { "CpcmdSave", 10, 1 }, 11943 { "RxPstructsFull", 8, 2 }, 11944 { "EpcmdDvld", 7, 1 }, 11945 { "EpcmdFlush", 6, 1 }, 11946 { "EpcmdTrimPrefix", 5, 1 }, 11947 { "EpcmdTrimPostfix", 4, 1 }, 11948 { "ERssIp4Pkt", 3, 1 }, 11949 { "ERssIp6Pkt", 2, 1 }, 11950 { "ERssTcpUdpPkt", 1, 1 }, 11951 { "ERssFceFipPkt", 0, 1 }, 11952 { NULL } 11953 }; 11954 11955 static void 11956 tp_la_show(struct sbuf *sb, uint64_t *p, int idx) 11957 { 11958 11959 field_desc_show(sb, *p, tp_la0); 11960 } 11961 11962 static void 11963 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) 11964 { 11965 11966 if (idx) 11967 sbuf_printf(sb, "\n"); 11968 field_desc_show(sb, p[0], tp_la0); 11969 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 11970 field_desc_show(sb, p[1], tp_la0); 11971 } 11972 11973 static void 11974 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) 11975 { 11976 11977 if (idx) 11978 sbuf_printf(sb, "\n"); 11979 field_desc_show(sb, p[0], tp_la0); 11980 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 11981 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); 11982 } 11983 11984 static int 11985 sysctl_tp_la(SYSCTL_HANDLER_ARGS) 11986 { 11987 struct adapter *sc = arg1; 11988 struct sbuf *sb; 11989 uint64_t *buf, *p; 11990 int rc; 11991 u_int i, inc; 11992 void (*show_func)(struct sbuf *, uint64_t *, int); 11993 11994 rc = 0; 11995 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 11996 if (sb == NULL) 11997 return (ENOMEM); 11998 11999 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); 12000 12001 mtx_lock(&sc->reg_lock); 12002 if (hw_off_limits(sc)) 12003 rc = ENXIO; 12004 else { 12005 t4_tp_read_la(sc, buf, NULL); 12006 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { 12007 case 2: 12008 inc = 2; 12009 show_func = tp_la_show2; 12010 break; 12011 case 3: 12012 inc = 2; 12013 show_func = tp_la_show3; 12014 break; 12015 default: 12016 inc = 1; 12017 show_func = tp_la_show; 12018 } 12019 } 12020 mtx_unlock(&sc->reg_lock); 12021 if (rc != 0) 12022 goto done; 12023 12024 p = buf; 12025 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) 12026 (*show_func)(sb, p, i); 12027 rc = sbuf_finish(sb); 12028 done: 12029 sbuf_delete(sb); 12030 free(buf, M_CXGBE); 12031 return (rc); 12032 } 12033 12034 static int 12035 sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 12036 { 12037 struct adapter *sc = arg1; 12038 struct sbuf *sb; 12039 int rc; 12040 u64 nrate[MAX_NCHAN], orate[MAX_NCHAN]; 12041 12042 rc = 0; 12043 mtx_lock(&sc->reg_lock); 12044 if (hw_off_limits(sc)) 12045 rc = ENXIO; 12046 else 12047 t4_get_chan_txrate(sc, nrate, orate); 12048 mtx_unlock(&sc->reg_lock); 12049 if (rc != 0) 12050 return (rc); 12051 12052 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 12053 if (sb == NULL) 12054 return (ENOMEM); 12055 12056 if (sc->chip_params->nchan > 2) { 12057 sbuf_printf(sb, " channel 0 channel 1" 12058 " channel 2 channel 3\n"); 12059 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 12060 nrate[0], nrate[1], nrate[2], nrate[3]); 12061 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 12062 orate[0], orate[1], orate[2], orate[3]); 12063 } else { 12064 sbuf_printf(sb, " channel 0 channel 1\n"); 12065 sbuf_printf(sb, "NIC B/s: %10ju %10ju\n", 12066 nrate[0], nrate[1]); 12067 sbuf_printf(sb, "Offload B/s: %10ju %10ju", 12068 orate[0], orate[1]); 12069 } 12070 12071 rc = sbuf_finish(sb); 12072 sbuf_delete(sb); 12073 12074 return (rc); 12075 } 12076 12077 static int 12078 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) 12079 { 12080 struct adapter *sc = arg1; 12081 struct sbuf *sb; 12082 uint32_t *buf, *p; 12083 int rc, i; 12084 12085 rc = 0; 12086 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 12087 if (sb == NULL) 12088 return (ENOMEM); 12089 12090 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, 12091 M_ZERO | M_WAITOK); 12092 12093 mtx_lock(&sc->reg_lock); 12094 if (hw_off_limits(sc)) 12095 rc = ENXIO; 12096 else 12097 t4_ulprx_read_la(sc, buf); 12098 mtx_unlock(&sc->reg_lock); 12099 if (rc != 0) 12100 goto done; 12101 12102 p = buf; 12103 sbuf_printf(sb, " Pcmd Type Message" 12104 " Data"); 12105 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { 12106 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", 12107 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); 12108 } 12109 rc = sbuf_finish(sb); 12110 done: 12111 sbuf_delete(sb); 12112 free(buf, M_CXGBE); 12113 return (rc); 12114 } 12115 12116 static int 12117 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 12118 { 12119 struct adapter *sc = arg1; 12120 struct sbuf *sb; 12121 int rc; 12122 uint32_t cfg, s1, s2; 12123 12124 MPASS(chip_id(sc) >= CHELSIO_T5); 12125 12126 rc = 0; 12127 mtx_lock(&sc->reg_lock); 12128 if (hw_off_limits(sc)) 12129 rc = ENXIO; 12130 else { 12131 cfg = t4_read_reg(sc, A_SGE_STAT_CFG); 12132 s1 = t4_read_reg(sc, A_SGE_STAT_TOTAL); 12133 s2 = t4_read_reg(sc, A_SGE_STAT_MATCH); 12134 } 12135 mtx_unlock(&sc->reg_lock); 12136 if (rc != 0) 12137 return (rc); 12138 12139 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 12140 if (sb == NULL) 12141 return (ENOMEM); 12142 12143 if (G_STATSOURCE_T5(cfg) == 7) { 12144 int mode; 12145 12146 mode = is_t5(sc) ? G_STATMODE(cfg) : G_T6_STATMODE(cfg); 12147 if (mode == 0) 12148 sbuf_printf(sb, "total %d, incomplete %d", s1, s2); 12149 else if (mode == 1) 12150 sbuf_printf(sb, "total %d, data overflow %d", s1, s2); 12151 else 12152 sbuf_printf(sb, "unknown mode %d", mode); 12153 } 12154 rc = sbuf_finish(sb); 12155 sbuf_delete(sb); 12156 12157 return (rc); 12158 } 12159 12160 static int 12161 sysctl_cpus(SYSCTL_HANDLER_ARGS) 12162 { 12163 struct adapter *sc = arg1; 12164 enum cpu_sets op = arg2; 12165 cpuset_t cpuset; 12166 struct sbuf *sb; 12167 int i, rc; 12168 12169 MPASS(op == LOCAL_CPUS || op == INTR_CPUS); 12170 12171 CPU_ZERO(&cpuset); 12172 rc = bus_get_cpus(sc->dev, op, sizeof(cpuset), &cpuset); 12173 if (rc != 0) 12174 return (rc); 12175 12176 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 12177 if (sb == NULL) 12178 return (ENOMEM); 12179 12180 CPU_FOREACH(i) 12181 sbuf_printf(sb, "%d ", i); 12182 rc = sbuf_finish(sb); 12183 sbuf_delete(sb); 12184 12185 return (rc); 12186 } 12187 12188 static int 12189 sysctl_reset(SYSCTL_HANDLER_ARGS) 12190 { 12191 struct adapter *sc = arg1; 12192 u_int val; 12193 int rc; 12194 12195 val = atomic_load_int(&sc->num_resets); 12196 rc = sysctl_handle_int(oidp, &val, 0, req); 12197 if (rc != 0 || req->newptr == NULL) 12198 return (rc); 12199 12200 if (val == 0) { 12201 /* Zero out the counter that tracks reset. */ 12202 atomic_store_int(&sc->num_resets, 0); 12203 return (0); 12204 } 12205 12206 if (val != 1) 12207 return (EINVAL); /* 0 or 1 are the only legal values */ 12208 12209 if (hw_off_limits(sc)) /* harmless race */ 12210 return (EALREADY); 12211 12212 taskqueue_enqueue(reset_tq, &sc->reset_task); 12213 return (0); 12214 } 12215 12216 static int 12217 sysctl_tcb_cache(SYSCTL_HANDLER_ARGS) 12218 { 12219 struct adapter *sc = arg1; 12220 u_int val, v; 12221 int rc; 12222 12223 mtx_lock(&sc->reg_lock); 12224 if (hw_off_limits(sc)) { 12225 rc = ENXIO; 12226 goto done; 12227 } 12228 t4_tp_pio_read(sc, &v, 1, A_TP_CMM_CONFIG, 1); 12229 mtx_unlock(&sc->reg_lock); 12230 12231 val = v & F_GLFL ? 0 : 1; 12232 rc = sysctl_handle_int(oidp, &val, 0, req); 12233 if (rc != 0 || req->newptr == NULL) 12234 return (rc); 12235 if (val == 0) 12236 v |= F_GLFL; 12237 else 12238 v &= ~F_GLFL; 12239 12240 mtx_lock(&sc->reg_lock); 12241 if (hw_off_limits(sc)) 12242 rc = ENXIO; 12243 else 12244 t4_tp_pio_write(sc, &v, 1, A_TP_CMM_CONFIG, 1); 12245 done: 12246 mtx_unlock(&sc->reg_lock); 12247 return (rc); 12248 } 12249 12250 #ifdef TCP_OFFLOAD 12251 static int 12252 sysctl_tls(SYSCTL_HANDLER_ARGS) 12253 { 12254 struct adapter *sc = arg1; 12255 int i, j, v, rc; 12256 struct vi_info *vi; 12257 12258 v = sc->tt.tls; 12259 rc = sysctl_handle_int(oidp, &v, 0, req); 12260 if (rc != 0 || req->newptr == NULL) 12261 return (rc); 12262 12263 if (v != 0 && !(sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS)) 12264 return (ENOTSUP); 12265 12266 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4stls"); 12267 if (rc) 12268 return (rc); 12269 if (hw_off_limits(sc)) 12270 rc = ENXIO; 12271 else { 12272 sc->tt.tls = !!v; 12273 for_each_port(sc, i) { 12274 for_each_vi(sc->port[i], j, vi) { 12275 if (vi->flags & VI_INIT_DONE) 12276 t4_update_fl_bufsize(vi->ifp); 12277 } 12278 } 12279 } 12280 end_synchronized_op(sc, 0); 12281 12282 return (rc); 12283 12284 } 12285 12286 static void 12287 unit_conv(char *buf, size_t len, u_int val, u_int factor) 12288 { 12289 u_int rem = val % factor; 12290 12291 if (rem == 0) 12292 snprintf(buf, len, "%u", val / factor); 12293 else { 12294 while (rem % 10 == 0) 12295 rem /= 10; 12296 snprintf(buf, len, "%u.%u", val / factor, rem); 12297 } 12298 } 12299 12300 static int 12301 sysctl_tp_tick(SYSCTL_HANDLER_ARGS) 12302 { 12303 struct adapter *sc = arg1; 12304 char buf[16]; 12305 u_int res, re; 12306 u_int cclk_ps = 1000000000 / sc->params.vpd.cclk; 12307 12308 mtx_lock(&sc->reg_lock); 12309 if (hw_off_limits(sc)) 12310 res = (u_int)-1; 12311 else 12312 res = t4_read_reg(sc, A_TP_TIMER_RESOLUTION); 12313 mtx_unlock(&sc->reg_lock); 12314 if (res == (u_int)-1) 12315 return (ENXIO); 12316 12317 switch (arg2) { 12318 case 0: 12319 /* timer_tick */ 12320 re = G_TIMERRESOLUTION(res); 12321 break; 12322 case 1: 12323 /* TCP timestamp tick */ 12324 re = G_TIMESTAMPRESOLUTION(res); 12325 break; 12326 case 2: 12327 /* DACK tick */ 12328 re = G_DELAYEDACKRESOLUTION(res); 12329 break; 12330 default: 12331 return (EDOOFUS); 12332 } 12333 12334 unit_conv(buf, sizeof(buf), (cclk_ps << re), 1000000); 12335 12336 return (sysctl_handle_string(oidp, buf, sizeof(buf), req)); 12337 } 12338 12339 static int 12340 sysctl_tp_dack_timer(SYSCTL_HANDLER_ARGS) 12341 { 12342 struct adapter *sc = arg1; 12343 int rc; 12344 u_int dack_tmr, dack_re, v; 12345 u_int cclk_ps = 1000000000 / sc->params.vpd.cclk; 12346 12347 mtx_lock(&sc->reg_lock); 12348 if (hw_off_limits(sc)) 12349 rc = ENXIO; 12350 else { 12351 rc = 0; 12352 dack_re = G_DELAYEDACKRESOLUTION(t4_read_reg(sc, 12353 A_TP_TIMER_RESOLUTION)); 12354 dack_tmr = t4_read_reg(sc, A_TP_DACK_TIMER); 12355 } 12356 mtx_unlock(&sc->reg_lock); 12357 if (rc != 0) 12358 return (rc); 12359 12360 v = ((cclk_ps << dack_re) / 1000000) * dack_tmr; 12361 12362 return (sysctl_handle_int(oidp, &v, 0, req)); 12363 } 12364 12365 static int 12366 sysctl_tp_timer(SYSCTL_HANDLER_ARGS) 12367 { 12368 struct adapter *sc = arg1; 12369 int rc, reg = arg2; 12370 u_int tre; 12371 u_long tp_tick_us, v; 12372 u_int cclk_ps = 1000000000 / sc->params.vpd.cclk; 12373 12374 MPASS(reg == A_TP_RXT_MIN || reg == A_TP_RXT_MAX || 12375 reg == A_TP_PERS_MIN || reg == A_TP_PERS_MAX || 12376 reg == A_TP_KEEP_IDLE || reg == A_TP_KEEP_INTVL || 12377 reg == A_TP_INIT_SRTT || reg == A_TP_FINWAIT2_TIMER); 12378 12379 mtx_lock(&sc->reg_lock); 12380 if (hw_off_limits(sc)) 12381 rc = ENXIO; 12382 else { 12383 rc = 0; 12384 tre = G_TIMERRESOLUTION(t4_read_reg(sc, A_TP_TIMER_RESOLUTION)); 12385 tp_tick_us = (cclk_ps << tre) / 1000000; 12386 if (reg == A_TP_INIT_SRTT) 12387 v = tp_tick_us * G_INITSRTT(t4_read_reg(sc, reg)); 12388 else 12389 v = tp_tick_us * t4_read_reg(sc, reg); 12390 } 12391 mtx_unlock(&sc->reg_lock); 12392 if (rc != 0) 12393 return (rc); 12394 else 12395 return (sysctl_handle_long(oidp, &v, 0, req)); 12396 } 12397 12398 /* 12399 * All fields in TP_SHIFT_CNT are 4b and the starting location of the field is 12400 * passed to this function. 12401 */ 12402 static int 12403 sysctl_tp_shift_cnt(SYSCTL_HANDLER_ARGS) 12404 { 12405 struct adapter *sc = arg1; 12406 int rc, idx = arg2; 12407 u_int v; 12408 12409 MPASS(idx >= 0 && idx <= 24); 12410 12411 mtx_lock(&sc->reg_lock); 12412 if (hw_off_limits(sc)) 12413 rc = ENXIO; 12414 else { 12415 rc = 0; 12416 v = (t4_read_reg(sc, A_TP_SHIFT_CNT) >> idx) & 0xf; 12417 } 12418 mtx_unlock(&sc->reg_lock); 12419 if (rc != 0) 12420 return (rc); 12421 else 12422 return (sysctl_handle_int(oidp, &v, 0, req)); 12423 } 12424 12425 static int 12426 sysctl_tp_backoff(SYSCTL_HANDLER_ARGS) 12427 { 12428 struct adapter *sc = arg1; 12429 int rc, idx = arg2; 12430 u_int shift, v, r; 12431 12432 MPASS(idx >= 0 && idx < 16); 12433 12434 r = A_TP_TCP_BACKOFF_REG0 + (idx & ~3); 12435 shift = (idx & 3) << 3; 12436 mtx_lock(&sc->reg_lock); 12437 if (hw_off_limits(sc)) 12438 rc = ENXIO; 12439 else { 12440 rc = 0; 12441 v = (t4_read_reg(sc, r) >> shift) & M_TIMERBACKOFFINDEX0; 12442 } 12443 mtx_unlock(&sc->reg_lock); 12444 if (rc != 0) 12445 return (rc); 12446 else 12447 return (sysctl_handle_int(oidp, &v, 0, req)); 12448 } 12449 12450 static int 12451 sysctl_holdoff_tmr_idx_ofld(SYSCTL_HANDLER_ARGS) 12452 { 12453 struct vi_info *vi = arg1; 12454 struct adapter *sc = vi->adapter; 12455 int idx, rc, i; 12456 struct sge_ofld_rxq *ofld_rxq; 12457 uint8_t v; 12458 12459 idx = vi->ofld_tmr_idx; 12460 12461 rc = sysctl_handle_int(oidp, &idx, 0, req); 12462 if (rc != 0 || req->newptr == NULL) 12463 return (rc); 12464 12465 if (idx < 0 || idx >= SGE_NTIMERS) 12466 return (EINVAL); 12467 12468 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 12469 "t4otmr"); 12470 if (rc) 12471 return (rc); 12472 12473 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(vi->ofld_pktc_idx != -1); 12474 for_each_ofld_rxq(vi, i, ofld_rxq) { 12475 #ifdef atomic_store_rel_8 12476 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); 12477 #else 12478 ofld_rxq->iq.intr_params = v; 12479 #endif 12480 } 12481 vi->ofld_tmr_idx = idx; 12482 12483 end_synchronized_op(sc, LOCK_HELD); 12484 return (0); 12485 } 12486 12487 static int 12488 sysctl_holdoff_pktc_idx_ofld(SYSCTL_HANDLER_ARGS) 12489 { 12490 struct vi_info *vi = arg1; 12491 struct adapter *sc = vi->adapter; 12492 int idx, rc; 12493 12494 idx = vi->ofld_pktc_idx; 12495 12496 rc = sysctl_handle_int(oidp, &idx, 0, req); 12497 if (rc != 0 || req->newptr == NULL) 12498 return (rc); 12499 12500 if (idx < -1 || idx >= SGE_NCOUNTERS) 12501 return (EINVAL); 12502 12503 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 12504 "t4opktc"); 12505 if (rc) 12506 return (rc); 12507 12508 if (vi->flags & VI_INIT_DONE) 12509 rc = EBUSY; /* cannot be changed once the queues are created */ 12510 else 12511 vi->ofld_pktc_idx = idx; 12512 12513 end_synchronized_op(sc, LOCK_HELD); 12514 return (rc); 12515 } 12516 #endif 12517 12518 static int 12519 get_sge_context(struct adapter *sc, int mem_id, uint32_t cid, int len, 12520 uint32_t *data) 12521 { 12522 int rc; 12523 12524 if (len < sc->chip_params->sge_ctxt_size) 12525 return (ENOBUFS); 12526 if (cid > M_CTXTQID) 12527 return (EINVAL); 12528 if (mem_id != CTXT_EGRESS && mem_id != CTXT_INGRESS && 12529 mem_id != CTXT_FLM && mem_id != CTXT_CNM) 12530 return (EINVAL); 12531 12532 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 12533 if (rc) 12534 return (rc); 12535 12536 if (hw_off_limits(sc)) { 12537 rc = ENXIO; 12538 goto done; 12539 } 12540 12541 if (sc->flags & FW_OK) { 12542 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cid, mem_id, data); 12543 if (rc == 0) 12544 goto done; 12545 } 12546 12547 /* 12548 * Read via firmware failed or wasn't even attempted. Read directly via 12549 * the backdoor. 12550 */ 12551 rc = -t4_sge_ctxt_rd_bd(sc, cid, mem_id, data); 12552 done: 12553 end_synchronized_op(sc, 0); 12554 return (rc); 12555 } 12556 12557 static int 12558 load_fw(struct adapter *sc, struct t4_data *fw) 12559 { 12560 int rc; 12561 uint8_t *fw_data; 12562 12563 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 12564 if (rc) 12565 return (rc); 12566 12567 if (hw_off_limits(sc)) { 12568 rc = ENXIO; 12569 goto done; 12570 } 12571 12572 /* 12573 * The firmware, with the sole exception of the memory parity error 12574 * handler, runs from memory and not flash. It is almost always safe to 12575 * install a new firmware on a running system. Just set bit 1 in 12576 * hw.cxgbe.dflags or dev.<nexus>.<n>.dflags first. 12577 */ 12578 if (sc->flags & FULL_INIT_DONE && 12579 (sc->debug_flags & DF_LOAD_FW_ANYTIME) == 0) { 12580 rc = EBUSY; 12581 goto done; 12582 } 12583 12584 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 12585 12586 rc = copyin(fw->data, fw_data, fw->len); 12587 if (rc == 0) 12588 rc = -t4_load_fw(sc, fw_data, fw->len); 12589 12590 free(fw_data, M_CXGBE); 12591 done: 12592 end_synchronized_op(sc, 0); 12593 return (rc); 12594 } 12595 12596 static int 12597 load_cfg(struct adapter *sc, struct t4_data *cfg) 12598 { 12599 int rc; 12600 uint8_t *cfg_data = NULL; 12601 12602 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldcf"); 12603 if (rc) 12604 return (rc); 12605 12606 if (hw_off_limits(sc)) { 12607 rc = ENXIO; 12608 goto done; 12609 } 12610 12611 if (cfg->len == 0) { 12612 /* clear */ 12613 rc = -t4_load_cfg(sc, NULL, 0); 12614 goto done; 12615 } 12616 12617 cfg_data = malloc(cfg->len, M_CXGBE, M_WAITOK); 12618 12619 rc = copyin(cfg->data, cfg_data, cfg->len); 12620 if (rc == 0) 12621 rc = -t4_load_cfg(sc, cfg_data, cfg->len); 12622 12623 free(cfg_data, M_CXGBE); 12624 done: 12625 end_synchronized_op(sc, 0); 12626 return (rc); 12627 } 12628 12629 static int 12630 load_boot(struct adapter *sc, struct t4_bootrom *br) 12631 { 12632 int rc; 12633 uint8_t *br_data = NULL; 12634 u_int offset; 12635 12636 if (br->len > 1024 * 1024) 12637 return (EFBIG); 12638 12639 if (br->pf_offset == 0) { 12640 /* pfidx */ 12641 if (br->pfidx_addr > 7) 12642 return (EINVAL); 12643 offset = G_OFFSET(t4_read_reg(sc, PF_REG(br->pfidx_addr, 12644 A_PCIE_PF_EXPROM_OFST))); 12645 } else if (br->pf_offset == 1) { 12646 /* offset */ 12647 offset = G_OFFSET(br->pfidx_addr); 12648 } else { 12649 return (EINVAL); 12650 } 12651 12652 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldbr"); 12653 if (rc) 12654 return (rc); 12655 12656 if (hw_off_limits(sc)) { 12657 rc = ENXIO; 12658 goto done; 12659 } 12660 12661 if (br->len == 0) { 12662 /* clear */ 12663 rc = -t4_load_boot(sc, NULL, offset, 0); 12664 goto done; 12665 } 12666 12667 br_data = malloc(br->len, M_CXGBE, M_WAITOK); 12668 12669 rc = copyin(br->data, br_data, br->len); 12670 if (rc == 0) 12671 rc = -t4_load_boot(sc, br_data, offset, br->len); 12672 12673 free(br_data, M_CXGBE); 12674 done: 12675 end_synchronized_op(sc, 0); 12676 return (rc); 12677 } 12678 12679 static int 12680 load_bootcfg(struct adapter *sc, struct t4_data *bc) 12681 { 12682 int rc; 12683 uint8_t *bc_data = NULL; 12684 12685 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldcf"); 12686 if (rc) 12687 return (rc); 12688 12689 if (hw_off_limits(sc)) { 12690 rc = ENXIO; 12691 goto done; 12692 } 12693 12694 if (bc->len == 0) { 12695 /* clear */ 12696 rc = -t4_load_bootcfg(sc, NULL, 0); 12697 goto done; 12698 } 12699 12700 bc_data = malloc(bc->len, M_CXGBE, M_WAITOK); 12701 12702 rc = copyin(bc->data, bc_data, bc->len); 12703 if (rc == 0) 12704 rc = -t4_load_bootcfg(sc, bc_data, bc->len); 12705 12706 free(bc_data, M_CXGBE); 12707 done: 12708 end_synchronized_op(sc, 0); 12709 return (rc); 12710 } 12711 12712 static int 12713 cudbg_dump(struct adapter *sc, struct t4_cudbg_dump *dump) 12714 { 12715 int rc; 12716 struct cudbg_init *cudbg; 12717 void *handle, *buf; 12718 12719 /* buf is large, don't block if no memory is available */ 12720 buf = malloc(dump->len, M_CXGBE, M_NOWAIT | M_ZERO); 12721 if (buf == NULL) 12722 return (ENOMEM); 12723 12724 handle = cudbg_alloc_handle(); 12725 if (handle == NULL) { 12726 rc = ENOMEM; 12727 goto done; 12728 } 12729 12730 cudbg = cudbg_get_init(handle); 12731 cudbg->adap = sc; 12732 cudbg->print = (cudbg_print_cb)printf; 12733 12734 #ifndef notyet 12735 device_printf(sc->dev, "%s: wr_flash %u, len %u, data %p.\n", 12736 __func__, dump->wr_flash, dump->len, dump->data); 12737 #endif 12738 12739 if (dump->wr_flash) 12740 cudbg->use_flash = 1; 12741 MPASS(sizeof(cudbg->dbg_bitmap) == sizeof(dump->bitmap)); 12742 memcpy(cudbg->dbg_bitmap, dump->bitmap, sizeof(cudbg->dbg_bitmap)); 12743 12744 rc = cudbg_collect(handle, buf, &dump->len); 12745 if (rc != 0) 12746 goto done; 12747 12748 rc = copyout(buf, dump->data, dump->len); 12749 done: 12750 cudbg_free_handle(handle); 12751 free(buf, M_CXGBE); 12752 return (rc); 12753 } 12754 12755 static void 12756 free_offload_policy(struct t4_offload_policy *op) 12757 { 12758 struct offload_rule *r; 12759 int i; 12760 12761 if (op == NULL) 12762 return; 12763 12764 r = &op->rule[0]; 12765 for (i = 0; i < op->nrules; i++, r++) { 12766 free(r->bpf_prog.bf_insns, M_CXGBE); 12767 } 12768 free(op->rule, M_CXGBE); 12769 free(op, M_CXGBE); 12770 } 12771 12772 static int 12773 set_offload_policy(struct adapter *sc, struct t4_offload_policy *uop) 12774 { 12775 int i, rc, len; 12776 struct t4_offload_policy *op, *old; 12777 struct bpf_program *bf; 12778 const struct offload_settings *s; 12779 struct offload_rule *r; 12780 void *u; 12781 12782 if (!is_offload(sc)) 12783 return (ENODEV); 12784 12785 if (uop->nrules == 0) { 12786 /* Delete installed policies. */ 12787 op = NULL; 12788 goto set_policy; 12789 } else if (uop->nrules > 256) { /* arbitrary */ 12790 return (E2BIG); 12791 } 12792 12793 /* Copy userspace offload policy to kernel */ 12794 op = malloc(sizeof(*op), M_CXGBE, M_ZERO | M_WAITOK); 12795 op->nrules = uop->nrules; 12796 len = op->nrules * sizeof(struct offload_rule); 12797 op->rule = malloc(len, M_CXGBE, M_ZERO | M_WAITOK); 12798 rc = copyin(uop->rule, op->rule, len); 12799 if (rc) { 12800 free(op->rule, M_CXGBE); 12801 free(op, M_CXGBE); 12802 return (rc); 12803 } 12804 12805 r = &op->rule[0]; 12806 for (i = 0; i < op->nrules; i++, r++) { 12807 12808 /* Validate open_type */ 12809 if (r->open_type != OPEN_TYPE_LISTEN && 12810 r->open_type != OPEN_TYPE_ACTIVE && 12811 r->open_type != OPEN_TYPE_PASSIVE && 12812 r->open_type != OPEN_TYPE_DONTCARE) { 12813 error: 12814 /* 12815 * Rules 0 to i have malloc'd filters that need to be 12816 * freed. Rules i+1 to nrules have userspace pointers 12817 * and should be left alone. 12818 */ 12819 op->nrules = i; 12820 free_offload_policy(op); 12821 return (rc); 12822 } 12823 12824 /* Validate settings */ 12825 s = &r->settings; 12826 if ((s->offload != 0 && s->offload != 1) || 12827 s->cong_algo < -1 || s->cong_algo > CONG_ALG_HIGHSPEED || 12828 s->sched_class < -1 || 12829 s->sched_class >= sc->params.nsched_cls) { 12830 rc = EINVAL; 12831 goto error; 12832 } 12833 12834 bf = &r->bpf_prog; 12835 u = bf->bf_insns; /* userspace ptr */ 12836 bf->bf_insns = NULL; 12837 if (bf->bf_len == 0) { 12838 /* legal, matches everything */ 12839 continue; 12840 } 12841 len = bf->bf_len * sizeof(*bf->bf_insns); 12842 bf->bf_insns = malloc(len, M_CXGBE, M_ZERO | M_WAITOK); 12843 rc = copyin(u, bf->bf_insns, len); 12844 if (rc != 0) 12845 goto error; 12846 12847 if (!bpf_validate(bf->bf_insns, bf->bf_len)) { 12848 rc = EINVAL; 12849 goto error; 12850 } 12851 } 12852 set_policy: 12853 rw_wlock(&sc->policy_lock); 12854 old = sc->policy; 12855 sc->policy = op; 12856 rw_wunlock(&sc->policy_lock); 12857 free_offload_policy(old); 12858 12859 return (0); 12860 } 12861 12862 #define MAX_READ_BUF_SIZE (128 * 1024) 12863 static int 12864 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 12865 { 12866 uint32_t addr, remaining, n; 12867 uint32_t *buf; 12868 int rc; 12869 uint8_t *dst; 12870 12871 mtx_lock(&sc->reg_lock); 12872 if (hw_off_limits(sc)) 12873 rc = ENXIO; 12874 else 12875 rc = validate_mem_range(sc, mr->addr, mr->len); 12876 mtx_unlock(&sc->reg_lock); 12877 if (rc != 0) 12878 return (rc); 12879 12880 buf = malloc(min(mr->len, MAX_READ_BUF_SIZE), M_CXGBE, M_WAITOK); 12881 addr = mr->addr; 12882 remaining = mr->len; 12883 dst = (void *)mr->data; 12884 12885 while (remaining) { 12886 n = min(remaining, MAX_READ_BUF_SIZE); 12887 mtx_lock(&sc->reg_lock); 12888 if (hw_off_limits(sc)) 12889 rc = ENXIO; 12890 else 12891 read_via_memwin(sc, 2, addr, buf, n); 12892 mtx_unlock(&sc->reg_lock); 12893 if (rc != 0) 12894 break; 12895 12896 rc = copyout(buf, dst, n); 12897 if (rc != 0) 12898 break; 12899 12900 dst += n; 12901 remaining -= n; 12902 addr += n; 12903 } 12904 12905 free(buf, M_CXGBE); 12906 return (rc); 12907 } 12908 #undef MAX_READ_BUF_SIZE 12909 12910 static int 12911 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 12912 { 12913 int rc; 12914 12915 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 12916 return (EINVAL); 12917 12918 if (i2cd->len > sizeof(i2cd->data)) 12919 return (EFBIG); 12920 12921 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 12922 if (rc) 12923 return (rc); 12924 if (hw_off_limits(sc)) 12925 rc = ENXIO; 12926 else 12927 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 12928 i2cd->offset, i2cd->len, &i2cd->data[0]); 12929 end_synchronized_op(sc, 0); 12930 12931 return (rc); 12932 } 12933 12934 static int 12935 clear_stats(struct adapter *sc, u_int port_id) 12936 { 12937 int i, v, chan_map; 12938 struct port_info *pi; 12939 struct vi_info *vi; 12940 struct sge_rxq *rxq; 12941 struct sge_txq *txq; 12942 struct sge_wrq *wrq; 12943 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 12944 struct sge_ofld_txq *ofld_txq; 12945 #endif 12946 #ifdef TCP_OFFLOAD 12947 struct sge_ofld_rxq *ofld_rxq; 12948 #endif 12949 12950 if (port_id >= sc->params.nports) 12951 return (EINVAL); 12952 pi = sc->port[port_id]; 12953 if (pi == NULL) 12954 return (EIO); 12955 12956 mtx_lock(&sc->reg_lock); 12957 if (!hw_off_limits(sc)) { 12958 /* MAC stats */ 12959 t4_clr_port_stats(sc, pi->hw_port); 12960 if (is_t6(sc)) { 12961 if (pi->fcs_reg != -1) 12962 pi->fcs_base = t4_read_reg64(sc, 12963 t4_port_reg(sc, pi->tx_chan, pi->fcs_reg)); 12964 else 12965 pi->stats.rx_fcs_err = 0; 12966 } 12967 for_each_vi(pi, v, vi) { 12968 if (vi->flags & VI_INIT_DONE) 12969 t4_clr_vi_stats(sc, vi->vin); 12970 } 12971 chan_map = pi->rx_e_chan_map; 12972 v = 0; /* reuse */ 12973 while (chan_map) { 12974 i = ffs(chan_map) - 1; 12975 t4_write_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 12976 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 12977 chan_map &= ~(1 << i); 12978 } 12979 } 12980 mtx_unlock(&sc->reg_lock); 12981 pi->tx_parse_error = 0; 12982 pi->tnl_cong_drops = 0; 12983 12984 /* 12985 * Since this command accepts a port, clear stats for 12986 * all VIs on this port. 12987 */ 12988 for_each_vi(pi, v, vi) { 12989 if (vi->flags & VI_INIT_DONE) { 12990 12991 for_each_rxq(vi, i, rxq) { 12992 #if defined(INET) || defined(INET6) 12993 rxq->lro.lro_queued = 0; 12994 rxq->lro.lro_flushed = 0; 12995 #endif 12996 rxq->rxcsum = 0; 12997 rxq->vlan_extraction = 0; 12998 rxq->vxlan_rxcsum = 0; 12999 13000 rxq->fl.cl_allocated = 0; 13001 rxq->fl.cl_recycled = 0; 13002 rxq->fl.cl_fast_recycled = 0; 13003 } 13004 13005 for_each_txq(vi, i, txq) { 13006 txq->txcsum = 0; 13007 txq->tso_wrs = 0; 13008 txq->vlan_insertion = 0; 13009 txq->imm_wrs = 0; 13010 txq->sgl_wrs = 0; 13011 txq->txpkt_wrs = 0; 13012 txq->txpkts0_wrs = 0; 13013 txq->txpkts1_wrs = 0; 13014 txq->txpkts0_pkts = 0; 13015 txq->txpkts1_pkts = 0; 13016 txq->txpkts_flush = 0; 13017 txq->raw_wrs = 0; 13018 txq->vxlan_tso_wrs = 0; 13019 txq->vxlan_txcsum = 0; 13020 txq->kern_tls_records = 0; 13021 txq->kern_tls_short = 0; 13022 txq->kern_tls_partial = 0; 13023 txq->kern_tls_full = 0; 13024 txq->kern_tls_octets = 0; 13025 txq->kern_tls_waste = 0; 13026 txq->kern_tls_header = 0; 13027 txq->kern_tls_fin_short = 0; 13028 txq->kern_tls_cbc = 0; 13029 txq->kern_tls_gcm = 0; 13030 if (is_t6(sc)) { 13031 txq->kern_tls_options = 0; 13032 txq->kern_tls_fin = 0; 13033 } else { 13034 txq->kern_tls_ghash_received = 0; 13035 txq->kern_tls_ghash_requested = 0; 13036 txq->kern_tls_lso = 0; 13037 txq->kern_tls_partial_ghash = 0; 13038 txq->kern_tls_splitmode = 0; 13039 txq->kern_tls_trailer = 0; 13040 } 13041 mp_ring_reset_stats(txq->r); 13042 } 13043 13044 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 13045 for_each_ofld_txq(vi, i, ofld_txq) { 13046 ofld_txq->wrq.tx_wrs_direct = 0; 13047 ofld_txq->wrq.tx_wrs_copied = 0; 13048 counter_u64_zero(ofld_txq->tx_iscsi_pdus); 13049 counter_u64_zero(ofld_txq->tx_iscsi_octets); 13050 counter_u64_zero(ofld_txq->tx_iscsi_iso_wrs); 13051 counter_u64_zero(ofld_txq->tx_nvme_pdus); 13052 counter_u64_zero(ofld_txq->tx_nvme_octets); 13053 counter_u64_zero(ofld_txq->tx_nvme_iso_wrs); 13054 counter_u64_zero(ofld_txq->tx_aio_jobs); 13055 counter_u64_zero(ofld_txq->tx_aio_octets); 13056 counter_u64_zero(ofld_txq->tx_toe_tls_records); 13057 counter_u64_zero(ofld_txq->tx_toe_tls_octets); 13058 } 13059 #endif 13060 #ifdef TCP_OFFLOAD 13061 for_each_ofld_rxq(vi, i, ofld_rxq) { 13062 ofld_rxq->fl.cl_allocated = 0; 13063 ofld_rxq->fl.cl_recycled = 0; 13064 ofld_rxq->fl.cl_fast_recycled = 0; 13065 counter_u64_zero( 13066 ofld_rxq->rx_iscsi_ddp_setup_ok); 13067 counter_u64_zero( 13068 ofld_rxq->rx_iscsi_ddp_setup_error); 13069 ofld_rxq->rx_iscsi_ddp_pdus = 0; 13070 ofld_rxq->rx_iscsi_ddp_octets = 0; 13071 ofld_rxq->rx_iscsi_fl_pdus = 0; 13072 ofld_rxq->rx_iscsi_fl_octets = 0; 13073 counter_u64_zero( 13074 ofld_rxq->rx_nvme_ddp_setup_ok); 13075 counter_u64_zero( 13076 ofld_rxq->rx_nvme_ddp_setup_no_stag); 13077 counter_u64_zero( 13078 ofld_rxq->rx_nvme_ddp_setup_error); 13079 counter_u64_zero(ofld_rxq->rx_nvme_ddp_pdus); 13080 counter_u64_zero(ofld_rxq->rx_nvme_ddp_octets); 13081 counter_u64_zero(ofld_rxq->rx_nvme_fl_pdus); 13082 counter_u64_zero(ofld_rxq->rx_nvme_fl_octets); 13083 counter_u64_zero( 13084 ofld_rxq->rx_nvme_invalid_headers); 13085 counter_u64_zero( 13086 ofld_rxq->rx_nvme_header_digest_errors); 13087 counter_u64_zero( 13088 ofld_rxq->rx_nvme_data_digest_errors); 13089 ofld_rxq->rx_aio_ddp_jobs = 0; 13090 ofld_rxq->rx_aio_ddp_octets = 0; 13091 ofld_rxq->rx_toe_tls_records = 0; 13092 ofld_rxq->rx_toe_tls_octets = 0; 13093 ofld_rxq->rx_toe_ddp_octets = 0; 13094 counter_u64_zero(ofld_rxq->ddp_buffer_alloc); 13095 counter_u64_zero(ofld_rxq->ddp_buffer_reuse); 13096 counter_u64_zero(ofld_rxq->ddp_buffer_free); 13097 } 13098 #endif 13099 13100 if (IS_MAIN_VI(vi)) { 13101 wrq = &sc->sge.ctrlq[pi->port_id]; 13102 wrq->tx_wrs_direct = 0; 13103 wrq->tx_wrs_copied = 0; 13104 } 13105 } 13106 } 13107 13108 return (0); 13109 } 13110 13111 static int 13112 hold_clip_addr(struct adapter *sc, struct t4_clip_addr *ca) 13113 { 13114 #ifdef INET6 13115 struct in6_addr in6; 13116 13117 bcopy(&ca->addr[0], &in6.s6_addr[0], sizeof(in6.s6_addr)); 13118 if (t4_get_clip_entry(sc, &in6, true) != NULL) 13119 return (0); 13120 else 13121 return (EIO); 13122 #else 13123 return (ENOTSUP); 13124 #endif 13125 } 13126 13127 static int 13128 release_clip_addr(struct adapter *sc, struct t4_clip_addr *ca) 13129 { 13130 #ifdef INET6 13131 struct in6_addr in6; 13132 13133 bcopy(&ca->addr[0], &in6.s6_addr[0], sizeof(in6.s6_addr)); 13134 return (t4_release_clip_addr(sc, &in6)); 13135 #else 13136 return (ENOTSUP); 13137 #endif 13138 } 13139 13140 int 13141 t4_os_find_pci_capability(struct adapter *sc, int cap) 13142 { 13143 int i; 13144 13145 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 13146 } 13147 13148 void 13149 t4_os_portmod_changed(struct port_info *pi) 13150 { 13151 struct adapter *sc = pi->adapter; 13152 struct vi_info *vi; 13153 if_t ifp; 13154 static const char *mod_str[] = { 13155 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM", 13156 "LR_SIMPLEX", "DR" 13157 }; 13158 13159 KASSERT((pi->flags & FIXED_IFMEDIA) == 0, 13160 ("%s: port_type %u", __func__, pi->port_type)); 13161 13162 vi = &pi->vi[0]; 13163 if (begin_synchronized_op(sc, vi, HOLD_LOCK, "t4mod") == 0) { 13164 PORT_LOCK(pi); 13165 build_medialist(pi); 13166 if (pi->mod_type != FW_PORT_MOD_TYPE_NONE) { 13167 fixup_link_config(pi); 13168 apply_link_config(pi); 13169 } 13170 PORT_UNLOCK(pi); 13171 end_synchronized_op(sc, LOCK_HELD); 13172 } 13173 13174 ifp = vi->ifp; 13175 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 13176 if_printf(ifp, "transceiver unplugged.\n"); 13177 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 13178 if_printf(ifp, "unknown transceiver inserted.\n"); 13179 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 13180 if_printf(ifp, "unsupported transceiver inserted.\n"); 13181 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 13182 if_printf(ifp, "%dGbps %s transceiver inserted.\n", 13183 port_top_speed(pi), mod_str[pi->mod_type]); 13184 } else { 13185 if_printf(ifp, "transceiver (type %d) inserted.\n", 13186 pi->mod_type); 13187 } 13188 } 13189 13190 void 13191 t4_os_link_changed(struct port_info *pi) 13192 { 13193 struct vi_info *vi; 13194 if_t ifp; 13195 struct link_config *lc = &pi->link_cfg; 13196 struct adapter *sc = pi->adapter; 13197 int v; 13198 13199 PORT_LOCK_ASSERT_OWNED(pi); 13200 13201 if (is_t6(sc)) { 13202 if (lc->link_ok) { 13203 if (lc->speed > 25000 || 13204 (lc->speed == 25000 && lc->fec == FEC_RS)) 13205 pi->fcs_reg = A_MAC_PORT_AFRAMECHECKSEQUENCEERRORS; 13206 else 13207 pi->fcs_reg = A_MAC_PORT_MTIP_1G10G_RX_CRCERRORS; 13208 pi->fcs_base = t4_read_reg64(sc, 13209 t4_port_reg(sc, pi->tx_chan, pi->fcs_reg)); 13210 pi->stats.rx_fcs_err = 0; 13211 } else { 13212 pi->fcs_reg = -1; 13213 } 13214 } else { 13215 MPASS(pi->fcs_reg != -1); 13216 MPASS(pi->fcs_base == 0); 13217 } 13218 13219 for_each_vi(pi, v, vi) { 13220 ifp = vi->ifp; 13221 if (ifp == NULL || IS_DETACHING(vi)) 13222 continue; 13223 13224 if (lc->link_ok) { 13225 if_setbaudrate(ifp, IF_Mbps(lc->speed)); 13226 if_link_state_change(ifp, LINK_STATE_UP); 13227 } else { 13228 if_link_state_change(ifp, LINK_STATE_DOWN); 13229 } 13230 } 13231 } 13232 13233 void 13234 t4_iterate(void (*func)(struct adapter *, void *), void *arg) 13235 { 13236 struct adapter *sc; 13237 13238 sx_slock(&t4_list_lock); 13239 SLIST_FOREACH(sc, &t4_list, link) { 13240 /* 13241 * func should not make any assumptions about what state sc is 13242 * in - the only guarantee is that sc->sc_lock is a valid lock. 13243 */ 13244 func(sc, arg); 13245 } 13246 sx_sunlock(&t4_list_lock); 13247 } 13248 13249 static int 13250 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 13251 struct thread *td) 13252 { 13253 int rc; 13254 struct adapter *sc = dev->si_drv1; 13255 13256 rc = priv_check(td, PRIV_DRIVER); 13257 if (rc != 0) 13258 return (rc); 13259 13260 switch (cmd) { 13261 case CHELSIO_T4_GETREG: { 13262 struct t4_reg *edata = (struct t4_reg *)data; 13263 13264 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 13265 return (EFAULT); 13266 13267 mtx_lock(&sc->reg_lock); 13268 if (hw_off_limits(sc)) 13269 rc = ENXIO; 13270 else if (edata->size == 4) 13271 edata->val = t4_read_reg(sc, edata->addr); 13272 else if (edata->size == 8) 13273 edata->val = t4_read_reg64(sc, edata->addr); 13274 else 13275 rc = EINVAL; 13276 mtx_unlock(&sc->reg_lock); 13277 13278 break; 13279 } 13280 case CHELSIO_T4_SETREG: { 13281 struct t4_reg *edata = (struct t4_reg *)data; 13282 13283 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 13284 return (EFAULT); 13285 13286 mtx_lock(&sc->reg_lock); 13287 if (hw_off_limits(sc)) 13288 rc = ENXIO; 13289 else if (edata->size == 4) { 13290 if (edata->val & 0xffffffff00000000) 13291 rc = EINVAL; 13292 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 13293 } else if (edata->size == 8) 13294 t4_write_reg64(sc, edata->addr, edata->val); 13295 else 13296 rc = EINVAL; 13297 mtx_unlock(&sc->reg_lock); 13298 13299 break; 13300 } 13301 case CHELSIO_T4_REGDUMP: { 13302 struct t4_regdump *regs = (struct t4_regdump *)data; 13303 int reglen = t4_get_regs_len(sc); 13304 uint8_t *buf; 13305 13306 if (regs->len < reglen) { 13307 regs->len = reglen; /* hint to the caller */ 13308 return (ENOBUFS); 13309 } 13310 13311 regs->len = reglen; 13312 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 13313 mtx_lock(&sc->reg_lock); 13314 if (hw_off_limits(sc)) 13315 rc = ENXIO; 13316 else 13317 get_regs(sc, regs, buf); 13318 mtx_unlock(&sc->reg_lock); 13319 if (rc == 0) 13320 rc = copyout(buf, regs->data, reglen); 13321 free(buf, M_CXGBE); 13322 break; 13323 } 13324 case CHELSIO_T4_GET_FILTER_MODE: 13325 rc = get_filter_mode(sc, (uint32_t *)data); 13326 break; 13327 case CHELSIO_T4_SET_FILTER_MODE: 13328 rc = set_filter_mode(sc, *(uint32_t *)data); 13329 break; 13330 case CHELSIO_T4_SET_FILTER_MASK: 13331 rc = set_filter_mask(sc, *(uint32_t *)data); 13332 break; 13333 case CHELSIO_T4_GET_FILTER: 13334 rc = get_filter(sc, (struct t4_filter *)data); 13335 break; 13336 case CHELSIO_T4_SET_FILTER: 13337 rc = set_filter(sc, (struct t4_filter *)data); 13338 break; 13339 case CHELSIO_T4_DEL_FILTER: 13340 rc = del_filter(sc, (struct t4_filter *)data); 13341 break; 13342 case CHELSIO_T4_GET_SGE_CONTEXT: { 13343 struct t4_sge_context *ctxt = (struct t4_sge_context *)data; 13344 13345 rc = get_sge_context(sc, ctxt->mem_id, ctxt->cid, 13346 sizeof(ctxt->data), &ctxt->data[0]); 13347 break; 13348 } 13349 case CHELSIO_T4_LOAD_FW: 13350 rc = load_fw(sc, (struct t4_data *)data); 13351 break; 13352 case CHELSIO_T4_GET_MEM: 13353 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 13354 break; 13355 case CHELSIO_T4_GET_I2C: 13356 rc = read_i2c(sc, (struct t4_i2c_data *)data); 13357 break; 13358 case CHELSIO_T4_CLEAR_STATS: 13359 rc = clear_stats(sc, *(uint32_t *)data); 13360 break; 13361 case CHELSIO_T4_SCHED_CLASS: 13362 rc = t4_set_sched_class(sc, (struct t4_sched_params *)data); 13363 break; 13364 case CHELSIO_T4_SCHED_QUEUE: 13365 rc = t4_set_sched_queue(sc, (struct t4_sched_queue *)data); 13366 break; 13367 case CHELSIO_T4_GET_TRACER: 13368 rc = t4_get_tracer(sc, (struct t4_tracer *)data); 13369 break; 13370 case CHELSIO_T4_SET_TRACER: 13371 rc = t4_set_tracer(sc, (struct t4_tracer *)data); 13372 break; 13373 case CHELSIO_T4_LOAD_CFG: 13374 rc = load_cfg(sc, (struct t4_data *)data); 13375 break; 13376 case CHELSIO_T4_LOAD_BOOT: 13377 rc = load_boot(sc, (struct t4_bootrom *)data); 13378 break; 13379 case CHELSIO_T4_LOAD_BOOTCFG: 13380 rc = load_bootcfg(sc, (struct t4_data *)data); 13381 break; 13382 case CHELSIO_T4_CUDBG_DUMP: 13383 rc = cudbg_dump(sc, (struct t4_cudbg_dump *)data); 13384 break; 13385 case CHELSIO_T4_SET_OFLD_POLICY: 13386 rc = set_offload_policy(sc, (struct t4_offload_policy *)data); 13387 break; 13388 case CHELSIO_T4_HOLD_CLIP_ADDR: 13389 rc = hold_clip_addr(sc, (struct t4_clip_addr *)data); 13390 break; 13391 case CHELSIO_T4_RELEASE_CLIP_ADDR: 13392 rc = release_clip_addr(sc, (struct t4_clip_addr *)data); 13393 break; 13394 case CHELSIO_T4_GET_SGE_CTXT: { 13395 struct t4_sge_ctxt *ctxt = (struct t4_sge_ctxt *)data; 13396 13397 rc = get_sge_context(sc, ctxt->mem_id, ctxt->cid, 13398 sizeof(ctxt->data), &ctxt->data[0]); 13399 break; 13400 } 13401 default: 13402 rc = ENOTTY; 13403 } 13404 13405 return (rc); 13406 } 13407 13408 #ifdef TCP_OFFLOAD 13409 int 13410 toe_capability(struct vi_info *vi, bool enable) 13411 { 13412 int rc; 13413 struct port_info *pi = vi->pi; 13414 struct adapter *sc = pi->adapter; 13415 13416 ASSERT_SYNCHRONIZED_OP(sc); 13417 13418 if (!is_offload(sc)) 13419 return (ENODEV); 13420 if (!hw_all_ok(sc)) 13421 return (ENXIO); 13422 13423 if (enable) { 13424 #ifdef KERN_TLS 13425 if (sc->flags & KERN_TLS_ON && is_t6(sc)) { 13426 int i, j, n; 13427 struct port_info *p; 13428 struct vi_info *v; 13429 13430 /* 13431 * Reconfigure hardware for TOE if TXTLS is not enabled 13432 * on any ifnet. 13433 */ 13434 n = 0; 13435 for_each_port(sc, i) { 13436 p = sc->port[i]; 13437 for_each_vi(p, j, v) { 13438 if (if_getcapenable(v->ifp) & IFCAP_TXTLS) { 13439 CH_WARN(sc, 13440 "%s has NIC TLS enabled.\n", 13441 device_get_nameunit(v->dev)); 13442 n++; 13443 } 13444 } 13445 } 13446 if (n > 0) { 13447 CH_WARN(sc, "Disable NIC TLS on all interfaces " 13448 "associated with this adapter before " 13449 "trying to enable TOE.\n"); 13450 return (EAGAIN); 13451 } 13452 rc = t6_config_kern_tls(sc, false); 13453 if (rc) 13454 return (rc); 13455 } 13456 #endif 13457 if ((if_getcapenable(vi->ifp) & IFCAP_TOE) != 0) { 13458 /* TOE is already enabled. */ 13459 return (0); 13460 } 13461 13462 /* 13463 * We need the port's queues around so that we're able to send 13464 * and receive CPLs to/from the TOE even if the ifnet for this 13465 * port has never been UP'd administratively. 13466 */ 13467 if (!(vi->flags & VI_INIT_DONE) && ((rc = vi_init(vi)) != 0)) 13468 return (rc); 13469 if (!(pi->vi[0].flags & VI_INIT_DONE) && 13470 ((rc = vi_init(&pi->vi[0])) != 0)) 13471 return (rc); 13472 13473 if (isset(&sc->offload_map, pi->port_id)) { 13474 /* TOE is enabled on another VI of this port. */ 13475 MPASS(pi->uld_vis > 0); 13476 pi->uld_vis++; 13477 return (0); 13478 } 13479 13480 if (!uld_active(sc, ULD_TOM)) { 13481 rc = t4_activate_uld(sc, ULD_TOM); 13482 if (rc == EAGAIN) { 13483 log(LOG_WARNING, 13484 "You must kldload t4_tom.ko before trying " 13485 "to enable TOE on a cxgbe interface.\n"); 13486 } 13487 if (rc != 0) 13488 return (rc); 13489 KASSERT(sc->tom_softc != NULL, 13490 ("%s: TOM activated but softc NULL", __func__)); 13491 KASSERT(uld_active(sc, ULD_TOM), 13492 ("%s: TOM activated but flag not set", __func__)); 13493 } 13494 13495 /* 13496 * Activate iWARP, iSCSI, and NVMe too, if the modules 13497 * are loaded. 13498 */ 13499 if (!uld_active(sc, ULD_IWARP)) 13500 (void) t4_activate_uld(sc, ULD_IWARP); 13501 if (!uld_active(sc, ULD_ISCSI)) 13502 (void) t4_activate_uld(sc, ULD_ISCSI); 13503 if (!uld_active(sc, ULD_NVME)) 13504 (void) t4_activate_uld(sc, ULD_NVME); 13505 13506 if (pi->uld_vis++ == 0) 13507 setbit(&sc->offload_map, pi->port_id); 13508 } else { 13509 if ((if_getcapenable(vi->ifp) & IFCAP_TOE) == 0) { 13510 /* TOE is already disabled. */ 13511 return (0); 13512 } 13513 MPASS(isset(&sc->offload_map, pi->port_id)); 13514 MPASS(pi->uld_vis > 0); 13515 if (--pi->uld_vis == 0) 13516 clrbit(&sc->offload_map, pi->port_id); 13517 } 13518 13519 return (0); 13520 } 13521 13522 /* 13523 * Add an upper layer driver to the global list. 13524 */ 13525 int 13526 t4_register_uld(struct uld_info *ui, int id) 13527 { 13528 int rc; 13529 13530 if (id < 0 || id > ULD_MAX) 13531 return (EINVAL); 13532 sx_xlock(&t4_uld_list_lock); 13533 if (t4_uld_list[id] != NULL) 13534 rc = EEXIST; 13535 else { 13536 t4_uld_list[id] = ui; 13537 rc = 0; 13538 } 13539 sx_xunlock(&t4_uld_list_lock); 13540 return (rc); 13541 } 13542 13543 int 13544 t4_unregister_uld(struct uld_info *ui, int id) 13545 { 13546 13547 if (id < 0 || id > ULD_MAX) 13548 return (EINVAL); 13549 sx_xlock(&t4_uld_list_lock); 13550 MPASS(t4_uld_list[id] == ui); 13551 t4_uld_list[id] = NULL; 13552 sx_xunlock(&t4_uld_list_lock); 13553 return (0); 13554 } 13555 13556 int 13557 t4_activate_uld(struct adapter *sc, int id) 13558 { 13559 int rc; 13560 13561 ASSERT_SYNCHRONIZED_OP(sc); 13562 13563 if (id < 0 || id > ULD_MAX) 13564 return (EINVAL); 13565 13566 /* Adapter needs to be initialized before any ULD can be activated. */ 13567 if (!(sc->flags & FULL_INIT_DONE)) { 13568 rc = adapter_init(sc); 13569 if (rc != 0) 13570 return (rc); 13571 } 13572 13573 sx_slock(&t4_uld_list_lock); 13574 if (t4_uld_list[id] == NULL) 13575 rc = EAGAIN; /* load the KLD with this ULD and try again. */ 13576 else { 13577 rc = t4_uld_list[id]->uld_activate(sc); 13578 if (rc == 0) 13579 setbit(&sc->active_ulds, id); 13580 } 13581 sx_sunlock(&t4_uld_list_lock); 13582 13583 return (rc); 13584 } 13585 13586 int 13587 t4_deactivate_uld(struct adapter *sc, int id) 13588 { 13589 int rc; 13590 13591 ASSERT_SYNCHRONIZED_OP(sc); 13592 13593 if (id < 0 || id > ULD_MAX) 13594 return (EINVAL); 13595 13596 sx_slock(&t4_uld_list_lock); 13597 if (t4_uld_list[id] == NULL) 13598 rc = ENXIO; 13599 else { 13600 rc = t4_uld_list[id]->uld_deactivate(sc); 13601 if (rc == 0) 13602 clrbit(&sc->active_ulds, id); 13603 } 13604 sx_sunlock(&t4_uld_list_lock); 13605 13606 return (rc); 13607 } 13608 13609 static int 13610 deactivate_all_uld(struct adapter *sc) 13611 { 13612 int i, rc; 13613 13614 rc = begin_synchronized_op(sc, NULL, SLEEP_OK, "t4detuld"); 13615 if (rc != 0) 13616 return (ENXIO); 13617 sx_slock(&t4_uld_list_lock); 13618 for (i = 0; i <= ULD_MAX; i++) { 13619 if (t4_uld_list[i] == NULL || !uld_active(sc, i)) 13620 continue; 13621 rc = t4_uld_list[i]->uld_deactivate(sc); 13622 if (rc != 0) 13623 break; 13624 clrbit(&sc->active_ulds, i); 13625 } 13626 sx_sunlock(&t4_uld_list_lock); 13627 end_synchronized_op(sc, 0); 13628 13629 return (rc); 13630 } 13631 13632 static void 13633 stop_all_uld(struct adapter *sc) 13634 { 13635 int i; 13636 13637 if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4uldst") != 0) 13638 return; 13639 sx_slock(&t4_uld_list_lock); 13640 for (i = 0; i <= ULD_MAX; i++) { 13641 if (t4_uld_list[i] == NULL || !uld_active(sc, i) || 13642 t4_uld_list[i]->uld_stop == NULL) 13643 continue; 13644 (void) t4_uld_list[i]->uld_stop(sc); 13645 } 13646 sx_sunlock(&t4_uld_list_lock); 13647 end_synchronized_op(sc, 0); 13648 } 13649 13650 static void 13651 restart_all_uld(struct adapter *sc) 13652 { 13653 int i; 13654 13655 if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4uldre") != 0) 13656 return; 13657 sx_slock(&t4_uld_list_lock); 13658 for (i = 0; i <= ULD_MAX; i++) { 13659 if (t4_uld_list[i] == NULL || !uld_active(sc, i) || 13660 t4_uld_list[i]->uld_restart == NULL) 13661 continue; 13662 (void) t4_uld_list[i]->uld_restart(sc); 13663 } 13664 sx_sunlock(&t4_uld_list_lock); 13665 end_synchronized_op(sc, 0); 13666 } 13667 13668 int 13669 uld_active(struct adapter *sc, int id) 13670 { 13671 13672 MPASS(id >= 0 && id <= ULD_MAX); 13673 13674 return (isset(&sc->active_ulds, id)); 13675 } 13676 #endif 13677 13678 #ifdef KERN_TLS 13679 static int 13680 ktls_capability(struct adapter *sc, bool enable) 13681 { 13682 ASSERT_SYNCHRONIZED_OP(sc); 13683 13684 if (!is_ktls(sc)) 13685 return (ENODEV); 13686 if (!is_t6(sc)) 13687 return (0); 13688 if (!hw_all_ok(sc)) 13689 return (ENXIO); 13690 13691 if (enable) { 13692 if (sc->flags & KERN_TLS_ON) 13693 return (0); /* already on */ 13694 if (sc->offload_map != 0) { 13695 CH_WARN(sc, 13696 "Disable TOE on all interfaces associated with " 13697 "this adapter before trying to enable NIC TLS.\n"); 13698 return (EAGAIN); 13699 } 13700 return (t6_config_kern_tls(sc, true)); 13701 } else { 13702 /* 13703 * Nothing to do for disable. If TOE is enabled sometime later 13704 * then toe_capability will reconfigure the hardware. 13705 */ 13706 return (0); 13707 } 13708 } 13709 #endif 13710 13711 /* 13712 * t = ptr to tunable. 13713 * nc = number of CPUs. 13714 * c = compiled in default for that tunable. 13715 */ 13716 static void 13717 calculate_nqueues(int *t, int nc, const int c) 13718 { 13719 int nq; 13720 13721 if (*t > 0) 13722 return; 13723 nq = *t < 0 ? -*t : c; 13724 *t = min(nc, nq); 13725 } 13726 13727 /* 13728 * Come up with reasonable defaults for some of the tunables, provided they're 13729 * not set by the user (in which case we'll use the values as is). 13730 */ 13731 static void 13732 tweak_tunables(void) 13733 { 13734 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 13735 13736 if (t4_ntxq < 1) { 13737 #ifdef RSS 13738 t4_ntxq = rss_getnumbuckets(); 13739 #else 13740 calculate_nqueues(&t4_ntxq, nc, NTXQ); 13741 #endif 13742 } 13743 13744 calculate_nqueues(&t4_ntxq_vi, nc, NTXQ_VI); 13745 13746 if (t4_nrxq < 1) { 13747 #ifdef RSS 13748 t4_nrxq = rss_getnumbuckets(); 13749 #else 13750 calculate_nqueues(&t4_nrxq, nc, NRXQ); 13751 #endif 13752 } 13753 13754 calculate_nqueues(&t4_nrxq_vi, nc, NRXQ_VI); 13755 13756 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 13757 calculate_nqueues(&t4_nofldtxq, nc, NOFLDTXQ); 13758 calculate_nqueues(&t4_nofldtxq_vi, nc, NOFLDTXQ_VI); 13759 #endif 13760 #ifdef TCP_OFFLOAD 13761 calculate_nqueues(&t4_nofldrxq, nc, NOFLDRXQ); 13762 calculate_nqueues(&t4_nofldrxq_vi, nc, NOFLDRXQ_VI); 13763 #endif 13764 13765 #if defined(TCP_OFFLOAD) || defined(KERN_TLS) 13766 if (t4_toecaps_allowed == -1) 13767 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 13768 #else 13769 if (t4_toecaps_allowed == -1) 13770 t4_toecaps_allowed = 0; 13771 #endif 13772 13773 #ifdef TCP_OFFLOAD 13774 if (t4_rdmacaps_allowed == -1) { 13775 t4_rdmacaps_allowed = FW_CAPS_CONFIG_RDMA_RDDP | 13776 FW_CAPS_CONFIG_RDMA_RDMAC; 13777 } 13778 13779 if (t4_iscsicaps_allowed == -1) { 13780 t4_iscsicaps_allowed = FW_CAPS_CONFIG_ISCSI_INITIATOR_PDU | 13781 FW_CAPS_CONFIG_ISCSI_TARGET_PDU | 13782 FW_CAPS_CONFIG_ISCSI_T10DIF; 13783 } 13784 13785 if (t4_nvmecaps_allowed == -1) 13786 t4_nvmecaps_allowed = FW_CAPS_CONFIG_NVME_TCP; 13787 13788 if (t4_tmr_idx_ofld < 0 || t4_tmr_idx_ofld >= SGE_NTIMERS) 13789 t4_tmr_idx_ofld = TMR_IDX_OFLD; 13790 13791 if (t4_pktc_idx_ofld < -1 || t4_pktc_idx_ofld >= SGE_NCOUNTERS) 13792 t4_pktc_idx_ofld = PKTC_IDX_OFLD; 13793 #else 13794 if (t4_rdmacaps_allowed == -1) 13795 t4_rdmacaps_allowed = 0; 13796 13797 if (t4_iscsicaps_allowed == -1) 13798 t4_iscsicaps_allowed = 0; 13799 13800 if (t4_nvmecaps_allowed == -1) 13801 t4_nvmecaps_allowed = 0; 13802 #endif 13803 13804 #ifdef DEV_NETMAP 13805 calculate_nqueues(&t4_nnmtxq, nc, NNMTXQ); 13806 calculate_nqueues(&t4_nnmrxq, nc, NNMRXQ); 13807 calculate_nqueues(&t4_nnmtxq_vi, nc, NNMTXQ_VI); 13808 calculate_nqueues(&t4_nnmrxq_vi, nc, NNMRXQ_VI); 13809 #endif 13810 13811 if (t4_tmr_idx < 0 || t4_tmr_idx >= SGE_NTIMERS) 13812 t4_tmr_idx = TMR_IDX; 13813 13814 if (t4_pktc_idx < -1 || t4_pktc_idx >= SGE_NCOUNTERS) 13815 t4_pktc_idx = PKTC_IDX; 13816 13817 if (t4_qsize_txq < 128) 13818 t4_qsize_txq = 128; 13819 13820 if (t4_qsize_rxq < 128) 13821 t4_qsize_rxq = 128; 13822 while (t4_qsize_rxq & 7) 13823 t4_qsize_rxq++; 13824 13825 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 13826 13827 /* 13828 * Number of VIs to create per-port. The first VI is the "main" regular 13829 * VI for the port. The rest are additional virtual interfaces on the 13830 * same physical port. Note that the main VI does not have native 13831 * netmap support but the extra VIs do. 13832 * 13833 * Limit the number of VIs per port to the number of available 13834 * MAC addresses per port. 13835 */ 13836 if (t4_num_vis < 1) 13837 t4_num_vis = 1; 13838 if (t4_num_vis > nitems(vi_mac_funcs)) { 13839 t4_num_vis = nitems(vi_mac_funcs); 13840 printf("cxgbe: number of VIs limited to %d\n", t4_num_vis); 13841 } 13842 13843 if (pcie_relaxed_ordering < 0 || pcie_relaxed_ordering > 2) { 13844 pcie_relaxed_ordering = 1; 13845 #if defined(__i386__) || defined(__amd64__) 13846 if (cpu_vendor_id == CPU_VENDOR_INTEL) 13847 pcie_relaxed_ordering = 0; 13848 #endif 13849 } 13850 } 13851 13852 #ifdef DDB 13853 static void 13854 t4_dump_mem(struct adapter *sc, u_int addr, u_int len) 13855 { 13856 uint32_t base, j, off, pf, reg, save, win_pos; 13857 13858 reg = chip_id(sc) > CHELSIO_T6 ? 13859 PCIE_MEM_ACCESS_T7_REG(A_PCIE_MEM_ACCESS_OFFSET0, 2) : 13860 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2); 13861 save = t4_read_reg(sc, reg); 13862 base = sc->memwin[2].mw_base; 13863 13864 if (is_t4(sc)) { 13865 pf = 0; 13866 win_pos = addr & ~0xf; /* start must be 16B aligned */ 13867 } else { 13868 pf = V_PFNUM(sc->pf); 13869 win_pos = addr & ~0x7f; /* start must be 128B aligned */ 13870 } 13871 off = addr - win_pos; 13872 if (chip_id(sc) > CHELSIO_T6) 13873 win_pos >>= X_T7_MEMOFST_SHIFT; 13874 t4_write_reg(sc, reg, win_pos | pf); 13875 t4_read_reg(sc, reg); 13876 13877 while (len > 0 && !db_pager_quit) { 13878 uint32_t buf[8]; 13879 for (j = 0; j < 8; j++, off += 4) 13880 buf[j] = htonl(t4_read_reg(sc, base + off)); 13881 13882 db_printf("%08x %08x %08x %08x %08x %08x %08x %08x\n", 13883 buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6], 13884 buf[7]); 13885 if (len <= sizeof(buf)) 13886 len = 0; 13887 else 13888 len -= sizeof(buf); 13889 } 13890 13891 t4_write_reg(sc, reg, save); 13892 t4_read_reg(sc, reg); 13893 } 13894 13895 static void 13896 t4_dump_tcb(struct adapter *sc, int tid) 13897 { 13898 uint32_t tcb_addr; 13899 13900 /* Dump TCB for the tid */ 13901 tcb_addr = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 13902 tcb_addr += tid * TCB_SIZE; 13903 t4_dump_mem(sc, tcb_addr, TCB_SIZE); 13904 } 13905 13906 static void 13907 t4_dump_devlog(struct adapter *sc) 13908 { 13909 struct devlog_params *dparams = &sc->params.devlog; 13910 struct fw_devlog_e e; 13911 int i, first, j, m, nentries, rc; 13912 uint64_t ftstamp = UINT64_MAX; 13913 13914 if (dparams->start == 0) { 13915 db_printf("devlog params not valid\n"); 13916 return; 13917 } 13918 13919 nentries = dparams->size / sizeof(struct fw_devlog_e); 13920 m = fwmtype_to_hwmtype(dparams->memtype); 13921 13922 /* Find the first entry. */ 13923 first = -1; 13924 for (i = 0; i < nentries && !db_pager_quit; i++) { 13925 rc = -t4_mem_read(sc, m, dparams->start + i * sizeof(e), 13926 sizeof(e), (void *)&e); 13927 if (rc != 0) 13928 break; 13929 13930 if (e.timestamp == 0) 13931 break; 13932 13933 e.timestamp = be64toh(e.timestamp); 13934 if (e.timestamp < ftstamp) { 13935 ftstamp = e.timestamp; 13936 first = i; 13937 } 13938 } 13939 13940 if (first == -1) 13941 return; 13942 13943 i = first; 13944 do { 13945 rc = -t4_mem_read(sc, m, dparams->start + i * sizeof(e), 13946 sizeof(e), (void *)&e); 13947 if (rc != 0) 13948 return; 13949 13950 if (e.timestamp == 0) 13951 return; 13952 13953 e.timestamp = be64toh(e.timestamp); 13954 e.seqno = be32toh(e.seqno); 13955 for (j = 0; j < 8; j++) 13956 e.params[j] = be32toh(e.params[j]); 13957 13958 db_printf("%10d %15ju %8s %8s ", 13959 e.seqno, e.timestamp, 13960 (e.level < nitems(devlog_level_strings) ? 13961 devlog_level_strings[e.level] : "UNKNOWN"), 13962 (e.facility < nitems(devlog_facility_strings) ? 13963 devlog_facility_strings[e.facility] : "UNKNOWN")); 13964 db_printf(e.fmt, e.params[0], e.params[1], e.params[2], 13965 e.params[3], e.params[4], e.params[5], e.params[6], 13966 e.params[7]); 13967 13968 if (++i == nentries) 13969 i = 0; 13970 } while (i != first && !db_pager_quit); 13971 } 13972 13973 static DB_DEFINE_TABLE(show, t4, show_t4); 13974 13975 DB_TABLE_COMMAND_FLAGS(show_t4, devlog, db_show_devlog, CS_OWN) 13976 { 13977 device_t dev; 13978 int t; 13979 bool valid; 13980 13981 valid = false; 13982 t = db_read_token(); 13983 if (t == tIDENT) { 13984 dev = device_lookup_by_name(db_tok_string); 13985 valid = true; 13986 } 13987 db_skip_to_eol(); 13988 if (!valid) { 13989 db_printf("usage: show t4 devlog <nexus>\n"); 13990 return; 13991 } 13992 13993 if (dev == NULL) { 13994 db_printf("device not found\n"); 13995 return; 13996 } 13997 13998 t4_dump_devlog(device_get_softc(dev)); 13999 } 14000 14001 DB_TABLE_COMMAND_FLAGS(show_t4, tcb, db_show_t4tcb, CS_OWN) 14002 { 14003 device_t dev; 14004 int radix, tid, t; 14005 bool valid; 14006 14007 valid = false; 14008 radix = db_radix; 14009 db_radix = 10; 14010 t = db_read_token(); 14011 if (t == tIDENT) { 14012 dev = device_lookup_by_name(db_tok_string); 14013 t = db_read_token(); 14014 if (t == tNUMBER) { 14015 tid = db_tok_number; 14016 valid = true; 14017 } 14018 } 14019 db_radix = radix; 14020 db_skip_to_eol(); 14021 if (!valid) { 14022 db_printf("usage: show t4 tcb <nexus> <tid>\n"); 14023 return; 14024 } 14025 14026 if (dev == NULL) { 14027 db_printf("device not found\n"); 14028 return; 14029 } 14030 if (tid < 0) { 14031 db_printf("invalid tid\n"); 14032 return; 14033 } 14034 14035 t4_dump_tcb(device_get_softc(dev), tid); 14036 } 14037 14038 DB_TABLE_COMMAND_FLAGS(show_t4, memdump, db_show_memdump, CS_OWN) 14039 { 14040 device_t dev; 14041 int radix, t; 14042 bool valid; 14043 14044 valid = false; 14045 radix = db_radix; 14046 db_radix = 10; 14047 t = db_read_token(); 14048 if (t == tIDENT) { 14049 dev = device_lookup_by_name(db_tok_string); 14050 t = db_read_token(); 14051 if (t == tNUMBER) { 14052 addr = db_tok_number; 14053 t = db_read_token(); 14054 if (t == tNUMBER) { 14055 count = db_tok_number; 14056 valid = true; 14057 } 14058 } 14059 } 14060 db_radix = radix; 14061 db_skip_to_eol(); 14062 if (!valid) { 14063 db_printf("usage: show t4 memdump <nexus> <addr> <len>\n"); 14064 return; 14065 } 14066 14067 if (dev == NULL) { 14068 db_printf("device not found\n"); 14069 return; 14070 } 14071 if (addr < 0) { 14072 db_printf("invalid address\n"); 14073 return; 14074 } 14075 if (count <= 0) { 14076 db_printf("invalid length\n"); 14077 return; 14078 } 14079 14080 t4_dump_mem(device_get_softc(dev), addr, count); 14081 } 14082 #endif 14083 14084 static eventhandler_tag vxlan_start_evtag; 14085 static eventhandler_tag vxlan_stop_evtag; 14086 14087 struct vxlan_evargs { 14088 if_t ifp; 14089 uint16_t port; 14090 }; 14091 14092 static void 14093 enable_vxlan_rx(struct adapter *sc) 14094 { 14095 int i, rc; 14096 struct port_info *pi; 14097 uint8_t match_all_mac[ETHER_ADDR_LEN] = {0}; 14098 14099 ASSERT_SYNCHRONIZED_OP(sc); 14100 14101 t4_write_reg(sc, A_MPS_RX_VXLAN_TYPE, V_VXLAN(sc->vxlan_port) | 14102 F_VXLAN_EN); 14103 for_each_port(sc, i) { 14104 pi = sc->port[i]; 14105 if (pi->vxlan_tcam_entry == true) 14106 continue; 14107 rc = t4_alloc_raw_mac_filt(sc, pi->vi[0].viid, match_all_mac, 14108 match_all_mac, sc->rawf_base + pi->port_id, 1, pi->port_id, 14109 true); 14110 if (rc < 0) { 14111 rc = -rc; 14112 CH_ERR(&pi->vi[0], 14113 "failed to add VXLAN TCAM entry: %d.\n", rc); 14114 } else { 14115 MPASS(rc == sc->rawf_base + pi->port_id); 14116 pi->vxlan_tcam_entry = true; 14117 } 14118 } 14119 } 14120 14121 static void 14122 t4_vxlan_start(struct adapter *sc, void *arg) 14123 { 14124 struct vxlan_evargs *v = arg; 14125 14126 if (sc->nrawf == 0 || chip_id(sc) <= CHELSIO_T5) 14127 return; 14128 if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4vxst") != 0) 14129 return; 14130 14131 if (sc->vxlan_refcount == 0) { 14132 sc->vxlan_port = v->port; 14133 sc->vxlan_refcount = 1; 14134 if (!hw_off_limits(sc)) 14135 enable_vxlan_rx(sc); 14136 } else if (sc->vxlan_port == v->port) { 14137 sc->vxlan_refcount++; 14138 } else { 14139 CH_ERR(sc, "VXLAN already configured on port %d; " 14140 "ignoring attempt to configure it on port %d\n", 14141 sc->vxlan_port, v->port); 14142 } 14143 end_synchronized_op(sc, 0); 14144 } 14145 14146 static void 14147 t4_vxlan_stop(struct adapter *sc, void *arg) 14148 { 14149 struct vxlan_evargs *v = arg; 14150 14151 if (sc->nrawf == 0 || chip_id(sc) <= CHELSIO_T5) 14152 return; 14153 if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4vxsp") != 0) 14154 return; 14155 14156 /* 14157 * VXLANs may have been configured before the driver was loaded so we 14158 * may see more stops than starts. This is not handled cleanly but at 14159 * least we keep the refcount sane. 14160 */ 14161 if (sc->vxlan_port != v->port) 14162 goto done; 14163 if (sc->vxlan_refcount == 0) { 14164 CH_ERR(sc, "VXLAN operation on port %d was stopped earlier; " 14165 "ignoring attempt to stop it again.\n", sc->vxlan_port); 14166 } else if (--sc->vxlan_refcount == 0 && !hw_off_limits(sc)) 14167 t4_set_reg_field(sc, A_MPS_RX_VXLAN_TYPE, F_VXLAN_EN, 0); 14168 done: 14169 end_synchronized_op(sc, 0); 14170 } 14171 14172 static void 14173 t4_vxlan_start_handler(void *arg __unused, if_t ifp, 14174 sa_family_t family, u_int port) 14175 { 14176 struct vxlan_evargs v; 14177 14178 MPASS(family == AF_INET || family == AF_INET6); 14179 v.ifp = ifp; 14180 v.port = port; 14181 14182 t4_iterate(t4_vxlan_start, &v); 14183 } 14184 14185 static void 14186 t4_vxlan_stop_handler(void *arg __unused, if_t ifp, sa_family_t family, 14187 u_int port) 14188 { 14189 struct vxlan_evargs v; 14190 14191 MPASS(family == AF_INET || family == AF_INET6); 14192 v.ifp = ifp; 14193 v.port = port; 14194 14195 t4_iterate(t4_vxlan_stop, &v); 14196 } 14197 14198 14199 static struct sx mlu; /* mod load unload */ 14200 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload"); 14201 14202 static int 14203 mod_event(module_t mod, int cmd, void *arg) 14204 { 14205 int rc = 0; 14206 static int loaded = 0; 14207 14208 switch (cmd) { 14209 case MOD_LOAD: 14210 sx_xlock(&mlu); 14211 if (loaded++ == 0) { 14212 t4_sge_modload(); 14213 t4_register_shared_cpl_handler(CPL_SET_TCB_RPL, 14214 t4_filter_rpl, CPL_COOKIE_FILTER); 14215 t4_register_shared_cpl_handler(CPL_L2T_WRITE_RPL, 14216 do_l2t_write_rpl, CPL_COOKIE_FILTER); 14217 t4_register_shared_cpl_handler(CPL_ACT_OPEN_RPL, 14218 t4_hashfilter_ao_rpl, CPL_COOKIE_HASHFILTER); 14219 t4_register_shared_cpl_handler(CPL_SET_TCB_RPL, 14220 t4_hashfilter_tcb_rpl, CPL_COOKIE_HASHFILTER); 14221 t4_register_shared_cpl_handler(CPL_ABORT_RPL_RSS, 14222 t4_del_hashfilter_rpl, CPL_COOKIE_HASHFILTER); 14223 t4_register_cpl_handler(CPL_TRACE_PKT, t4_trace_pkt); 14224 t4_register_cpl_handler(CPL_T5_TRACE_PKT, t5_trace_pkt); 14225 t4_register_cpl_handler(CPL_SMT_WRITE_RPL, 14226 do_smt_write_rpl); 14227 sx_init(&t4_list_lock, "T4/T5 adapters"); 14228 SLIST_INIT(&t4_list); 14229 callout_init(&fatal_callout, 1); 14230 #ifdef TCP_OFFLOAD 14231 sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); 14232 #endif 14233 #ifdef INET6 14234 t4_clip_modload(); 14235 #endif 14236 #ifdef KERN_TLS 14237 t6_ktls_modload(); 14238 t7_ktls_modload(); 14239 #endif 14240 t4_tracer_modload(); 14241 tweak_tunables(); 14242 vxlan_start_evtag = 14243 EVENTHANDLER_REGISTER(vxlan_start, 14244 t4_vxlan_start_handler, NULL, 14245 EVENTHANDLER_PRI_ANY); 14246 vxlan_stop_evtag = 14247 EVENTHANDLER_REGISTER(vxlan_stop, 14248 t4_vxlan_stop_handler, NULL, 14249 EVENTHANDLER_PRI_ANY); 14250 reset_tq = taskqueue_create("t4_rst_tq", M_WAITOK, 14251 taskqueue_thread_enqueue, &reset_tq); 14252 taskqueue_start_threads(&reset_tq, 1, PI_SOFT, 14253 "t4_rst_thr"); 14254 } 14255 sx_xunlock(&mlu); 14256 break; 14257 14258 case MOD_UNLOAD: 14259 sx_xlock(&mlu); 14260 if (--loaded == 0) { 14261 #ifdef TCP_OFFLOAD 14262 int i; 14263 #endif 14264 int tries; 14265 14266 taskqueue_free(reset_tq); 14267 14268 tries = 0; 14269 while (tries++ < 5 && t4_sge_extfree_refs() != 0) { 14270 uprintf("%ju clusters with custom free routine " 14271 "still is use.\n", t4_sge_extfree_refs()); 14272 pause("t4unload", 2 * hz); 14273 } 14274 14275 sx_slock(&t4_list_lock); 14276 if (!SLIST_EMPTY(&t4_list)) { 14277 rc = EBUSY; 14278 sx_sunlock(&t4_list_lock); 14279 goto done_unload; 14280 } 14281 #ifdef TCP_OFFLOAD 14282 sx_slock(&t4_uld_list_lock); 14283 for (i = 0; i <= ULD_MAX; i++) { 14284 if (t4_uld_list[i] != NULL) { 14285 rc = EBUSY; 14286 sx_sunlock(&t4_uld_list_lock); 14287 sx_sunlock(&t4_list_lock); 14288 goto done_unload; 14289 } 14290 } 14291 sx_sunlock(&t4_uld_list_lock); 14292 #endif 14293 sx_sunlock(&t4_list_lock); 14294 14295 if (t4_sge_extfree_refs() == 0) { 14296 EVENTHANDLER_DEREGISTER(vxlan_start, 14297 vxlan_start_evtag); 14298 EVENTHANDLER_DEREGISTER(vxlan_stop, 14299 vxlan_stop_evtag); 14300 t4_tracer_modunload(); 14301 #ifdef KERN_TLS 14302 t7_ktls_modunload(); 14303 t6_ktls_modunload(); 14304 #endif 14305 #ifdef INET6 14306 t4_clip_modunload(); 14307 #endif 14308 #ifdef TCP_OFFLOAD 14309 sx_destroy(&t4_uld_list_lock); 14310 #endif 14311 sx_destroy(&t4_list_lock); 14312 t4_sge_modunload(); 14313 loaded = 0; 14314 } else { 14315 rc = EBUSY; 14316 loaded++; /* undo earlier decrement */ 14317 } 14318 } 14319 done_unload: 14320 sx_xunlock(&mlu); 14321 break; 14322 } 14323 14324 return (rc); 14325 } 14326 14327 DRIVER_MODULE(t4nex, pci, t4_driver, mod_event, 0); 14328 MODULE_VERSION(t4nex, 1); 14329 MODULE_DEPEND(t4nex, firmware, 1, 1, 1); 14330 #ifdef DEV_NETMAP 14331 MODULE_DEPEND(t4nex, netmap, 1, 1, 1); 14332 #endif /* DEV_NETMAP */ 14333 14334 DRIVER_MODULE(t5nex, pci, t5_driver, mod_event, 0); 14335 MODULE_VERSION(t5nex, 1); 14336 MODULE_DEPEND(t5nex, firmware, 1, 1, 1); 14337 #ifdef DEV_NETMAP 14338 MODULE_DEPEND(t5nex, netmap, 1, 1, 1); 14339 #endif /* DEV_NETMAP */ 14340 14341 DRIVER_MODULE(t6nex, pci, t6_driver, mod_event, 0); 14342 MODULE_VERSION(t6nex, 1); 14343 MODULE_DEPEND(t6nex, crypto, 1, 1, 1); 14344 MODULE_DEPEND(t6nex, firmware, 1, 1, 1); 14345 #ifdef DEV_NETMAP 14346 MODULE_DEPEND(t6nex, netmap, 1, 1, 1); 14347 #endif /* DEV_NETMAP */ 14348 14349 DRIVER_MODULE(chnex, pci, ch_driver, mod_event, 0); 14350 MODULE_VERSION(chnex, 1); 14351 MODULE_DEPEND(chnex, crypto, 1, 1, 1); 14352 MODULE_DEPEND(chnex, firmware, 1, 1, 1); 14353 #ifdef DEV_NETMAP 14354 MODULE_DEPEND(chnex, netmap, 1, 1, 1); 14355 #endif /* DEV_NETMAP */ 14356 14357 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, 0, 0); 14358 MODULE_VERSION(cxgbe, 1); 14359 14360 DRIVER_MODULE(cxl, t5nex, cxl_driver, 0, 0); 14361 MODULE_VERSION(cxl, 1); 14362 14363 DRIVER_MODULE(cc, t6nex, cc_driver, 0, 0); 14364 MODULE_VERSION(cc, 1); 14365 14366 DRIVER_MODULE(che, chnex, che_driver, 0, 0); 14367 MODULE_VERSION(che, 1); 14368 14369 DRIVER_MODULE(vcxgbe, cxgbe, vcxgbe_driver, 0, 0); 14370 MODULE_VERSION(vcxgbe, 1); 14371 14372 DRIVER_MODULE(vcxl, cxl, vcxl_driver, 0, 0); 14373 MODULE_VERSION(vcxl, 1); 14374 14375 DRIVER_MODULE(vcc, cc, vcc_driver, 0, 0); 14376 MODULE_VERSION(vcc, 1); 14377 14378 DRIVER_MODULE(vche, che, vche_driver, 0, 0); 14379 MODULE_VERSION(vche, 1); 14380