1 /*- 2 * Copyright (c) 2016 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: John Baldwin <jhb@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 #include "opt_inet.h" 30 #include "opt_inet6.h" 31 32 #include <sys/param.h> 33 #include <sys/bus.h> 34 #include <sys/conf.h> 35 #include <sys/kernel.h> 36 #include <sys/module.h> 37 #include <sys/priv.h> 38 #include <dev/pci/pcivar.h> 39 #if defined(__i386__) || defined(__amd64__) 40 #include <vm/vm.h> 41 #include <vm/pmap.h> 42 #endif 43 44 #include "common/common.h" 45 #include "common/t4_regs.h" 46 #include "t4_ioctl.h" 47 #include "t4_mp_ring.h" 48 49 /* 50 * Some notes: 51 * 52 * The Virtual Interfaces are connected to an internal switch on the chip 53 * which allows VIs attached to the same port to talk to each other even when 54 * the port link is down. As a result, we might want to always report a 55 * VF's link as being "up". 56 * 57 * XXX: Add a TUNABLE and possible per-device sysctl for this? 58 */ 59 60 struct intrs_and_queues { 61 uint16_t intr_type; /* MSI, or MSI-X */ 62 uint16_t nirq; /* Total # of vectors */ 63 uint16_t ntxq; /* # of NIC txq's for each port */ 64 uint16_t nrxq; /* # of NIC rxq's for each port */ 65 }; 66 67 struct { 68 uint16_t device; 69 char *desc; 70 } t4vf_pciids[] = { 71 {0x4800, "Chelsio T440-dbg VF"}, 72 {0x4801, "Chelsio T420-CR VF"}, 73 {0x4802, "Chelsio T422-CR VF"}, 74 {0x4803, "Chelsio T440-CR VF"}, 75 {0x4804, "Chelsio T420-BCH VF"}, 76 {0x4805, "Chelsio T440-BCH VF"}, 77 {0x4806, "Chelsio T440-CH VF"}, 78 {0x4807, "Chelsio T420-SO VF"}, 79 {0x4808, "Chelsio T420-CX VF"}, 80 {0x4809, "Chelsio T420-BT VF"}, 81 {0x480a, "Chelsio T404-BT VF"}, 82 {0x480e, "Chelsio T440-LP-CR VF"}, 83 }, t5vf_pciids[] = { 84 {0x5800, "Chelsio T580-dbg VF"}, 85 {0x5801, "Chelsio T520-CR VF"}, /* 2 x 10G */ 86 {0x5802, "Chelsio T522-CR VF"}, /* 2 x 10G, 2 X 1G */ 87 {0x5803, "Chelsio T540-CR VF"}, /* 4 x 10G */ 88 {0x5807, "Chelsio T520-SO VF"}, /* 2 x 10G, nomem */ 89 {0x5809, "Chelsio T520-BT VF"}, /* 2 x 10GBaseT */ 90 {0x580a, "Chelsio T504-BT VF"}, /* 4 x 1G */ 91 {0x580d, "Chelsio T580-CR VF"}, /* 2 x 40G */ 92 {0x580e, "Chelsio T540-LP-CR VF"}, /* 4 x 10G */ 93 {0x5810, "Chelsio T580-LP-CR VF"}, /* 2 x 40G */ 94 {0x5811, "Chelsio T520-LL-CR VF"}, /* 2 x 10G */ 95 {0x5812, "Chelsio T560-CR VF"}, /* 1 x 40G, 2 x 10G */ 96 {0x5814, "Chelsio T580-LP-SO-CR VF"}, /* 2 x 40G, nomem */ 97 {0x5815, "Chelsio T502-BT VF"}, /* 2 x 1G */ 98 {0x5818, "Chelsio T540-BT VF"}, /* 4 x 10GBaseT */ 99 {0x5819, "Chelsio T540-LP-BT VF"}, /* 4 x 10GBaseT */ 100 {0x581a, "Chelsio T540-SO-BT VF"}, /* 4 x 10GBaseT, nomem */ 101 {0x581b, "Chelsio T540-SO-CR VF"}, /* 4 x 10G, nomem */ 102 }, t6vf_pciids[] = { 103 {0x6800, "Chelsio T6-DBG-25 VF"}, /* 2 x 10/25G, debug */ 104 {0x6801, "Chelsio T6225-CR VF"}, /* 2 x 10/25G */ 105 {0x6802, "Chelsio T6225-SO-CR VF"}, /* 2 x 10/25G, nomem */ 106 {0x6803, "Chelsio T6425-CR VF"}, /* 4 x 10/25G */ 107 {0x6804, "Chelsio T6425-SO-CR VF"}, /* 4 x 10/25G, nomem */ 108 {0x6805, "Chelsio T6225-SO-OCP3 VF"}, /* 2 x 10/25G, nomem */ 109 {0x6806, "Chelsio T6225-OCP3 VF"}, /* 2 x 10/25G */ 110 {0x6807, "Chelsio T62100-LP-CR VF"}, /* 2 x 40/50/100G */ 111 {0x6808, "Chelsio T62100-SO-CR VF"}, /* 2 x 40/50/100G, nomem */ 112 {0x6809, "Chelsio T6210-BT VF"}, /* 2 x 10GBASE-T */ 113 {0x680d, "Chelsio T62100-CR VF"}, /* 2 x 40/50/100G */ 114 {0x6810, "Chelsio T6-DBG-100 VF"}, /* 2 x 40/50/100G, debug */ 115 {0x6811, "Chelsio T6225-LL-CR VF"}, /* 2 x 10/25G */ 116 {0x6814, "Chelsio T62100-SO-OCP3 VF"}, /* 2 x 40/50/100G, nomem */ 117 {0x6815, "Chelsio T6201-BT VF"}, /* 2 x 1000BASE-T */ 118 119 /* Custom */ 120 {0x6880, "Chelsio T6225 80 VF"}, 121 {0x6881, "Chelsio T62100 81 VF"}, 122 {0x6882, "Chelsio T6225-CR 82 VF"}, 123 {0x6883, "Chelsio T62100-CR 83 VF"}, 124 {0x6884, "Chelsio T64100-CR 84 VF"}, 125 {0x6885, "Chelsio T6240-SO 85 VF"}, 126 {0x6886, "Chelsio T6225-SO-CR 86 VF"}, 127 {0x6887, "Chelsio T6225-CR 87 VF"}, 128 }, t7vf_pciids[] = { 129 {0xd800, "Chelsio T7 FPGA VF"}, /* T7 PE12K FPGA */ 130 {0x7800, "Chelsio T72200-DBG VF"}, /* 2 x 200G, debug */ 131 {0x7801, "Chelsio T7250 VF"}, /* 2 x 10/25/50G, 1 mem */ 132 {0x7802, "Chelsio S7250 VF"}, /* 2 x 10/25/50G, nomem */ 133 {0x7803, "Chelsio T7450 VF"}, /* 4 x 10/25/50G, 1 mem */ 134 {0x7804, "Chelsio S7450 VF"}, /* 4 x 10/25/50G, nomem */ 135 {0x7805, "Chelsio T72200 VF"}, /* 2 x 40/100/200G, 1 mem */ 136 {0x7806, "Chelsio S72200 VF"}, /* 2 x 40/100/200G, nomem */ 137 {0x7807, "Chelsio T72200-FH VF"}, /* 2 x 40/100/200G, 2 mem */ 138 {0x7808, "Chelsio T71400 VF"}, /* 1 x 400G, nomem */ 139 {0x7809, "Chelsio S7210-BT VF"}, /* 2 x 10GBASE-T, nomem */ 140 {0x780a, "Chelsio T7450-RC VF"}, /* 4 x 10/25/50G, 1 mem, RC */ 141 {0x780b, "Chelsio T72200-RC VF"}, /* 2 x 40/100/200G, 1 mem, RC */ 142 {0x780c, "Chelsio T72200-FH-RC VF"}, /* 2 x 40/100/200G, 2 mem, RC */ 143 {0x780d, "Chelsio S72200-OCP3 VF"}, /* 2 x 40/100/200G OCP3 */ 144 {0x780e, "Chelsio S7450-OCP3 VF"}, /* 4 x 1/20/25/50G OCP3 */ 145 {0x780f, "Chelsio S7410-BT-OCP3 VF"}, /* 4 x 10GBASE-T OCP3 */ 146 {0x7810, "Chelsio S7210-BT-A VF"}, /* 2 x 10GBASE-T */ 147 {0x7811, "Chelsio T7_MAYRA_7 VF"}, /* Motherboard */ 148 149 {0x7880, "Custom T7 VF"}, 150 }; 151 152 static d_ioctl_t t4vf_ioctl; 153 154 static struct cdevsw t4vf_cdevsw = { 155 .d_version = D_VERSION, 156 .d_ioctl = t4vf_ioctl, 157 .d_name = "t4vf", 158 }; 159 160 static int 161 t4vf_probe(device_t dev) 162 { 163 uint16_t d, v; 164 size_t i; 165 166 v = pci_get_vendor(dev); 167 if (v != PCI_VENDOR_ID_CHELSIO) 168 return (ENXIO); 169 170 d = pci_get_device(dev); 171 for (i = 0; i < nitems(t4vf_pciids); i++) { 172 if (d == t4vf_pciids[i].device) { 173 device_set_desc(dev, t4vf_pciids[i].desc); 174 return (BUS_PROBE_DEFAULT); 175 } 176 } 177 return (ENXIO); 178 } 179 180 static int 181 t5vf_probe(device_t dev) 182 { 183 uint16_t d, v; 184 size_t i; 185 186 v = pci_get_vendor(dev); 187 if (v != PCI_VENDOR_ID_CHELSIO) 188 return (ENXIO); 189 190 d = pci_get_device(dev); 191 for (i = 0; i < nitems(t5vf_pciids); i++) { 192 if (d == t5vf_pciids[i].device) { 193 device_set_desc(dev, t5vf_pciids[i].desc); 194 return (BUS_PROBE_DEFAULT); 195 } 196 } 197 return (ENXIO); 198 } 199 200 static int 201 t6vf_probe(device_t dev) 202 { 203 uint16_t d, v; 204 size_t i; 205 206 v = pci_get_vendor(dev); 207 if (v != PCI_VENDOR_ID_CHELSIO) 208 return (ENXIO); 209 210 d = pci_get_device(dev); 211 for (i = 0; i < nitems(t6vf_pciids); i++) { 212 if (d == t6vf_pciids[i].device) { 213 device_set_desc(dev, t6vf_pciids[i].desc); 214 return (BUS_PROBE_DEFAULT); 215 } 216 } 217 return (ENXIO); 218 } 219 220 static int 221 chvf_probe(device_t dev) 222 { 223 uint16_t d, v; 224 size_t i; 225 226 v = pci_get_vendor(dev); 227 if (v != PCI_VENDOR_ID_CHELSIO) 228 return (ENXIO); 229 230 d = pci_get_device(dev); 231 for (i = 0; i < nitems(t7vf_pciids); i++) { 232 if (d == t7vf_pciids[i].device) { 233 device_set_desc(dev, t7vf_pciids[i].desc); 234 return (BUS_PROBE_DEFAULT); 235 } 236 } 237 return (ENXIO); 238 } 239 240 #define FW_PARAM_DEV(param) \ 241 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 242 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 243 #define FW_PARAM_PFVF(param) \ 244 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 245 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 246 247 static int 248 get_params__pre_init(struct adapter *sc) 249 { 250 int rc; 251 uint32_t param[3], val[3]; 252 253 param[0] = FW_PARAM_DEV(FWREV); 254 param[1] = FW_PARAM_DEV(TPREV); 255 param[2] = FW_PARAM_DEV(CCLK); 256 rc = -t4vf_query_params(sc, nitems(param), param, val); 257 if (rc != 0) { 258 device_printf(sc->dev, 259 "failed to query parameters (pre_init): %d.\n", rc); 260 return (rc); 261 } 262 263 sc->params.fw_vers = val[0]; 264 sc->params.tp_vers = val[1]; 265 sc->params.vpd.cclk = val[2]; 266 267 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 268 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 269 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 270 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 271 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 272 273 snprintf(sc->tp_version, sizeof(sc->tp_version), "%u.%u.%u.%u", 274 G_FW_HDR_FW_VER_MAJOR(sc->params.tp_vers), 275 G_FW_HDR_FW_VER_MINOR(sc->params.tp_vers), 276 G_FW_HDR_FW_VER_MICRO(sc->params.tp_vers), 277 G_FW_HDR_FW_VER_BUILD(sc->params.tp_vers)); 278 279 return (0); 280 } 281 282 static int 283 get_params__post_init(struct adapter *sc) 284 { 285 int rc; 286 uint32_t param, val; 287 288 rc = -t4vf_get_sge_params(sc); 289 if (rc != 0) { 290 device_printf(sc->dev, 291 "unable to retrieve adapter SGE parameters: %d\n", rc); 292 return (rc); 293 } 294 295 rc = -t4vf_get_rss_glb_config(sc); 296 if (rc != 0) { 297 device_printf(sc->dev, 298 "unable to retrieve adapter RSS parameters: %d\n", rc); 299 return (rc); 300 } 301 if (sc->params.rss.mode != FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL) { 302 device_printf(sc->dev, 303 "unable to operate with global RSS mode %d\n", 304 sc->params.rss.mode); 305 return (EINVAL); 306 } 307 308 /* 309 * Grab our Virtual Interface resource allocation, extract the 310 * features that we're interested in and do a bit of sanity testing on 311 * what we discover. 312 */ 313 rc = -t4vf_get_vfres(sc); 314 if (rc != 0) { 315 device_printf(sc->dev, 316 "unable to get virtual interface resources: %d\n", rc); 317 return (rc); 318 } 319 320 /* 321 * Check for various parameter sanity issues. 322 */ 323 if (sc->params.vfres.pmask == 0) { 324 device_printf(sc->dev, "no port access configured/usable!\n"); 325 return (EINVAL); 326 } 327 if (sc->params.vfres.nvi == 0) { 328 device_printf(sc->dev, 329 "no virtual interfaces configured/usable!\n"); 330 return (EINVAL); 331 } 332 sc->params.portvec = sc->params.vfres.pmask; 333 334 param = FW_PARAM_PFVF(MAX_PKTS_PER_ETH_TX_PKTS_WR); 335 rc = -t4vf_query_params(sc, 1, ¶m, &val); 336 if (rc == 0) 337 sc->params.max_pkts_per_eth_tx_pkts_wr = val; 338 else 339 sc->params.max_pkts_per_eth_tx_pkts_wr = 14; 340 341 rc = t4_verify_chip_settings(sc); 342 if (rc != 0) 343 return (rc); 344 t4_init_rx_buf_info(sc); 345 346 return (0); 347 } 348 349 static int 350 set_params__post_init(struct adapter *sc) 351 { 352 uint32_t param, val; 353 354 /* ask for encapsulated CPLs */ 355 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 356 val = 1; 357 (void)t4vf_set_params(sc, 1, ¶m, &val); 358 359 /* Enable 32b port caps if the firmware supports it. */ 360 param = FW_PARAM_PFVF(PORT_CAPS32); 361 val = 1; 362 if (t4vf_set_params(sc, 1, ¶m, &val) == 0) 363 sc->params.port_caps32 = 1; 364 365 return (0); 366 } 367 368 #undef FW_PARAM_PFVF 369 #undef FW_PARAM_DEV 370 371 static int 372 cfg_itype_and_nqueues(struct adapter *sc, struct intrs_and_queues *iaq) 373 { 374 struct vf_resources *vfres; 375 int nrxq, ntxq, nports; 376 int itype, iq_avail, navail, rc; 377 378 /* 379 * Figure out the layout of queues across our VIs and ensure 380 * we can allocate enough interrupts for our layout. 381 */ 382 vfres = &sc->params.vfres; 383 nports = sc->params.nports; 384 bzero(iaq, sizeof(*iaq)); 385 386 for (itype = INTR_MSIX; itype != 0; itype >>= 1) { 387 if (itype == INTR_INTX) 388 continue; 389 390 if (itype == INTR_MSIX) 391 navail = pci_msix_count(sc->dev); 392 else 393 navail = pci_msi_count(sc->dev); 394 395 if (navail == 0) 396 continue; 397 398 iaq->intr_type = itype; 399 400 /* 401 * XXX: The Linux driver reserves an Ingress Queue for 402 * forwarded interrupts when using MSI (but not MSI-X). 403 * It seems it just always asks for 2 interrupts and 404 * forwards all rxqs to the forwarded interrupt. 405 * 406 * We must reserve one IRQ for the for the firmware 407 * event queue. 408 * 409 * Every rxq requires an ingress queue with a free 410 * list and interrupts and an egress queue. Every txq 411 * requires an ETH egress queue. 412 */ 413 iaq->nirq = T4VF_EXTRA_INTR; 414 415 /* 416 * First, determine how many queues we can allocate. 417 * Start by finding the upper bound on rxqs from the 418 * limit on ingress queues. 419 */ 420 iq_avail = vfres->niqflint - iaq->nirq; 421 if (iq_avail < nports) { 422 device_printf(sc->dev, 423 "Not enough ingress queues (%d) for %d ports\n", 424 vfres->niqflint, nports); 425 return (ENXIO); 426 } 427 428 /* 429 * Try to honor the cap on interrupts. If there aren't 430 * enough interrupts for at least one interrupt per 431 * port, then don't bother, we will just forward all 432 * interrupts to one interrupt in that case. 433 */ 434 if (iaq->nirq + nports <= navail) { 435 if (iq_avail > navail - iaq->nirq) 436 iq_avail = navail - iaq->nirq; 437 } 438 439 nrxq = nports * t4_nrxq; 440 if (nrxq > iq_avail) { 441 /* 442 * Too many ingress queues. Use what we can. 443 */ 444 nrxq = (iq_avail / nports) * nports; 445 } 446 KASSERT(nrxq <= iq_avail, ("too many ingress queues")); 447 448 /* 449 * Next, determine the upper bound on txqs from the limit 450 * on ETH queues. 451 */ 452 if (vfres->nethctrl < nports) { 453 device_printf(sc->dev, 454 "Not enough ETH queues (%d) for %d ports\n", 455 vfres->nethctrl, nports); 456 return (ENXIO); 457 } 458 459 ntxq = nports * t4_ntxq; 460 if (ntxq > vfres->nethctrl) { 461 /* 462 * Too many ETH queues. Use what we can. 463 */ 464 ntxq = (vfres->nethctrl / nports) * nports; 465 } 466 KASSERT(ntxq <= vfres->nethctrl, ("too many ETH queues")); 467 468 /* 469 * Finally, ensure we have enough egress queues. 470 */ 471 if (vfres->neq < nports * 2) { 472 device_printf(sc->dev, 473 "Not enough egress queues (%d) for %d ports\n", 474 vfres->neq, nports); 475 return (ENXIO); 476 } 477 if (nrxq + ntxq > vfres->neq) { 478 /* Just punt and use 1 for everything. */ 479 nrxq = ntxq = nports; 480 } 481 KASSERT(nrxq <= iq_avail, ("too many ingress queues")); 482 KASSERT(ntxq <= vfres->nethctrl, ("too many ETH queues")); 483 KASSERT(nrxq + ntxq <= vfres->neq, ("too many egress queues")); 484 485 /* 486 * Do we have enough interrupts? For MSI the interrupts 487 * have to be a power of 2 as well. 488 */ 489 iaq->nirq += nrxq; 490 iaq->ntxq = ntxq; 491 iaq->nrxq = nrxq; 492 if (iaq->nirq <= navail && 493 (itype != INTR_MSI || powerof2(iaq->nirq))) { 494 navail = iaq->nirq; 495 if (itype == INTR_MSIX) 496 rc = pci_alloc_msix(sc->dev, &navail); 497 else 498 rc = pci_alloc_msi(sc->dev, &navail); 499 if (rc != 0) { 500 device_printf(sc->dev, 501 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 502 itype, rc, iaq->nirq, navail); 503 return (rc); 504 } 505 if (navail == iaq->nirq) { 506 return (0); 507 } 508 pci_release_msi(sc->dev); 509 } 510 511 /* Fall back to a single interrupt. */ 512 iaq->nirq = 1; 513 navail = iaq->nirq; 514 if (itype == INTR_MSIX) 515 rc = pci_alloc_msix(sc->dev, &navail); 516 else 517 rc = pci_alloc_msi(sc->dev, &navail); 518 if (rc != 0) 519 device_printf(sc->dev, 520 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 521 itype, rc, iaq->nirq, navail); 522 return (rc); 523 } 524 525 device_printf(sc->dev, 526 "failed to find a usable interrupt type. " 527 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 528 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 529 530 return (ENXIO); 531 } 532 533 static int 534 t4vf_attach(device_t dev) 535 { 536 struct adapter *sc; 537 int rc = 0, i, j, rqidx, tqidx, n, p, pmask; 538 struct make_dev_args mda; 539 struct intrs_and_queues iaq; 540 struct sge *s; 541 542 sc = device_get_softc(dev); 543 sc->dev = dev; 544 sysctl_ctx_init(&sc->ctx); 545 pci_enable_busmaster(dev); 546 pci_set_max_read_req(dev, 4096); 547 sc->params.pci.mps = pci_get_max_payload(dev); 548 549 sc->flags |= IS_VF; 550 TUNABLE_INT_FETCH("hw.cxgbe.dflags", &sc->debug_flags); 551 552 sc->sge_gts_reg = VF_SGE_REG(A_SGE_VF_GTS); 553 sc->sge_kdoorbell_reg = VF_SGE_REG(A_SGE_VF_KDOORBELL); 554 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 555 device_get_nameunit(dev)); 556 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 557 t4_add_adapter(sc); 558 559 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 560 TAILQ_INIT(&sc->sfl); 561 callout_init_mtx(&sc->sfl_callout, &sc->sfl_lock, 0); 562 563 mtx_init(&sc->reg_lock, "indirect register access", 0, MTX_DEF); 564 565 rc = t4_map_bars_0_and_4(sc); 566 if (rc != 0) 567 goto done; /* error message displayed already */ 568 569 rc = -t4vf_prep_adapter(sc); 570 if (rc != 0) 571 goto done; 572 573 t4_init_devnames(sc); 574 if (sc->names == NULL) { 575 rc = ENOTSUP; 576 goto done; /* error message displayed already */ 577 } 578 579 /* 580 * Leave the 'pf' and 'mbox' values as zero. This ensures 581 * that various firmware messages do not set the fields which 582 * is the correct thing to do for a VF. 583 */ 584 585 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 586 587 make_dev_args_init(&mda); 588 mda.mda_devsw = &t4vf_cdevsw; 589 mda.mda_uid = UID_ROOT; 590 mda.mda_gid = GID_WHEEL; 591 mda.mda_mode = 0600; 592 mda.mda_si_drv1 = sc; 593 rc = make_dev_s(&mda, &sc->cdev, "%s", device_get_nameunit(dev)); 594 if (rc != 0) 595 device_printf(dev, "failed to create nexus char device: %d.\n", 596 rc); 597 598 #if defined(__i386__) 599 if ((cpu_feature & CPUID_CX8) == 0) { 600 device_printf(dev, "64 bit atomics not available.\n"); 601 rc = ENOTSUP; 602 goto done; 603 } 604 #endif 605 606 /* 607 * Some environments do not properly handle PCIE FLRs -- e.g. in Linux 608 * 2.6.31 and later we can't call pci_reset_function() in order to 609 * issue an FLR because of a self- deadlock on the device semaphore. 610 * Meanwhile, the OS infrastructure doesn't issue FLRs in all the 611 * cases where they're needed -- for instance, some versions of KVM 612 * fail to reset "Assigned Devices" when the VM reboots. Therefore we 613 * use the firmware based reset in order to reset any per function 614 * state. 615 */ 616 rc = -t4vf_fw_reset(sc); 617 if (rc != 0) { 618 device_printf(dev, "FW reset failed: %d\n", rc); 619 goto done; 620 } 621 sc->flags |= FW_OK; 622 623 /* 624 * Grab basic operational parameters. These will predominantly have 625 * been set up by the Physical Function Driver or will be hard coded 626 * into the adapter. We just have to live with them ... Note that 627 * we _must_ get our VPD parameters before our SGE parameters because 628 * we need to know the adapter's core clock from the VPD in order to 629 * properly decode the SGE Timer Values. 630 */ 631 rc = get_params__pre_init(sc); 632 if (rc != 0) 633 goto done; /* error message displayed already */ 634 rc = get_params__post_init(sc); 635 if (rc != 0) 636 goto done; /* error message displayed already */ 637 638 rc = set_params__post_init(sc); 639 if (rc != 0) 640 goto done; /* error message displayed already */ 641 642 rc = t4_map_bar_2(sc); 643 if (rc != 0) 644 goto done; /* error message displayed already */ 645 646 rc = t4_adj_doorbells(sc); 647 if (rc != 0) 648 goto done; /* error message displayed already */ 649 650 rc = t4_create_dma_tag(sc); 651 if (rc != 0) 652 goto done; /* error message displayed already */ 653 654 /* 655 * The number of "ports" which we support is equal to the number of 656 * Virtual Interfaces with which we've been provisioned. 657 */ 658 sc->params.nports = imin(sc->params.vfres.nvi, MAX_NPORTS); 659 660 /* 661 * We may have been provisioned with more VIs than the number of 662 * ports we're allowed to access (our Port Access Rights Mask). 663 * Just use a single VI for each port. 664 */ 665 sc->params.nports = imin(sc->params.nports, 666 bitcount32(sc->params.vfres.pmask)); 667 668 #ifdef notyet 669 /* 670 * XXX: The Linux VF driver will lower nports if it thinks there 671 * are too few resources in vfres (niqflint, nethctrl, neq). 672 */ 673 #endif 674 675 /* 676 * First pass over all the ports - allocate VIs and initialize some 677 * basic parameters like mac address, port type, etc. 678 */ 679 pmask = sc->params.vfres.pmask; 680 for_each_port(sc, i) { 681 struct port_info *pi; 682 uint8_t mac[ETHER_ADDR_LEN]; 683 684 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 685 sc->port[i] = pi; 686 687 /* These must be set before t4_port_init */ 688 pi->adapter = sc; 689 pi->port_id = i; 690 pi->nvi = 1; 691 pi->vi = malloc(sizeof(struct vi_info) * pi->nvi, M_CXGBE, 692 M_ZERO | M_WAITOK); 693 694 /* 695 * Allocate the "main" VI and initialize parameters 696 * like mac addr. 697 */ 698 rc = -t4_port_init(sc, sc->mbox, sc->pf, 0, i); 699 if (rc != 0) { 700 device_printf(dev, "unable to initialize port %d: %d\n", 701 i, rc); 702 free(pi->vi, M_CXGBE); 703 free(pi, M_CXGBE); 704 sc->port[i] = NULL; 705 goto done; 706 } 707 708 /* Prefer the MAC address set by the PF, if there is one. */ 709 n = 1; 710 p = ffs(pmask) - 1; 711 MPASS(p >= 0); 712 rc = t4vf_get_vf_mac(sc, p, &n, mac); 713 if (rc == 0 && n == 1) 714 t4_os_set_hw_addr(pi, mac); 715 pmask &= ~(1 << p); 716 717 sc->vlan_id = t4vf_get_vf_vlan(sc); 718 719 /* No t4_link_start. */ 720 721 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 722 device_get_nameunit(dev), i); 723 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 724 sc->chan_map[pi->tx_chan] = i; 725 726 /* All VIs on this port share this media. */ 727 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 728 cxgbe_media_status); 729 730 pi->dev = device_add_child(dev, sc->names->vf_ifnet_name, DEVICE_UNIT_ANY); 731 if (pi->dev == NULL) { 732 device_printf(dev, 733 "failed to add device for port %d.\n", i); 734 rc = ENXIO; 735 goto done; 736 } 737 pi->vi[0].dev = pi->dev; 738 device_set_softc(pi->dev, pi); 739 } 740 741 /* 742 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 743 */ 744 rc = cfg_itype_and_nqueues(sc, &iaq); 745 if (rc != 0) 746 goto done; /* error message displayed already */ 747 748 sc->intr_type = iaq.intr_type; 749 sc->intr_count = iaq.nirq; 750 751 s = &sc->sge; 752 s->nrxq = sc->params.nports * iaq.nrxq; 753 s->ntxq = sc->params.nports * iaq.ntxq; 754 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 755 s->neq += sc->params.nports; /* ctrl queues: 1 per port */ 756 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 757 758 s->iqmap_sz = s->niq; 759 s->eqmap_sz = s->neq; 760 761 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 762 M_ZERO | M_WAITOK); 763 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 764 M_ZERO | M_WAITOK); 765 s->iqmap = malloc(s->iqmap_sz * sizeof(struct sge_iq *), M_CXGBE, 766 M_ZERO | M_WAITOK); 767 s->eqmap = malloc(s->eqmap_sz * sizeof(struct sge_eq *), M_CXGBE, 768 M_ZERO | M_WAITOK); 769 770 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 771 M_ZERO | M_WAITOK); 772 773 /* 774 * Second pass over the ports. This time we know the number of rx and 775 * tx queues that each port should get. 776 */ 777 rqidx = tqidx = 0; 778 for_each_port(sc, i) { 779 struct port_info *pi = sc->port[i]; 780 struct vi_info *vi; 781 782 if (pi == NULL) 783 continue; 784 785 for_each_vi(pi, j, vi) { 786 vi->pi = pi; 787 vi->adapter = sc; 788 vi->qsize_rxq = t4_qsize_rxq; 789 vi->qsize_txq = t4_qsize_txq; 790 791 vi->first_rxq = rqidx; 792 vi->first_txq = tqidx; 793 vi->tmr_idx = t4_tmr_idx; 794 vi->pktc_idx = t4_pktc_idx; 795 vi->nrxq = j == 0 ? iaq.nrxq: 1; 796 vi->ntxq = j == 0 ? iaq.ntxq: 1; 797 798 rqidx += vi->nrxq; 799 tqidx += vi->ntxq; 800 801 vi->rsrv_noflowq = 0; 802 } 803 } 804 805 rc = t4_setup_intr_handlers(sc); 806 if (rc != 0) { 807 device_printf(dev, 808 "failed to setup interrupt handlers: %d\n", rc); 809 goto done; 810 } 811 812 bus_attach_children(dev); 813 814 device_printf(dev, 815 "%d ports, %d %s interrupt%s, %d eq, %d iq\n", 816 sc->params.nports, sc->intr_count, sc->intr_type == INTR_MSIX ? 817 "MSI-X" : "MSI", sc->intr_count > 1 ? "s" : "", sc->sge.neq, 818 sc->sge.niq); 819 820 done: 821 if (rc != 0) 822 t4_detach_common(dev); 823 else 824 t4_sysctls(sc); 825 826 return (rc); 827 } 828 829 static void 830 get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 831 { 832 833 /* 0x3f is used as the revision for VFs. */ 834 regs->version = chip_id(sc) | (0x3f << 10); 835 t4_get_regs(sc, buf, regs->len); 836 } 837 838 static void 839 t4_clr_vi_stats(struct adapter *sc) 840 { 841 int reg; 842 843 for (reg = A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L; 844 reg <= A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H; reg += 4) 845 t4_write_reg(sc, VF_MPS_REG(reg), 0); 846 } 847 848 static int 849 t4vf_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 850 struct thread *td) 851 { 852 int rc; 853 struct adapter *sc = dev->si_drv1; 854 855 rc = priv_check(td, PRIV_DRIVER); 856 if (rc != 0) 857 return (rc); 858 859 switch (cmd) { 860 case CHELSIO_T4_GETREG: { 861 struct t4_reg *edata = (struct t4_reg *)data; 862 863 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 864 return (EFAULT); 865 866 if (edata->size == 4) 867 edata->val = t4_read_reg(sc, edata->addr); 868 else if (edata->size == 8) 869 edata->val = t4_read_reg64(sc, edata->addr); 870 else 871 return (EINVAL); 872 873 break; 874 } 875 case CHELSIO_T4_SETREG: { 876 struct t4_reg *edata = (struct t4_reg *)data; 877 878 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 879 return (EFAULT); 880 881 if (edata->size == 4) { 882 if (edata->val & 0xffffffff00000000) 883 return (EINVAL); 884 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 885 } else if (edata->size == 8) 886 t4_write_reg64(sc, edata->addr, edata->val); 887 else 888 return (EINVAL); 889 break; 890 } 891 case CHELSIO_T4_REGDUMP: { 892 struct t4_regdump *regs = (struct t4_regdump *)data; 893 int reglen = t4_get_regs_len(sc); 894 uint8_t *buf; 895 896 if (regs->len < reglen) { 897 regs->len = reglen; /* hint to the caller */ 898 return (ENOBUFS); 899 } 900 901 regs->len = reglen; 902 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 903 get_regs(sc, regs, buf); 904 rc = copyout(buf, regs->data, reglen); 905 free(buf, M_CXGBE); 906 break; 907 } 908 case CHELSIO_T4_CLEAR_STATS: { 909 int i, v; 910 u_int port_id = *(uint32_t *)data; 911 struct port_info *pi; 912 struct vi_info *vi; 913 914 if (port_id >= sc->params.nports) 915 return (EINVAL); 916 pi = sc->port[port_id]; 917 918 /* MAC stats */ 919 pi->tx_parse_error = 0; 920 t4_clr_vi_stats(sc); 921 922 /* 923 * Since this command accepts a port, clear stats for 924 * all VIs on this port. 925 */ 926 for_each_vi(pi, v, vi) { 927 if (vi->flags & VI_INIT_DONE) { 928 struct sge_rxq *rxq; 929 struct sge_txq *txq; 930 931 for_each_rxq(vi, i, rxq) { 932 #if defined(INET) || defined(INET6) 933 rxq->lro.lro_queued = 0; 934 rxq->lro.lro_flushed = 0; 935 #endif 936 rxq->rxcsum = 0; 937 rxq->vlan_extraction = 0; 938 } 939 940 for_each_txq(vi, i, txq) { 941 txq->txcsum = 0; 942 txq->tso_wrs = 0; 943 txq->vlan_insertion = 0; 944 txq->imm_wrs = 0; 945 txq->sgl_wrs = 0; 946 txq->txpkt_wrs = 0; 947 txq->txpkts0_wrs = 0; 948 txq->txpkts1_wrs = 0; 949 txq->txpkts0_pkts = 0; 950 txq->txpkts1_pkts = 0; 951 txq->txpkts_flush = 0; 952 mp_ring_reset_stats(txq->r); 953 } 954 } 955 } 956 break; 957 } 958 case CHELSIO_T4_SCHED_CLASS: 959 rc = t4_set_sched_class(sc, (struct t4_sched_params *)data); 960 break; 961 case CHELSIO_T4_SCHED_QUEUE: 962 rc = t4_set_sched_queue(sc, (struct t4_sched_queue *)data); 963 break; 964 default: 965 rc = ENOTTY; 966 } 967 968 return (rc); 969 } 970 971 static device_method_t t4vf_methods[] = { 972 DEVMETHOD(device_probe, t4vf_probe), 973 DEVMETHOD(device_attach, t4vf_attach), 974 DEVMETHOD(device_detach, t4_detach_common), 975 976 DEVMETHOD_END 977 }; 978 979 static driver_t t4vf_driver = { 980 "t4vf", 981 t4vf_methods, 982 sizeof(struct adapter) 983 }; 984 985 static device_method_t t5vf_methods[] = { 986 DEVMETHOD(device_probe, t5vf_probe), 987 DEVMETHOD(device_attach, t4vf_attach), 988 DEVMETHOD(device_detach, t4_detach_common), 989 990 DEVMETHOD_END 991 }; 992 993 static driver_t t5vf_driver = { 994 "t5vf", 995 t5vf_methods, 996 sizeof(struct adapter) 997 }; 998 999 static device_method_t t6vf_methods[] = { 1000 DEVMETHOD(device_probe, t6vf_probe), 1001 DEVMETHOD(device_attach, t4vf_attach), 1002 DEVMETHOD(device_detach, t4_detach_common), 1003 1004 DEVMETHOD_END 1005 }; 1006 1007 static driver_t t6vf_driver = { 1008 "t6vf", 1009 t6vf_methods, 1010 sizeof(struct adapter) 1011 }; 1012 1013 static device_method_t chvf_methods[] = { 1014 DEVMETHOD(device_probe, chvf_probe), 1015 DEVMETHOD(device_attach, t4vf_attach), 1016 DEVMETHOD(device_detach, t4_detach_common), 1017 1018 DEVMETHOD_END 1019 }; 1020 1021 static driver_t chvf_driver = { 1022 "chvf", 1023 chvf_methods, 1024 sizeof(struct adapter) 1025 }; 1026 1027 static driver_t cxgbev_driver = { 1028 "cxgbev", 1029 cxgbe_methods, 1030 sizeof(struct port_info) 1031 }; 1032 1033 static driver_t cxlv_driver = { 1034 "cxlv", 1035 cxgbe_methods, 1036 sizeof(struct port_info) 1037 }; 1038 1039 static driver_t ccv_driver = { 1040 "ccv", 1041 cxgbe_methods, 1042 sizeof(struct port_info) 1043 }; 1044 1045 static driver_t chev_driver = { 1046 "chev", 1047 cxgbe_methods, 1048 sizeof(struct port_info) 1049 }; 1050 1051 DRIVER_MODULE(t4vf, pci, t4vf_driver, 0, 0); 1052 MODULE_VERSION(t4vf, 1); 1053 MODULE_DEPEND(t4vf, t4nex, 1, 1, 1); 1054 1055 DRIVER_MODULE(t5vf, pci, t5vf_driver, 0, 0); 1056 MODULE_VERSION(t5vf, 1); 1057 MODULE_DEPEND(t5vf, t5nex, 1, 1, 1); 1058 1059 DRIVER_MODULE(t6vf, pci, t6vf_driver, 0, 0); 1060 MODULE_VERSION(t6vf, 1); 1061 MODULE_DEPEND(t6vf, t6nex, 1, 1, 1); 1062 1063 DRIVER_MODULE(chvf, pci, chvf_driver, 0, 0); 1064 MODULE_VERSION(chvf, 1); 1065 MODULE_DEPEND(chvf, chnex, 1, 1, 1); 1066 1067 DRIVER_MODULE(cxgbev, t4vf, cxgbev_driver, 0, 0); 1068 MODULE_VERSION(cxgbev, 1); 1069 1070 DRIVER_MODULE(cxlv, t5vf, cxlv_driver, 0, 0); 1071 MODULE_VERSION(cxlv, 1); 1072 1073 DRIVER_MODULE(ccv, t6vf, ccv_driver, 0, 0); 1074 MODULE_VERSION(ccv, 1); 1075 1076 DRIVER_MODULE(chev, chvf, chev_driver, 0, 0); 1077 MODULE_VERSION(chev, 1); 1078