1 /*- 2 * Copyright (c) 2016 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: John Baldwin <jhb@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 #include "opt_inet.h" 30 #include "opt_inet6.h" 31 32 #include <sys/param.h> 33 #include <sys/bus.h> 34 #include <sys/conf.h> 35 #include <sys/kernel.h> 36 #include <sys/module.h> 37 #include <sys/priv.h> 38 #include <dev/pci/pcivar.h> 39 #if defined(__i386__) || defined(__amd64__) 40 #include <vm/vm.h> 41 #include <vm/pmap.h> 42 #endif 43 44 #include "common/common.h" 45 #include "common/t4_regs.h" 46 #include "t4_ioctl.h" 47 #include "t4_mp_ring.h" 48 49 /* 50 * Some notes: 51 * 52 * The Virtual Interfaces are connected to an internal switch on the chip 53 * which allows VIs attached to the same port to talk to each other even when 54 * the port link is down. As a result, we might want to always report a 55 * VF's link as being "up". 56 * 57 * XXX: Add a TUNABLE and possible per-device sysctl for this? 58 */ 59 60 struct intrs_and_queues { 61 uint16_t intr_type; /* MSI, or MSI-X */ 62 uint16_t nirq; /* Total # of vectors */ 63 uint16_t ntxq; /* # of NIC txq's for each port */ 64 uint16_t nrxq; /* # of NIC rxq's for each port */ 65 }; 66 67 struct { 68 uint16_t device; 69 char *desc; 70 } t4vf_pciids[] = { 71 {0x4800, "Chelsio T440-dbg VF"}, 72 {0x4801, "Chelsio T420-CR VF"}, 73 {0x4802, "Chelsio T422-CR VF"}, 74 {0x4803, "Chelsio T440-CR VF"}, 75 {0x4804, "Chelsio T420-BCH VF"}, 76 {0x4805, "Chelsio T440-BCH VF"}, 77 {0x4806, "Chelsio T440-CH VF"}, 78 {0x4807, "Chelsio T420-SO VF"}, 79 {0x4808, "Chelsio T420-CX VF"}, 80 {0x4809, "Chelsio T420-BT VF"}, 81 {0x480a, "Chelsio T404-BT VF"}, 82 {0x480e, "Chelsio T440-LP-CR VF"}, 83 }, t5vf_pciids[] = { 84 {0x5800, "Chelsio T580-dbg VF"}, 85 {0x5801, "Chelsio T520-CR VF"}, /* 2 x 10G */ 86 {0x5802, "Chelsio T522-CR VF"}, /* 2 x 10G, 2 X 1G */ 87 {0x5803, "Chelsio T540-CR VF"}, /* 4 x 10G */ 88 {0x5807, "Chelsio T520-SO VF"}, /* 2 x 10G, nomem */ 89 {0x5809, "Chelsio T520-BT VF"}, /* 2 x 10GBaseT */ 90 {0x580a, "Chelsio T504-BT VF"}, /* 4 x 1G */ 91 {0x580d, "Chelsio T580-CR VF"}, /* 2 x 40G */ 92 {0x580e, "Chelsio T540-LP-CR VF"}, /* 4 x 10G */ 93 {0x5810, "Chelsio T580-LP-CR VF"}, /* 2 x 40G */ 94 {0x5811, "Chelsio T520-LL-CR VF"}, /* 2 x 10G */ 95 {0x5812, "Chelsio T560-CR VF"}, /* 1 x 40G, 2 x 10G */ 96 {0x5814, "Chelsio T580-LP-SO-CR VF"}, /* 2 x 40G, nomem */ 97 {0x5815, "Chelsio T502-BT VF"}, /* 2 x 1G */ 98 {0x5818, "Chelsio T540-BT VF"}, /* 4 x 10GBaseT */ 99 {0x5819, "Chelsio T540-LP-BT VF"}, /* 4 x 10GBaseT */ 100 {0x581a, "Chelsio T540-SO-BT VF"}, /* 4 x 10GBaseT, nomem */ 101 {0x581b, "Chelsio T540-SO-CR VF"}, /* 4 x 10G, nomem */ 102 }, t6vf_pciids[] = { 103 {0x6800, "Chelsio T6-DBG-25 VF"}, /* 2 x 10/25G, debug */ 104 {0x6801, "Chelsio T6225-CR VF"}, /* 2 x 10/25G */ 105 {0x6802, "Chelsio T6225-SO-CR VF"}, /* 2 x 10/25G, nomem */ 106 {0x6803, "Chelsio T6425-CR VF"}, /* 4 x 10/25G */ 107 {0x6804, "Chelsio T6425-SO-CR VF"}, /* 4 x 10/25G, nomem */ 108 {0x6805, "Chelsio T6225-OCP-SO VF"}, /* 2 x 10/25G, nomem */ 109 {0x6806, "Chelsio T62100-OCP-SO VF"}, /* 2 x 40/50/100G, nomem */ 110 {0x6807, "Chelsio T62100-LP-CR VF"}, /* 2 x 40/50/100G */ 111 {0x6808, "Chelsio T62100-SO-CR VF"}, /* 2 x 40/50/100G, nomem */ 112 {0x6809, "Chelsio T6210-BT VF"}, /* 2 x 10GBASE-T */ 113 {0x680d, "Chelsio T62100-CR VF"}, /* 2 x 40/50/100G */ 114 {0x6810, "Chelsio T6-DBG-100 VF"}, /* 2 x 40/50/100G, debug */ 115 {0x6811, "Chelsio T6225-LL-CR VF"}, /* 2 x 10/25G */ 116 {0x6814, "Chelsio T61100-OCP-SO VF"}, /* 1 x 40/50/100G, nomem */ 117 {0x6815, "Chelsio T6201-BT VF"}, /* 2 x 1000BASE-T */ 118 119 /* Custom */ 120 {0x6880, "Chelsio T6225 80 VF"}, 121 {0x6881, "Chelsio T62100 81 VF"}, 122 {0x6882, "Chelsio T6225-CR 82 VF"}, 123 {0x6883, "Chelsio T62100-CR 83 VF"}, 124 {0x6884, "Chelsio T64100-CR 84 VF"}, 125 {0x6885, "Chelsio T6240-SO 85 VF"}, 126 {0x6886, "Chelsio T6225-SO-CR 86 VF"}, 127 {0x6887, "Chelsio T6225-CR 87 VF"}, 128 }; 129 130 static d_ioctl_t t4vf_ioctl; 131 132 static struct cdevsw t4vf_cdevsw = { 133 .d_version = D_VERSION, 134 .d_ioctl = t4vf_ioctl, 135 .d_name = "t4vf", 136 }; 137 138 static int 139 t4vf_probe(device_t dev) 140 { 141 uint16_t d; 142 size_t i; 143 144 d = pci_get_device(dev); 145 for (i = 0; i < nitems(t4vf_pciids); i++) { 146 if (d == t4vf_pciids[i].device) { 147 device_set_desc(dev, t4vf_pciids[i].desc); 148 return (BUS_PROBE_DEFAULT); 149 } 150 } 151 return (ENXIO); 152 } 153 154 static int 155 t5vf_probe(device_t dev) 156 { 157 uint16_t d; 158 size_t i; 159 160 d = pci_get_device(dev); 161 for (i = 0; i < nitems(t5vf_pciids); i++) { 162 if (d == t5vf_pciids[i].device) { 163 device_set_desc(dev, t5vf_pciids[i].desc); 164 return (BUS_PROBE_DEFAULT); 165 } 166 } 167 return (ENXIO); 168 } 169 170 static int 171 t6vf_probe(device_t dev) 172 { 173 uint16_t d; 174 size_t i; 175 176 d = pci_get_device(dev); 177 for (i = 0; i < nitems(t6vf_pciids); i++) { 178 if (d == t6vf_pciids[i].device) { 179 device_set_desc(dev, t6vf_pciids[i].desc); 180 return (BUS_PROBE_DEFAULT); 181 } 182 } 183 return (ENXIO); 184 } 185 186 #define FW_PARAM_DEV(param) \ 187 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 188 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 189 #define FW_PARAM_PFVF(param) \ 190 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 191 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 192 193 static int 194 get_params__pre_init(struct adapter *sc) 195 { 196 int rc; 197 uint32_t param[3], val[3]; 198 199 param[0] = FW_PARAM_DEV(FWREV); 200 param[1] = FW_PARAM_DEV(TPREV); 201 param[2] = FW_PARAM_DEV(CCLK); 202 rc = -t4vf_query_params(sc, nitems(param), param, val); 203 if (rc != 0) { 204 device_printf(sc->dev, 205 "failed to query parameters (pre_init): %d.\n", rc); 206 return (rc); 207 } 208 209 sc->params.fw_vers = val[0]; 210 sc->params.tp_vers = val[1]; 211 sc->params.vpd.cclk = val[2]; 212 213 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 214 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 215 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 216 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 217 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 218 219 snprintf(sc->tp_version, sizeof(sc->tp_version), "%u.%u.%u.%u", 220 G_FW_HDR_FW_VER_MAJOR(sc->params.tp_vers), 221 G_FW_HDR_FW_VER_MINOR(sc->params.tp_vers), 222 G_FW_HDR_FW_VER_MICRO(sc->params.tp_vers), 223 G_FW_HDR_FW_VER_BUILD(sc->params.tp_vers)); 224 225 return (0); 226 } 227 228 static int 229 get_params__post_init(struct adapter *sc) 230 { 231 int rc; 232 uint32_t param, val; 233 234 rc = -t4vf_get_sge_params(sc); 235 if (rc != 0) { 236 device_printf(sc->dev, 237 "unable to retrieve adapter SGE parameters: %d\n", rc); 238 return (rc); 239 } 240 241 rc = -t4vf_get_rss_glb_config(sc); 242 if (rc != 0) { 243 device_printf(sc->dev, 244 "unable to retrieve adapter RSS parameters: %d\n", rc); 245 return (rc); 246 } 247 if (sc->params.rss.mode != FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL) { 248 device_printf(sc->dev, 249 "unable to operate with global RSS mode %d\n", 250 sc->params.rss.mode); 251 return (EINVAL); 252 } 253 254 /* 255 * Grab our Virtual Interface resource allocation, extract the 256 * features that we're interested in and do a bit of sanity testing on 257 * what we discover. 258 */ 259 rc = -t4vf_get_vfres(sc); 260 if (rc != 0) { 261 device_printf(sc->dev, 262 "unable to get virtual interface resources: %d\n", rc); 263 return (rc); 264 } 265 266 /* 267 * Check for various parameter sanity issues. 268 */ 269 if (sc->params.vfres.pmask == 0) { 270 device_printf(sc->dev, "no port access configured/usable!\n"); 271 return (EINVAL); 272 } 273 if (sc->params.vfres.nvi == 0) { 274 device_printf(sc->dev, 275 "no virtual interfaces configured/usable!\n"); 276 return (EINVAL); 277 } 278 sc->params.portvec = sc->params.vfres.pmask; 279 280 param = FW_PARAM_PFVF(MAX_PKTS_PER_ETH_TX_PKTS_WR); 281 rc = -t4vf_query_params(sc, 1, ¶m, &val); 282 if (rc == 0) 283 sc->params.max_pkts_per_eth_tx_pkts_wr = val; 284 else 285 sc->params.max_pkts_per_eth_tx_pkts_wr = 14; 286 287 rc = t4_verify_chip_settings(sc); 288 if (rc != 0) 289 return (rc); 290 t4_init_rx_buf_info(sc); 291 292 return (0); 293 } 294 295 static int 296 set_params__post_init(struct adapter *sc) 297 { 298 uint32_t param, val; 299 300 /* ask for encapsulated CPLs */ 301 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 302 val = 1; 303 (void)t4vf_set_params(sc, 1, ¶m, &val); 304 305 /* Enable 32b port caps if the firmware supports it. */ 306 param = FW_PARAM_PFVF(PORT_CAPS32); 307 val = 1; 308 if (t4vf_set_params(sc, 1, ¶m, &val) == 0) 309 sc->params.port_caps32 = 1; 310 311 return (0); 312 } 313 314 #undef FW_PARAM_PFVF 315 #undef FW_PARAM_DEV 316 317 static int 318 cfg_itype_and_nqueues(struct adapter *sc, struct intrs_and_queues *iaq) 319 { 320 struct vf_resources *vfres; 321 int nrxq, ntxq, nports; 322 int itype, iq_avail, navail, rc; 323 324 /* 325 * Figure out the layout of queues across our VIs and ensure 326 * we can allocate enough interrupts for our layout. 327 */ 328 vfres = &sc->params.vfres; 329 nports = sc->params.nports; 330 bzero(iaq, sizeof(*iaq)); 331 332 for (itype = INTR_MSIX; itype != 0; itype >>= 1) { 333 if (itype == INTR_INTX) 334 continue; 335 336 if (itype == INTR_MSIX) 337 navail = pci_msix_count(sc->dev); 338 else 339 navail = pci_msi_count(sc->dev); 340 341 if (navail == 0) 342 continue; 343 344 iaq->intr_type = itype; 345 346 /* 347 * XXX: The Linux driver reserves an Ingress Queue for 348 * forwarded interrupts when using MSI (but not MSI-X). 349 * It seems it just always asks for 2 interrupts and 350 * forwards all rxqs to the forwarded interrupt. 351 * 352 * We must reserve one IRQ for the for the firmware 353 * event queue. 354 * 355 * Every rxq requires an ingress queue with a free 356 * list and interrupts and an egress queue. Every txq 357 * requires an ETH egress queue. 358 */ 359 iaq->nirq = T4VF_EXTRA_INTR; 360 361 /* 362 * First, determine how many queues we can allocate. 363 * Start by finding the upper bound on rxqs from the 364 * limit on ingress queues. 365 */ 366 iq_avail = vfres->niqflint - iaq->nirq; 367 if (iq_avail < nports) { 368 device_printf(sc->dev, 369 "Not enough ingress queues (%d) for %d ports\n", 370 vfres->niqflint, nports); 371 return (ENXIO); 372 } 373 374 /* 375 * Try to honor the cap on interrupts. If there aren't 376 * enough interrupts for at least one interrupt per 377 * port, then don't bother, we will just forward all 378 * interrupts to one interrupt in that case. 379 */ 380 if (iaq->nirq + nports <= navail) { 381 if (iq_avail > navail - iaq->nirq) 382 iq_avail = navail - iaq->nirq; 383 } 384 385 nrxq = nports * t4_nrxq; 386 if (nrxq > iq_avail) { 387 /* 388 * Too many ingress queues. Use what we can. 389 */ 390 nrxq = (iq_avail / nports) * nports; 391 } 392 KASSERT(nrxq <= iq_avail, ("too many ingress queues")); 393 394 /* 395 * Next, determine the upper bound on txqs from the limit 396 * on ETH queues. 397 */ 398 if (vfres->nethctrl < nports) { 399 device_printf(sc->dev, 400 "Not enough ETH queues (%d) for %d ports\n", 401 vfres->nethctrl, nports); 402 return (ENXIO); 403 } 404 405 ntxq = nports * t4_ntxq; 406 if (ntxq > vfres->nethctrl) { 407 /* 408 * Too many ETH queues. Use what we can. 409 */ 410 ntxq = (vfres->nethctrl / nports) * nports; 411 } 412 KASSERT(ntxq <= vfres->nethctrl, ("too many ETH queues")); 413 414 /* 415 * Finally, ensure we have enough egress queues. 416 */ 417 if (vfres->neq < nports * 2) { 418 device_printf(sc->dev, 419 "Not enough egress queues (%d) for %d ports\n", 420 vfres->neq, nports); 421 return (ENXIO); 422 } 423 if (nrxq + ntxq > vfres->neq) { 424 /* Just punt and use 1 for everything. */ 425 nrxq = ntxq = nports; 426 } 427 KASSERT(nrxq <= iq_avail, ("too many ingress queues")); 428 KASSERT(ntxq <= vfres->nethctrl, ("too many ETH queues")); 429 KASSERT(nrxq + ntxq <= vfres->neq, ("too many egress queues")); 430 431 /* 432 * Do we have enough interrupts? For MSI the interrupts 433 * have to be a power of 2 as well. 434 */ 435 iaq->nirq += nrxq; 436 iaq->ntxq = ntxq; 437 iaq->nrxq = nrxq; 438 if (iaq->nirq <= navail && 439 (itype != INTR_MSI || powerof2(iaq->nirq))) { 440 navail = iaq->nirq; 441 if (itype == INTR_MSIX) 442 rc = pci_alloc_msix(sc->dev, &navail); 443 else 444 rc = pci_alloc_msi(sc->dev, &navail); 445 if (rc != 0) { 446 device_printf(sc->dev, 447 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 448 itype, rc, iaq->nirq, navail); 449 return (rc); 450 } 451 if (navail == iaq->nirq) { 452 return (0); 453 } 454 pci_release_msi(sc->dev); 455 } 456 457 /* Fall back to a single interrupt. */ 458 iaq->nirq = 1; 459 navail = iaq->nirq; 460 if (itype == INTR_MSIX) 461 rc = pci_alloc_msix(sc->dev, &navail); 462 else 463 rc = pci_alloc_msi(sc->dev, &navail); 464 if (rc != 0) 465 device_printf(sc->dev, 466 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 467 itype, rc, iaq->nirq, navail); 468 return (rc); 469 } 470 471 device_printf(sc->dev, 472 "failed to find a usable interrupt type. " 473 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 474 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 475 476 return (ENXIO); 477 } 478 479 static int 480 t4vf_attach(device_t dev) 481 { 482 struct adapter *sc; 483 int rc = 0, i, j, rqidx, tqidx, n, p, pmask; 484 struct make_dev_args mda; 485 struct intrs_and_queues iaq; 486 struct sge *s; 487 488 sc = device_get_softc(dev); 489 sc->dev = dev; 490 sysctl_ctx_init(&sc->ctx); 491 pci_enable_busmaster(dev); 492 pci_set_max_read_req(dev, 4096); 493 sc->params.pci.mps = pci_get_max_payload(dev); 494 495 sc->flags |= IS_VF; 496 TUNABLE_INT_FETCH("hw.cxgbe.dflags", &sc->debug_flags); 497 498 sc->sge_gts_reg = VF_SGE_REG(A_SGE_VF_GTS); 499 sc->sge_kdoorbell_reg = VF_SGE_REG(A_SGE_VF_KDOORBELL); 500 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 501 device_get_nameunit(dev)); 502 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 503 t4_add_adapter(sc); 504 505 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 506 TAILQ_INIT(&sc->sfl); 507 callout_init_mtx(&sc->sfl_callout, &sc->sfl_lock, 0); 508 509 mtx_init(&sc->reg_lock, "indirect register access", 0, MTX_DEF); 510 511 rc = t4_map_bars_0_and_4(sc); 512 if (rc != 0) 513 goto done; /* error message displayed already */ 514 515 rc = -t4vf_prep_adapter(sc); 516 if (rc != 0) 517 goto done; 518 519 t4_init_devnames(sc); 520 if (sc->names == NULL) { 521 rc = ENOTSUP; 522 goto done; /* error message displayed already */ 523 } 524 525 /* 526 * Leave the 'pf' and 'mbox' values as zero. This ensures 527 * that various firmware messages do not set the fields which 528 * is the correct thing to do for a VF. 529 */ 530 531 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 532 533 make_dev_args_init(&mda); 534 mda.mda_devsw = &t4vf_cdevsw; 535 mda.mda_uid = UID_ROOT; 536 mda.mda_gid = GID_WHEEL; 537 mda.mda_mode = 0600; 538 mda.mda_si_drv1 = sc; 539 rc = make_dev_s(&mda, &sc->cdev, "%s", device_get_nameunit(dev)); 540 if (rc != 0) 541 device_printf(dev, "failed to create nexus char device: %d.\n", 542 rc); 543 544 #if defined(__i386__) 545 if ((cpu_feature & CPUID_CX8) == 0) { 546 device_printf(dev, "64 bit atomics not available.\n"); 547 rc = ENOTSUP; 548 goto done; 549 } 550 #endif 551 552 /* 553 * Some environments do not properly handle PCIE FLRs -- e.g. in Linux 554 * 2.6.31 and later we can't call pci_reset_function() in order to 555 * issue an FLR because of a self- deadlock on the device semaphore. 556 * Meanwhile, the OS infrastructure doesn't issue FLRs in all the 557 * cases where they're needed -- for instance, some versions of KVM 558 * fail to reset "Assigned Devices" when the VM reboots. Therefore we 559 * use the firmware based reset in order to reset any per function 560 * state. 561 */ 562 rc = -t4vf_fw_reset(sc); 563 if (rc != 0) { 564 device_printf(dev, "FW reset failed: %d\n", rc); 565 goto done; 566 } 567 sc->flags |= FW_OK; 568 569 /* 570 * Grab basic operational parameters. These will predominantly have 571 * been set up by the Physical Function Driver or will be hard coded 572 * into the adapter. We just have to live with them ... Note that 573 * we _must_ get our VPD parameters before our SGE parameters because 574 * we need to know the adapter's core clock from the VPD in order to 575 * properly decode the SGE Timer Values. 576 */ 577 rc = get_params__pre_init(sc); 578 if (rc != 0) 579 goto done; /* error message displayed already */ 580 rc = get_params__post_init(sc); 581 if (rc != 0) 582 goto done; /* error message displayed already */ 583 584 rc = set_params__post_init(sc); 585 if (rc != 0) 586 goto done; /* error message displayed already */ 587 588 rc = t4_map_bar_2(sc); 589 if (rc != 0) 590 goto done; /* error message displayed already */ 591 592 rc = t4_adj_doorbells(sc); 593 if (rc != 0) 594 goto done; /* error message displayed already */ 595 596 rc = t4_create_dma_tag(sc); 597 if (rc != 0) 598 goto done; /* error message displayed already */ 599 600 /* 601 * The number of "ports" which we support is equal to the number of 602 * Virtual Interfaces with which we've been provisioned. 603 */ 604 sc->params.nports = imin(sc->params.vfres.nvi, MAX_NPORTS); 605 606 /* 607 * We may have been provisioned with more VIs than the number of 608 * ports we're allowed to access (our Port Access Rights Mask). 609 * Just use a single VI for each port. 610 */ 611 sc->params.nports = imin(sc->params.nports, 612 bitcount32(sc->params.vfres.pmask)); 613 614 #ifdef notyet 615 /* 616 * XXX: The Linux VF driver will lower nports if it thinks there 617 * are too few resources in vfres (niqflint, nethctrl, neq). 618 */ 619 #endif 620 621 /* 622 * First pass over all the ports - allocate VIs and initialize some 623 * basic parameters like mac address, port type, etc. 624 */ 625 pmask = sc->params.vfres.pmask; 626 for_each_port(sc, i) { 627 struct port_info *pi; 628 uint8_t mac[ETHER_ADDR_LEN]; 629 630 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 631 sc->port[i] = pi; 632 633 /* These must be set before t4_port_init */ 634 pi->adapter = sc; 635 pi->port_id = i; 636 pi->nvi = 1; 637 pi->vi = malloc(sizeof(struct vi_info) * pi->nvi, M_CXGBE, 638 M_ZERO | M_WAITOK); 639 640 /* 641 * Allocate the "main" VI and initialize parameters 642 * like mac addr. 643 */ 644 rc = -t4_port_init(sc, sc->mbox, sc->pf, 0, i); 645 if (rc != 0) { 646 device_printf(dev, "unable to initialize port %d: %d\n", 647 i, rc); 648 free(pi->vi, M_CXGBE); 649 free(pi, M_CXGBE); 650 sc->port[i] = NULL; 651 goto done; 652 } 653 654 /* Prefer the MAC address set by the PF, if there is one. */ 655 n = 1; 656 p = ffs(pmask) - 1; 657 MPASS(p >= 0); 658 rc = t4vf_get_vf_mac(sc, p, &n, mac); 659 if (rc == 0 && n == 1) 660 t4_os_set_hw_addr(pi, mac); 661 pmask &= ~(1 << p); 662 663 sc->vlan_id = t4vf_get_vf_vlan(sc); 664 665 /* No t4_link_start. */ 666 667 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 668 device_get_nameunit(dev), i); 669 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 670 sc->chan_map[pi->tx_chan] = i; 671 672 /* All VIs on this port share this media. */ 673 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 674 cxgbe_media_status); 675 676 pi->dev = device_add_child(dev, sc->names->vf_ifnet_name, DEVICE_UNIT_ANY); 677 if (pi->dev == NULL) { 678 device_printf(dev, 679 "failed to add device for port %d.\n", i); 680 rc = ENXIO; 681 goto done; 682 } 683 pi->vi[0].dev = pi->dev; 684 device_set_softc(pi->dev, pi); 685 } 686 687 /* 688 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 689 */ 690 rc = cfg_itype_and_nqueues(sc, &iaq); 691 if (rc != 0) 692 goto done; /* error message displayed already */ 693 694 sc->intr_type = iaq.intr_type; 695 sc->intr_count = iaq.nirq; 696 697 s = &sc->sge; 698 s->nrxq = sc->params.nports * iaq.nrxq; 699 s->ntxq = sc->params.nports * iaq.ntxq; 700 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 701 s->neq += sc->params.nports; /* ctrl queues: 1 per port */ 702 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 703 704 s->iqmap_sz = s->niq; 705 s->eqmap_sz = s->neq; 706 707 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 708 M_ZERO | M_WAITOK); 709 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 710 M_ZERO | M_WAITOK); 711 s->iqmap = malloc(s->iqmap_sz * sizeof(struct sge_iq *), M_CXGBE, 712 M_ZERO | M_WAITOK); 713 s->eqmap = malloc(s->eqmap_sz * sizeof(struct sge_eq *), M_CXGBE, 714 M_ZERO | M_WAITOK); 715 716 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 717 M_ZERO | M_WAITOK); 718 719 /* 720 * Second pass over the ports. This time we know the number of rx and 721 * tx queues that each port should get. 722 */ 723 rqidx = tqidx = 0; 724 for_each_port(sc, i) { 725 struct port_info *pi = sc->port[i]; 726 struct vi_info *vi; 727 728 if (pi == NULL) 729 continue; 730 731 for_each_vi(pi, j, vi) { 732 vi->pi = pi; 733 vi->adapter = sc; 734 vi->qsize_rxq = t4_qsize_rxq; 735 vi->qsize_txq = t4_qsize_txq; 736 737 vi->first_rxq = rqidx; 738 vi->first_txq = tqidx; 739 vi->tmr_idx = t4_tmr_idx; 740 vi->pktc_idx = t4_pktc_idx; 741 vi->nrxq = j == 0 ? iaq.nrxq: 1; 742 vi->ntxq = j == 0 ? iaq.ntxq: 1; 743 744 rqidx += vi->nrxq; 745 tqidx += vi->ntxq; 746 747 vi->rsrv_noflowq = 0; 748 } 749 } 750 751 rc = t4_setup_intr_handlers(sc); 752 if (rc != 0) { 753 device_printf(dev, 754 "failed to setup interrupt handlers: %d\n", rc); 755 goto done; 756 } 757 758 rc = bus_generic_attach(dev); 759 if (rc != 0) { 760 device_printf(dev, 761 "failed to attach all child ports: %d\n", rc); 762 goto done; 763 } 764 765 device_printf(dev, 766 "%d ports, %d %s interrupt%s, %d eq, %d iq\n", 767 sc->params.nports, sc->intr_count, sc->intr_type == INTR_MSIX ? 768 "MSI-X" : "MSI", sc->intr_count > 1 ? "s" : "", sc->sge.neq, 769 sc->sge.niq); 770 771 done: 772 if (rc != 0) 773 t4_detach_common(dev); 774 else 775 t4_sysctls(sc); 776 777 return (rc); 778 } 779 780 static void 781 get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 782 { 783 784 /* 0x3f is used as the revision for VFs. */ 785 regs->version = chip_id(sc) | (0x3f << 10); 786 t4_get_regs(sc, buf, regs->len); 787 } 788 789 static void 790 t4_clr_vi_stats(struct adapter *sc) 791 { 792 int reg; 793 794 for (reg = A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L; 795 reg <= A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H; reg += 4) 796 t4_write_reg(sc, VF_MPS_REG(reg), 0); 797 } 798 799 static int 800 t4vf_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 801 struct thread *td) 802 { 803 int rc; 804 struct adapter *sc = dev->si_drv1; 805 806 rc = priv_check(td, PRIV_DRIVER); 807 if (rc != 0) 808 return (rc); 809 810 switch (cmd) { 811 case CHELSIO_T4_GETREG: { 812 struct t4_reg *edata = (struct t4_reg *)data; 813 814 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 815 return (EFAULT); 816 817 if (edata->size == 4) 818 edata->val = t4_read_reg(sc, edata->addr); 819 else if (edata->size == 8) 820 edata->val = t4_read_reg64(sc, edata->addr); 821 else 822 return (EINVAL); 823 824 break; 825 } 826 case CHELSIO_T4_SETREG: { 827 struct t4_reg *edata = (struct t4_reg *)data; 828 829 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 830 return (EFAULT); 831 832 if (edata->size == 4) { 833 if (edata->val & 0xffffffff00000000) 834 return (EINVAL); 835 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 836 } else if (edata->size == 8) 837 t4_write_reg64(sc, edata->addr, edata->val); 838 else 839 return (EINVAL); 840 break; 841 } 842 case CHELSIO_T4_REGDUMP: { 843 struct t4_regdump *regs = (struct t4_regdump *)data; 844 int reglen = t4_get_regs_len(sc); 845 uint8_t *buf; 846 847 if (regs->len < reglen) { 848 regs->len = reglen; /* hint to the caller */ 849 return (ENOBUFS); 850 } 851 852 regs->len = reglen; 853 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 854 get_regs(sc, regs, buf); 855 rc = copyout(buf, regs->data, reglen); 856 free(buf, M_CXGBE); 857 break; 858 } 859 case CHELSIO_T4_CLEAR_STATS: { 860 int i, v; 861 u_int port_id = *(uint32_t *)data; 862 struct port_info *pi; 863 struct vi_info *vi; 864 865 if (port_id >= sc->params.nports) 866 return (EINVAL); 867 pi = sc->port[port_id]; 868 869 /* MAC stats */ 870 pi->tx_parse_error = 0; 871 t4_clr_vi_stats(sc); 872 873 /* 874 * Since this command accepts a port, clear stats for 875 * all VIs on this port. 876 */ 877 for_each_vi(pi, v, vi) { 878 if (vi->flags & VI_INIT_DONE) { 879 struct sge_rxq *rxq; 880 struct sge_txq *txq; 881 882 for_each_rxq(vi, i, rxq) { 883 #if defined(INET) || defined(INET6) 884 rxq->lro.lro_queued = 0; 885 rxq->lro.lro_flushed = 0; 886 #endif 887 rxq->rxcsum = 0; 888 rxq->vlan_extraction = 0; 889 } 890 891 for_each_txq(vi, i, txq) { 892 txq->txcsum = 0; 893 txq->tso_wrs = 0; 894 txq->vlan_insertion = 0; 895 txq->imm_wrs = 0; 896 txq->sgl_wrs = 0; 897 txq->txpkt_wrs = 0; 898 txq->txpkts0_wrs = 0; 899 txq->txpkts1_wrs = 0; 900 txq->txpkts0_pkts = 0; 901 txq->txpkts1_pkts = 0; 902 txq->txpkts_flush = 0; 903 mp_ring_reset_stats(txq->r); 904 } 905 } 906 } 907 break; 908 } 909 case CHELSIO_T4_SCHED_CLASS: 910 rc = t4_set_sched_class(sc, (struct t4_sched_params *)data); 911 break; 912 case CHELSIO_T4_SCHED_QUEUE: 913 rc = t4_set_sched_queue(sc, (struct t4_sched_queue *)data); 914 break; 915 default: 916 rc = ENOTTY; 917 } 918 919 return (rc); 920 } 921 922 static device_method_t t4vf_methods[] = { 923 DEVMETHOD(device_probe, t4vf_probe), 924 DEVMETHOD(device_attach, t4vf_attach), 925 DEVMETHOD(device_detach, t4_detach_common), 926 927 DEVMETHOD_END 928 }; 929 930 static driver_t t4vf_driver = { 931 "t4vf", 932 t4vf_methods, 933 sizeof(struct adapter) 934 }; 935 936 static device_method_t t5vf_methods[] = { 937 DEVMETHOD(device_probe, t5vf_probe), 938 DEVMETHOD(device_attach, t4vf_attach), 939 DEVMETHOD(device_detach, t4_detach_common), 940 941 DEVMETHOD_END 942 }; 943 944 static driver_t t5vf_driver = { 945 "t5vf", 946 t5vf_methods, 947 sizeof(struct adapter) 948 }; 949 950 static device_method_t t6vf_methods[] = { 951 DEVMETHOD(device_probe, t6vf_probe), 952 DEVMETHOD(device_attach, t4vf_attach), 953 DEVMETHOD(device_detach, t4_detach_common), 954 955 DEVMETHOD_END 956 }; 957 958 static driver_t t6vf_driver = { 959 "t6vf", 960 t6vf_methods, 961 sizeof(struct adapter) 962 }; 963 964 static driver_t cxgbev_driver = { 965 "cxgbev", 966 cxgbe_methods, 967 sizeof(struct port_info) 968 }; 969 970 static driver_t cxlv_driver = { 971 "cxlv", 972 cxgbe_methods, 973 sizeof(struct port_info) 974 }; 975 976 static driver_t ccv_driver = { 977 "ccv", 978 cxgbe_methods, 979 sizeof(struct port_info) 980 }; 981 982 DRIVER_MODULE(t4vf, pci, t4vf_driver, 0, 0); 983 MODULE_VERSION(t4vf, 1); 984 MODULE_DEPEND(t4vf, t4nex, 1, 1, 1); 985 986 DRIVER_MODULE(t5vf, pci, t5vf_driver, 0, 0); 987 MODULE_VERSION(t5vf, 1); 988 MODULE_DEPEND(t5vf, t5nex, 1, 1, 1); 989 990 DRIVER_MODULE(t6vf, pci, t6vf_driver, 0, 0); 991 MODULE_VERSION(t6vf, 1); 992 MODULE_DEPEND(t6vf, t6nex, 1, 1, 1); 993 994 DRIVER_MODULE(cxgbev, t4vf, cxgbev_driver, 0, 0); 995 MODULE_VERSION(cxgbev, 1); 996 997 DRIVER_MODULE(cxlv, t5vf, cxlv_driver, 0, 0); 998 MODULE_VERSION(cxlv, 1); 999 1000 DRIVER_MODULE(ccv, t6vf, ccv_driver, 0, 0); 1001 MODULE_VERSION(ccv, 1); 1002