1 /* 2 * Copyright (C) 2015 Cavium Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 * 28 */ 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include "opt_inet.h" 33 #include "opt_inet6.h" 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/bitset.h> 38 #include <sys/bitstring.h> 39 #include <sys/bus.h> 40 #include <sys/endian.h> 41 #include <sys/kernel.h> 42 #include <sys/malloc.h> 43 #include <sys/mbuf.h> 44 #include <sys/module.h> 45 #include <sys/rman.h> 46 #include <sys/pciio.h> 47 #include <sys/pcpu.h> 48 #include <sys/proc.h> 49 #include <sys/socket.h> 50 #include <sys/sockio.h> 51 #include <sys/stdatomic.h> 52 #include <sys/cpuset.h> 53 #include <sys/lock.h> 54 #include <sys/mutex.h> 55 #include <sys/smp.h> 56 #include <sys/taskqueue.h> 57 58 #include <net/bpf.h> 59 #include <net/ethernet.h> 60 #include <net/if.h> 61 #include <net/if_var.h> 62 #include <net/if_arp.h> 63 #include <net/if_dl.h> 64 #include <net/if_media.h> 65 #include <net/if_types.h> 66 #include <net/if_vlan_var.h> 67 68 #include <netinet/in.h> 69 #include <netinet/ip.h> 70 #include <netinet/if_ether.h> 71 #include <netinet/tcp_lro.h> 72 73 #include <dev/pci/pcireg.h> 74 #include <dev/pci/pcivar.h> 75 76 #include <sys/dnv.h> 77 #include <sys/nv.h> 78 #include <sys/iov_schema.h> 79 80 #include <machine/bus.h> 81 82 #include "thunder_bgx.h" 83 #include "nic_reg.h" 84 #include "nic.h" 85 #include "nicvf_queues.h" 86 87 #define VNIC_VF_DEVSTR "Cavium Thunder NIC Virtual Function Driver" 88 89 #define VNIC_VF_REG_RID PCIR_BAR(PCI_CFG_REG_BAR_NUM) 90 91 /* Lock for core interface settings */ 92 #define NICVF_CORE_LOCK_INIT(nic) \ 93 sx_init(&(nic)->core_sx, device_get_nameunit((nic)->dev)) 94 95 #define NICVF_CORE_LOCK_DESTROY(nic) \ 96 sx_destroy(&(nic)->core_sx) 97 98 #define NICVF_CORE_LOCK(nic) sx_xlock(&(nic)->core_sx) 99 #define NICVF_CORE_UNLOCK(nic) sx_xunlock(&(nic)->core_sx) 100 101 #define NICVF_CORE_LOCK_ASSERT(nic) sx_assert(&(nic)->core_sx, SA_XLOCKED) 102 103 #define SPEED_10 10 104 #define SPEED_100 100 105 #define SPEED_1000 1000 106 #define SPEED_10000 10000 107 #define SPEED_40000 40000 108 109 MALLOC_DEFINE(M_NICVF, "nicvf", "ThunderX VNIC VF dynamic memory"); 110 111 static int nicvf_probe(device_t); 112 static int nicvf_attach(device_t); 113 static int nicvf_detach(device_t); 114 115 static device_method_t nicvf_methods[] = { 116 /* Device interface */ 117 DEVMETHOD(device_probe, nicvf_probe), 118 DEVMETHOD(device_attach, nicvf_attach), 119 DEVMETHOD(device_detach, nicvf_detach), 120 121 DEVMETHOD_END, 122 }; 123 124 static driver_t nicvf_driver = { 125 "vnic", 126 nicvf_methods, 127 sizeof(struct nicvf), 128 }; 129 130 static devclass_t nicvf_devclass; 131 132 DRIVER_MODULE(nicvf, pci, nicvf_driver, nicvf_devclass, 0, 0); 133 MODULE_DEPEND(nicvf, pci, 1, 1, 1); 134 MODULE_DEPEND(nicvf, ether, 1, 1, 1); 135 MODULE_DEPEND(nicvf, vnic_pf, 1, 1, 1); 136 137 static int nicvf_allocate_misc_interrupt(struct nicvf *); 138 static int nicvf_enable_misc_interrupt(struct nicvf *); 139 static int nicvf_allocate_net_interrupts(struct nicvf *); 140 static void nicvf_release_all_interrupts(struct nicvf *); 141 static int nicvf_update_hw_max_frs(struct nicvf *, int); 142 static int nicvf_hw_set_mac_addr(struct nicvf *, uint8_t *); 143 static void nicvf_config_cpi(struct nicvf *); 144 static int nicvf_rss_init(struct nicvf *); 145 static int nicvf_init_resources(struct nicvf *); 146 147 static int nicvf_setup_ifnet(struct nicvf *); 148 static int nicvf_setup_ifmedia(struct nicvf *); 149 static void nicvf_hw_addr_random(uint8_t *); 150 151 static int nicvf_if_ioctl(struct ifnet *, u_long, caddr_t); 152 static void nicvf_if_init(void *); 153 static void nicvf_if_init_locked(struct nicvf *); 154 static int nicvf_if_transmit(struct ifnet *, struct mbuf *); 155 static void nicvf_if_qflush(struct ifnet *); 156 static uint64_t nicvf_if_getcounter(struct ifnet *, ift_counter); 157 158 static int nicvf_stop_locked(struct nicvf *); 159 160 static void nicvf_media_status(struct ifnet *, struct ifmediareq *); 161 static int nicvf_media_change(struct ifnet *); 162 163 static void nicvf_tick_stats(void *); 164 165 static int 166 nicvf_probe(device_t dev) 167 { 168 uint16_t vendor_id; 169 uint16_t device_id; 170 171 vendor_id = pci_get_vendor(dev); 172 device_id = pci_get_device(dev); 173 174 if (vendor_id != PCI_VENDOR_ID_CAVIUM) 175 return (ENXIO); 176 177 if (device_id == PCI_DEVICE_ID_THUNDER_NIC_VF || 178 device_id == PCI_DEVICE_ID_THUNDER_PASS1_NIC_VF) { 179 device_set_desc(dev, VNIC_VF_DEVSTR); 180 return (BUS_PROBE_DEFAULT); 181 } 182 183 return (ENXIO); 184 } 185 186 static int 187 nicvf_attach(device_t dev) 188 { 189 struct nicvf *nic; 190 int rid, qcount; 191 int err = 0; 192 uint8_t hwaddr[ETHER_ADDR_LEN]; 193 uint8_t zeromac[] = {[0 ... (ETHER_ADDR_LEN - 1)] = 0}; 194 195 nic = device_get_softc(dev); 196 nic->dev = dev; 197 nic->pnicvf = nic; 198 199 NICVF_CORE_LOCK_INIT(nic); 200 /* Enable HW TSO on Pass2 */ 201 if (!pass1_silicon(dev)) 202 nic->hw_tso = TRUE; 203 204 rid = VNIC_VF_REG_RID; 205 nic->reg_base = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 206 RF_ACTIVE); 207 if (nic->reg_base == NULL) { 208 device_printf(dev, "Could not allocate registers memory\n"); 209 return (ENXIO); 210 } 211 212 qcount = MAX_CMP_QUEUES_PER_QS; 213 nic->max_queues = qcount; 214 215 err = nicvf_set_qset_resources(nic); 216 if (err != 0) 217 goto err_free_res; 218 219 /* Check if PF is alive and get MAC address for this VF */ 220 err = nicvf_allocate_misc_interrupt(nic); 221 if (err != 0) 222 goto err_free_res; 223 224 NICVF_CORE_LOCK(nic); 225 err = nicvf_enable_misc_interrupt(nic); 226 NICVF_CORE_UNLOCK(nic); 227 if (err != 0) 228 goto err_release_intr; 229 230 err = nicvf_allocate_net_interrupts(nic); 231 if (err != 0) { 232 device_printf(dev, 233 "Could not allocate network interface interrupts\n"); 234 goto err_free_ifnet; 235 } 236 237 /* If no MAC address was obtained we generate random one */ 238 if (memcmp(nic->hwaddr, zeromac, ETHER_ADDR_LEN) == 0) { 239 nicvf_hw_addr_random(hwaddr); 240 memcpy(nic->hwaddr, hwaddr, ETHER_ADDR_LEN); 241 NICVF_CORE_LOCK(nic); 242 nicvf_hw_set_mac_addr(nic, hwaddr); 243 NICVF_CORE_UNLOCK(nic); 244 } 245 246 /* Configure CPI alorithm */ 247 nic->cpi_alg = CPI_ALG_NONE; 248 NICVF_CORE_LOCK(nic); 249 nicvf_config_cpi(nic); 250 /* Configure receive side scaling */ 251 if (nic->qs->rq_cnt > 1) 252 nicvf_rss_init(nic); 253 NICVF_CORE_UNLOCK(nic); 254 255 err = nicvf_setup_ifnet(nic); 256 if (err != 0) { 257 device_printf(dev, "Could not set-up ifnet\n"); 258 goto err_release_intr; 259 } 260 261 err = nicvf_setup_ifmedia(nic); 262 if (err != 0) { 263 device_printf(dev, "Could not set-up ifmedia\n"); 264 goto err_free_ifnet; 265 } 266 267 mtx_init(&nic->stats_mtx, "VNIC stats", NULL, MTX_DEF); 268 callout_init_mtx(&nic->stats_callout, &nic->stats_mtx, 0); 269 270 ether_ifattach(nic->ifp, nic->hwaddr); 271 272 return (0); 273 274 err_free_ifnet: 275 if_free(nic->ifp); 276 err_release_intr: 277 nicvf_release_all_interrupts(nic); 278 err_free_res: 279 bus_release_resource(dev, SYS_RES_MEMORY, rman_get_rid(nic->reg_base), 280 nic->reg_base); 281 282 return (err); 283 } 284 285 static int 286 nicvf_detach(device_t dev) 287 { 288 struct nicvf *nic; 289 290 nic = device_get_softc(dev); 291 292 NICVF_CORE_LOCK(nic); 293 /* Shut down the port and release ring resources */ 294 nicvf_stop_locked(nic); 295 /* Release stats lock */ 296 mtx_destroy(&nic->stats_mtx); 297 /* Release interrupts */ 298 nicvf_release_all_interrupts(nic); 299 /* Release memory resource */ 300 if (nic->reg_base != NULL) { 301 bus_release_resource(dev, SYS_RES_MEMORY, 302 rman_get_rid(nic->reg_base), nic->reg_base); 303 } 304 305 /* Remove all ifmedia configurations */ 306 ifmedia_removeall(&nic->if_media); 307 /* Free this ifnet */ 308 if_free(nic->ifp); 309 NICVF_CORE_UNLOCK(nic); 310 /* Finally destroy the lock */ 311 NICVF_CORE_LOCK_DESTROY(nic); 312 313 return (0); 314 } 315 316 static void 317 nicvf_hw_addr_random(uint8_t *hwaddr) 318 { 319 uint32_t rnd; 320 uint8_t addr[ETHER_ADDR_LEN]; 321 322 /* 323 * Create randomized MAC address. 324 * Set 'bsd' + random 24 low-order bits. 325 */ 326 rnd = arc4random() & 0x00ffffff; 327 addr[0] = 'b'; 328 addr[1] = 's'; 329 addr[2] = 'd'; 330 addr[3] = rnd >> 16; 331 addr[4] = rnd >> 8; 332 addr[5] = rnd >> 0; 333 334 memcpy(hwaddr, addr, ETHER_ADDR_LEN); 335 } 336 337 static int 338 nicvf_setup_ifnet(struct nicvf *nic) 339 { 340 struct ifnet *ifp; 341 342 ifp = if_alloc(IFT_ETHER); 343 if (ifp == NULL) { 344 device_printf(nic->dev, "Could not allocate ifnet structure\n"); 345 return (ENOMEM); 346 } 347 348 nic->ifp = ifp; 349 350 if_setsoftc(ifp, nic); 351 if_initname(ifp, device_get_name(nic->dev), device_get_unit(nic->dev)); 352 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX); 353 354 if_settransmitfn(ifp, nicvf_if_transmit); 355 if_setqflushfn(ifp, nicvf_if_qflush); 356 if_setioctlfn(ifp, nicvf_if_ioctl); 357 if_setinitfn(ifp, nicvf_if_init); 358 if_setgetcounterfn(ifp, nicvf_if_getcounter); 359 360 if_setmtu(ifp, ETHERMTU); 361 362 /* Reset caps */ 363 if_setcapabilities(ifp, 0); 364 365 /* Set the default values */ 366 if_setcapabilitiesbit(ifp, IFCAP_VLAN_MTU | IFCAP_JUMBO_MTU, 0); 367 if_setcapabilitiesbit(ifp, IFCAP_LRO, 0); 368 if (nic->hw_tso) { 369 /* TSO */ 370 if_setcapabilitiesbit(ifp, IFCAP_TSO4, 0); 371 /* TSO parameters */ 372 ifp->if_hw_tsomax = NICVF_TSO_MAXSIZE; 373 ifp->if_hw_tsomaxsegcount = NICVF_TSO_NSEGS; 374 ifp->if_hw_tsomaxsegsize = MCLBYTES; 375 } 376 /* IP/TCP/UDP HW checksums */ 377 if_setcapabilitiesbit(ifp, IFCAP_HWCSUM, 0); 378 if_setcapabilitiesbit(ifp, IFCAP_HWSTATS, 0); 379 /* 380 * HW offload enable 381 */ 382 if_clearhwassist(ifp); 383 if_sethwassistbits(ifp, (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_SCTP), 0); 384 if (nic->hw_tso) 385 if_sethwassistbits(ifp, (CSUM_TSO), 0); 386 if_setcapenable(ifp, if_getcapabilities(ifp)); 387 388 return (0); 389 } 390 391 static int 392 nicvf_setup_ifmedia(struct nicvf *nic) 393 { 394 395 ifmedia_init(&nic->if_media, IFM_IMASK, nicvf_media_change, 396 nicvf_media_status); 397 398 /* 399 * Advertise availability of all possible connection types, 400 * even though not all are possible at the same time. 401 */ 402 403 ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_10_T | IFM_FDX), 404 0, NULL); 405 ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_100_TX | IFM_FDX), 406 0, NULL); 407 ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_1000_T | IFM_FDX), 408 0, NULL); 409 ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_10G_SR | IFM_FDX), 410 0, NULL); 411 ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_40G_CR4 | IFM_FDX), 412 0, NULL); 413 ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_AUTO | IFM_FDX), 414 0, NULL); 415 416 ifmedia_set(&nic->if_media, (IFM_ETHER | IFM_AUTO | IFM_FDX)); 417 418 return (0); 419 } 420 421 static int 422 nicvf_if_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 423 { 424 struct nicvf *nic; 425 struct rcv_queue *rq; 426 struct ifreq *ifr; 427 uint32_t flags; 428 int mask, err; 429 int rq_idx; 430 #if defined(INET) || defined(INET6) 431 struct ifaddr *ifa; 432 boolean_t avoid_reset = FALSE; 433 #endif 434 435 nic = if_getsoftc(ifp); 436 ifr = (struct ifreq *)data; 437 #if defined(INET) || defined(INET6) 438 ifa = (struct ifaddr *)data; 439 #endif 440 err = 0; 441 switch (cmd) { 442 case SIOCSIFADDR: 443 #ifdef INET 444 if (ifa->ifa_addr->sa_family == AF_INET) 445 avoid_reset = TRUE; 446 #endif 447 #ifdef INET6 448 if (ifa->ifa_addr->sa_family == AF_INET6) 449 avoid_reset = TRUE; 450 #endif 451 452 #if defined(INET) || defined(INET6) 453 /* Avoid reinitialization unless it's necessary */ 454 if (avoid_reset) { 455 ifp->if_flags |= IFF_UP; 456 if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) 457 nicvf_if_init(nic); 458 #ifdef INET 459 if (!(if_getflags(ifp) & IFF_NOARP)) 460 arp_ifinit(ifp, ifa); 461 #endif 462 463 return (0); 464 } 465 #endif 466 err = ether_ioctl(ifp, cmd, data); 467 break; 468 case SIOCSIFMTU: 469 if (ifr->ifr_mtu < NIC_HW_MIN_FRS || 470 ifr->ifr_mtu > NIC_HW_MAX_FRS) { 471 err = EINVAL; 472 } else { 473 NICVF_CORE_LOCK(nic); 474 err = nicvf_update_hw_max_frs(nic, ifr->ifr_mtu); 475 if (err == 0) 476 if_setmtu(ifp, ifr->ifr_mtu); 477 NICVF_CORE_UNLOCK(nic); 478 } 479 break; 480 case SIOCSIFFLAGS: 481 NICVF_CORE_LOCK(nic); 482 if (if_getflags(ifp) & IFF_UP) { 483 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 484 flags = ifp->if_flags ^ nic->if_flags; 485 if ((nic->if_flags & ifp->if_flags) & 486 IFF_PROMISC) { 487 /* Change promiscous mode */ 488 #if 0 489 /* ARM64TODO */ 490 nicvf_set_promiscous(nic); 491 #endif 492 } 493 494 if ((nic->if_flags ^ ifp->if_flags) & 495 IFF_ALLMULTI) { 496 /* Change multicasting settings */ 497 #if 0 498 /* ARM64TODO */ 499 nicvf_set_multicast(nic); 500 #endif 501 } 502 } else { 503 nicvf_if_init_locked(nic); 504 } 505 } else if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) 506 nicvf_stop_locked(nic); 507 508 nic->if_flags = ifp->if_flags; 509 NICVF_CORE_UNLOCK(nic); 510 break; 511 512 case SIOCADDMULTI: 513 case SIOCDELMULTI: 514 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 515 #if 0 516 NICVF_CORE_LOCK(nic); 517 /* ARM64TODO */ 518 nicvf_set_multicast(nic); 519 NICVF_CORE_UNLOCK(nic); 520 #endif 521 } 522 break; 523 524 case SIOCSIFMEDIA: 525 case SIOCGIFMEDIA: 526 err = ifmedia_ioctl(ifp, ifr, &nic->if_media, cmd); 527 break; 528 529 case SIOCSIFCAP: 530 mask = ifp->if_capenable ^ ifr->ifr_reqcap; 531 if (mask & IFCAP_VLAN_MTU) { 532 /* No work to do except acknowledge the change took. */ 533 ifp->if_capenable ^= IFCAP_VLAN_MTU; 534 } 535 if (mask & IFCAP_TXCSUM) 536 ifp->if_capenable ^= IFCAP_TXCSUM; 537 if (mask & IFCAP_RXCSUM) 538 ifp->if_capenable ^= IFCAP_RXCSUM; 539 if ((mask & IFCAP_TSO4) && nic->hw_tso) 540 ifp->if_capenable ^= IFCAP_TSO4; 541 if (mask & IFCAP_LRO) { 542 /* 543 * Lock the driver for a moment to avoid 544 * mismatch in per-queue settings. 545 */ 546 NICVF_CORE_LOCK(nic); 547 ifp->if_capenable ^= IFCAP_LRO; 548 if ((if_getdrvflags(nic->ifp) & IFF_DRV_RUNNING) != 0) { 549 /* 550 * Now disable LRO for subsequent packets. 551 * Atomicity of this change is not necessary 552 * as we don't need precise toggle of this 553 * feature for all threads processing the 554 * completion queue. 555 */ 556 for (rq_idx = 0; 557 rq_idx < nic->qs->rq_cnt; rq_idx++) { 558 rq = &nic->qs->rq[rq_idx]; 559 rq->lro_enabled = !rq->lro_enabled; 560 } 561 } 562 NICVF_CORE_UNLOCK(nic); 563 } 564 565 break; 566 567 default: 568 err = ether_ioctl(ifp, cmd, data); 569 break; 570 } 571 572 return (err); 573 } 574 575 static void 576 nicvf_if_init_locked(struct nicvf *nic) 577 { 578 struct queue_set *qs = nic->qs; 579 struct ifnet *ifp; 580 int qidx; 581 int err; 582 caddr_t if_addr; 583 584 NICVF_CORE_LOCK_ASSERT(nic); 585 ifp = nic->ifp; 586 587 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) 588 nicvf_stop_locked(nic); 589 590 err = nicvf_enable_misc_interrupt(nic); 591 if (err != 0) { 592 if_printf(ifp, "Could not reenable Mbox interrupt\n"); 593 return; 594 } 595 596 /* Get the latest MAC address */ 597 if_addr = if_getlladdr(ifp); 598 /* Update MAC address if changed */ 599 if (memcmp(nic->hwaddr, if_addr, ETHER_ADDR_LEN) != 0) { 600 memcpy(nic->hwaddr, if_addr, ETHER_ADDR_LEN); 601 nicvf_hw_set_mac_addr(nic, if_addr); 602 } 603 604 /* Initialize the queues */ 605 err = nicvf_init_resources(nic); 606 if (err != 0) 607 goto error; 608 609 /* Make sure queue initialization is written */ 610 wmb(); 611 612 nicvf_reg_write(nic, NIC_VF_INT, ~0UL); 613 /* Enable Qset err interrupt */ 614 nicvf_enable_intr(nic, NICVF_INTR_QS_ERR, 0); 615 616 /* Enable completion queue interrupt */ 617 for (qidx = 0; qidx < qs->cq_cnt; qidx++) 618 nicvf_enable_intr(nic, NICVF_INTR_CQ, qidx); 619 620 /* Enable RBDR threshold interrupt */ 621 for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) 622 nicvf_enable_intr(nic, NICVF_INTR_RBDR, qidx); 623 624 nic->drv_stats.txq_stop = 0; 625 nic->drv_stats.txq_wake = 0; 626 627 /* Activate network interface */ 628 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); 629 630 /* Schedule callout to update stats */ 631 callout_reset(&nic->stats_callout, hz, nicvf_tick_stats, nic); 632 633 return; 634 635 error: 636 /* Something went very wrong. Disable this ifnet for good */ 637 if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); 638 } 639 640 static void 641 nicvf_if_init(void *if_softc) 642 { 643 struct nicvf *nic = if_softc; 644 645 NICVF_CORE_LOCK(nic); 646 nicvf_if_init_locked(nic); 647 NICVF_CORE_UNLOCK(nic); 648 } 649 650 static int 651 nicvf_if_transmit(struct ifnet *ifp, struct mbuf *mbuf) 652 { 653 struct nicvf *nic = if_getsoftc(ifp); 654 struct queue_set *qs = nic->qs; 655 struct snd_queue *sq; 656 struct mbuf *mtmp; 657 int qidx; 658 int err = 0; 659 660 661 if (__predict_false(qs == NULL)) { 662 panic("%s: missing queue set for %s", __func__, 663 device_get_nameunit(nic->dev)); 664 } 665 666 /* Select queue */ 667 if (M_HASHTYPE_GET(mbuf) != M_HASHTYPE_NONE) 668 qidx = mbuf->m_pkthdr.flowid % qs->sq_cnt; 669 else 670 qidx = curcpu % qs->sq_cnt; 671 672 sq = &qs->sq[qidx]; 673 674 if (mbuf->m_next != NULL && 675 (mbuf->m_pkthdr.csum_flags & 676 (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_SCTP)) != 0) { 677 if (M_WRITABLE(mbuf) == 0) { 678 mtmp = m_dup(mbuf, M_NOWAIT); 679 m_freem(mbuf); 680 if (mtmp == NULL) 681 return (ENOBUFS); 682 mbuf = mtmp; 683 } 684 } 685 686 err = drbr_enqueue(ifp, sq->br, mbuf); 687 if (((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 688 IFF_DRV_RUNNING) || !nic->link_up || (err != 0)) { 689 /* 690 * Try to enqueue packet to the ring buffer. 691 * If the driver is not active, link down or enqueue operation 692 * failed, return with the appropriate error code. 693 */ 694 return (err); 695 } 696 697 if (NICVF_TX_TRYLOCK(sq) != 0) { 698 err = nicvf_xmit_locked(sq); 699 NICVF_TX_UNLOCK(sq); 700 return (err); 701 } else 702 taskqueue_enqueue(sq->snd_taskq, &sq->snd_task); 703 704 return (0); 705 } 706 707 static void 708 nicvf_if_qflush(struct ifnet *ifp) 709 { 710 struct nicvf *nic; 711 struct queue_set *qs; 712 struct snd_queue *sq; 713 struct mbuf *mbuf; 714 size_t idx; 715 716 nic = if_getsoftc(ifp); 717 qs = nic->qs; 718 719 for (idx = 0; idx < qs->sq_cnt; idx++) { 720 sq = &qs->sq[idx]; 721 NICVF_TX_LOCK(sq); 722 while ((mbuf = buf_ring_dequeue_sc(sq->br)) != NULL) 723 m_freem(mbuf); 724 NICVF_TX_UNLOCK(sq); 725 } 726 if_qflush(ifp); 727 } 728 729 static uint64_t 730 nicvf_if_getcounter(struct ifnet *ifp, ift_counter cnt) 731 { 732 struct nicvf *nic; 733 struct nicvf_hw_stats *hw_stats; 734 struct nicvf_drv_stats *drv_stats; 735 736 nic = if_getsoftc(ifp); 737 hw_stats = &nic->hw_stats; 738 drv_stats = &nic->drv_stats; 739 740 switch (cnt) { 741 case IFCOUNTER_IPACKETS: 742 return (drv_stats->rx_frames_ok); 743 case IFCOUNTER_OPACKETS: 744 return (drv_stats->tx_frames_ok); 745 case IFCOUNTER_IBYTES: 746 return (hw_stats->rx_bytes); 747 case IFCOUNTER_OBYTES: 748 return (hw_stats->tx_bytes_ok); 749 case IFCOUNTER_IMCASTS: 750 return (hw_stats->rx_mcast_frames); 751 case IFCOUNTER_COLLISIONS: 752 return (0); 753 case IFCOUNTER_IQDROPS: 754 return (drv_stats->rx_drops); 755 case IFCOUNTER_OQDROPS: 756 return (drv_stats->tx_drops); 757 default: 758 return (if_get_counter_default(ifp, cnt)); 759 } 760 761 } 762 763 static void 764 nicvf_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 765 { 766 struct nicvf *nic = if_getsoftc(ifp); 767 768 NICVF_CORE_LOCK(nic); 769 770 ifmr->ifm_status = IFM_AVALID; 771 ifmr->ifm_active = IFM_ETHER; 772 773 if (nic->link_up) { 774 /* Device attached to working network */ 775 ifmr->ifm_status |= IFM_ACTIVE; 776 } 777 778 switch (nic->speed) { 779 case SPEED_10: 780 ifmr->ifm_active |= IFM_10_T; 781 break; 782 case SPEED_100: 783 ifmr->ifm_active |= IFM_100_TX; 784 break; 785 case SPEED_1000: 786 ifmr->ifm_active |= IFM_1000_T; 787 break; 788 case SPEED_10000: 789 ifmr->ifm_active |= IFM_10G_SR; 790 break; 791 case SPEED_40000: 792 ifmr->ifm_active |= IFM_40G_CR4; 793 break; 794 default: 795 ifmr->ifm_active |= IFM_AUTO; 796 break; 797 } 798 799 if (nic->duplex) 800 ifmr->ifm_active |= IFM_FDX; 801 else 802 ifmr->ifm_active |= IFM_HDX; 803 804 NICVF_CORE_UNLOCK(nic); 805 } 806 807 static int 808 nicvf_media_change(struct ifnet *ifp __unused) 809 { 810 811 return (0); 812 } 813 814 /* Register read/write APIs */ 815 void 816 nicvf_reg_write(struct nicvf *nic, bus_space_handle_t offset, uint64_t val) 817 { 818 819 bus_write_8(nic->reg_base, offset, val); 820 } 821 822 uint64_t 823 nicvf_reg_read(struct nicvf *nic, uint64_t offset) 824 { 825 826 return (bus_read_8(nic->reg_base, offset)); 827 } 828 829 void 830 nicvf_queue_reg_write(struct nicvf *nic, bus_space_handle_t offset, 831 uint64_t qidx, uint64_t val) 832 { 833 834 bus_write_8(nic->reg_base, offset + (qidx << NIC_Q_NUM_SHIFT), val); 835 } 836 837 uint64_t 838 nicvf_queue_reg_read(struct nicvf *nic, bus_space_handle_t offset, 839 uint64_t qidx) 840 { 841 842 return (bus_read_8(nic->reg_base, offset + (qidx << NIC_Q_NUM_SHIFT))); 843 } 844 845 /* VF -> PF mailbox communication */ 846 static void 847 nicvf_write_to_mbx(struct nicvf *nic, union nic_mbx *mbx) 848 { 849 uint64_t *msg = (uint64_t *)mbx; 850 851 nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 0, msg[0]); 852 nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 8, msg[1]); 853 } 854 855 int 856 nicvf_send_msg_to_pf(struct nicvf *nic, union nic_mbx *mbx) 857 { 858 int timeout = NIC_MBOX_MSG_TIMEOUT * 10; 859 int sleep = 2; 860 861 NICVF_CORE_LOCK_ASSERT(nic); 862 863 nic->pf_acked = FALSE; 864 nic->pf_nacked = FALSE; 865 866 nicvf_write_to_mbx(nic, mbx); 867 868 /* Wait for previous message to be acked, timeout 2sec */ 869 while (!nic->pf_acked) { 870 if (nic->pf_nacked) 871 return (EINVAL); 872 873 DELAY(sleep * 1000); 874 875 if (nic->pf_acked) 876 break; 877 timeout -= sleep; 878 if (!timeout) { 879 device_printf(nic->dev, 880 "PF didn't ack to mbox msg %d from VF%d\n", 881 (mbx->msg.msg & 0xFF), nic->vf_id); 882 883 return (EBUSY); 884 } 885 } 886 return (0); 887 } 888 889 /* 890 * Checks if VF is able to comminicate with PF 891 * and also gets the VNIC number this VF is associated to. 892 */ 893 static int 894 nicvf_check_pf_ready(struct nicvf *nic) 895 { 896 union nic_mbx mbx = {}; 897 898 mbx.msg.msg = NIC_MBOX_MSG_READY; 899 if (nicvf_send_msg_to_pf(nic, &mbx)) { 900 device_printf(nic->dev, 901 "PF didn't respond to READY msg\n"); 902 return 0; 903 } 904 905 return 1; 906 } 907 908 static void 909 nicvf_read_bgx_stats(struct nicvf *nic, struct bgx_stats_msg *bgx) 910 { 911 912 if (bgx->rx) 913 nic->bgx_stats.rx_stats[bgx->idx] = bgx->stats; 914 else 915 nic->bgx_stats.tx_stats[bgx->idx] = bgx->stats; 916 } 917 918 static void 919 nicvf_handle_mbx_intr(struct nicvf *nic) 920 { 921 union nic_mbx mbx = {}; 922 uint64_t *mbx_data; 923 uint64_t mbx_addr; 924 int i; 925 926 mbx_addr = NIC_VF_PF_MAILBOX_0_1; 927 mbx_data = (uint64_t *)&mbx; 928 929 for (i = 0; i < NIC_PF_VF_MAILBOX_SIZE; i++) { 930 *mbx_data = nicvf_reg_read(nic, mbx_addr); 931 mbx_data++; 932 mbx_addr += sizeof(uint64_t); 933 } 934 935 switch (mbx.msg.msg) { 936 case NIC_MBOX_MSG_READY: 937 nic->pf_acked = TRUE; 938 nic->vf_id = mbx.nic_cfg.vf_id & 0x7F; 939 nic->tns_mode = mbx.nic_cfg.tns_mode & 0x7F; 940 nic->node = mbx.nic_cfg.node_id; 941 memcpy(nic->hwaddr, mbx.nic_cfg.mac_addr, ETHER_ADDR_LEN); 942 nic->loopback_supported = mbx.nic_cfg.loopback_supported; 943 nic->link_up = FALSE; 944 nic->duplex = 0; 945 nic->speed = 0; 946 break; 947 case NIC_MBOX_MSG_ACK: 948 nic->pf_acked = TRUE; 949 break; 950 case NIC_MBOX_MSG_NACK: 951 nic->pf_nacked = TRUE; 952 break; 953 case NIC_MBOX_MSG_RSS_SIZE: 954 nic->rss_info.rss_size = mbx.rss_size.ind_tbl_size; 955 nic->pf_acked = TRUE; 956 break; 957 case NIC_MBOX_MSG_BGX_STATS: 958 nicvf_read_bgx_stats(nic, &mbx.bgx_stats); 959 nic->pf_acked = TRUE; 960 break; 961 case NIC_MBOX_MSG_BGX_LINK_CHANGE: 962 nic->pf_acked = TRUE; 963 nic->link_up = mbx.link_status.link_up; 964 nic->duplex = mbx.link_status.duplex; 965 nic->speed = mbx.link_status.speed; 966 if (nic->link_up) { 967 if_setbaudrate(nic->ifp, nic->speed * 1000000); 968 if_link_state_change(nic->ifp, LINK_STATE_UP); 969 } else { 970 if_setbaudrate(nic->ifp, 0); 971 if_link_state_change(nic->ifp, LINK_STATE_DOWN); 972 } 973 break; 974 default: 975 device_printf(nic->dev, 976 "Invalid message from PF, msg 0x%x\n", mbx.msg.msg); 977 break; 978 } 979 nicvf_clear_intr(nic, NICVF_INTR_MBOX, 0); 980 } 981 982 static int 983 nicvf_update_hw_max_frs(struct nicvf *nic, int mtu) 984 { 985 union nic_mbx mbx = {}; 986 987 mbx.frs.msg = NIC_MBOX_MSG_SET_MAX_FRS; 988 mbx.frs.max_frs = mtu; 989 mbx.frs.vf_id = nic->vf_id; 990 991 return nicvf_send_msg_to_pf(nic, &mbx); 992 } 993 994 static int 995 nicvf_hw_set_mac_addr(struct nicvf *nic, uint8_t *hwaddr) 996 { 997 union nic_mbx mbx = {}; 998 999 mbx.mac.msg = NIC_MBOX_MSG_SET_MAC; 1000 mbx.mac.vf_id = nic->vf_id; 1001 memcpy(mbx.mac.mac_addr, hwaddr, ETHER_ADDR_LEN); 1002 1003 return (nicvf_send_msg_to_pf(nic, &mbx)); 1004 } 1005 1006 static void 1007 nicvf_config_cpi(struct nicvf *nic) 1008 { 1009 union nic_mbx mbx = {}; 1010 1011 mbx.cpi_cfg.msg = NIC_MBOX_MSG_CPI_CFG; 1012 mbx.cpi_cfg.vf_id = nic->vf_id; 1013 mbx.cpi_cfg.cpi_alg = nic->cpi_alg; 1014 mbx.cpi_cfg.rq_cnt = nic->qs->rq_cnt; 1015 1016 nicvf_send_msg_to_pf(nic, &mbx); 1017 } 1018 1019 static void 1020 nicvf_get_rss_size(struct nicvf *nic) 1021 { 1022 union nic_mbx mbx = {}; 1023 1024 mbx.rss_size.msg = NIC_MBOX_MSG_RSS_SIZE; 1025 mbx.rss_size.vf_id = nic->vf_id; 1026 nicvf_send_msg_to_pf(nic, &mbx); 1027 } 1028 1029 static void 1030 nicvf_config_rss(struct nicvf *nic) 1031 { 1032 union nic_mbx mbx = {}; 1033 struct nicvf_rss_info *rss; 1034 int ind_tbl_len; 1035 int i, nextq; 1036 1037 rss = &nic->rss_info; 1038 ind_tbl_len = rss->rss_size; 1039 nextq = 0; 1040 1041 mbx.rss_cfg.vf_id = nic->vf_id; 1042 mbx.rss_cfg.hash_bits = rss->hash_bits; 1043 while (ind_tbl_len != 0) { 1044 mbx.rss_cfg.tbl_offset = nextq; 1045 mbx.rss_cfg.tbl_len = MIN(ind_tbl_len, 1046 RSS_IND_TBL_LEN_PER_MBX_MSG); 1047 mbx.rss_cfg.msg = mbx.rss_cfg.tbl_offset ? 1048 NIC_MBOX_MSG_RSS_CFG_CONT : NIC_MBOX_MSG_RSS_CFG; 1049 1050 for (i = 0; i < mbx.rss_cfg.tbl_len; i++) 1051 mbx.rss_cfg.ind_tbl[i] = rss->ind_tbl[nextq++]; 1052 1053 nicvf_send_msg_to_pf(nic, &mbx); 1054 1055 ind_tbl_len -= mbx.rss_cfg.tbl_len; 1056 } 1057 } 1058 1059 static void 1060 nicvf_set_rss_key(struct nicvf *nic) 1061 { 1062 struct nicvf_rss_info *rss; 1063 uint64_t key_addr; 1064 int idx; 1065 1066 rss = &nic->rss_info; 1067 key_addr = NIC_VNIC_RSS_KEY_0_4; 1068 1069 for (idx = 0; idx < RSS_HASH_KEY_SIZE; idx++) { 1070 nicvf_reg_write(nic, key_addr, rss->key[idx]); 1071 key_addr += sizeof(uint64_t); 1072 } 1073 } 1074 1075 static int 1076 nicvf_rss_init(struct nicvf *nic) 1077 { 1078 struct nicvf_rss_info *rss; 1079 int idx; 1080 1081 nicvf_get_rss_size(nic); 1082 1083 rss = &nic->rss_info; 1084 if (nic->cpi_alg != CPI_ALG_NONE) { 1085 rss->enable = FALSE; 1086 rss->hash_bits = 0; 1087 return (ENXIO); 1088 } 1089 1090 rss->enable = TRUE; 1091 1092 /* Using the HW reset value for now */ 1093 rss->key[0] = 0xFEED0BADFEED0BADUL; 1094 rss->key[1] = 0xFEED0BADFEED0BADUL; 1095 rss->key[2] = 0xFEED0BADFEED0BADUL; 1096 rss->key[3] = 0xFEED0BADFEED0BADUL; 1097 rss->key[4] = 0xFEED0BADFEED0BADUL; 1098 1099 nicvf_set_rss_key(nic); 1100 1101 rss->cfg = RSS_IP_HASH_ENA | RSS_TCP_HASH_ENA | RSS_UDP_HASH_ENA; 1102 nicvf_reg_write(nic, NIC_VNIC_RSS_CFG, rss->cfg); 1103 1104 rss->hash_bits = fls(rss->rss_size) - 1; 1105 for (idx = 0; idx < rss->rss_size; idx++) 1106 rss->ind_tbl[idx] = idx % nic->rx_queues; 1107 1108 nicvf_config_rss(nic); 1109 1110 return (0); 1111 } 1112 1113 static int 1114 nicvf_init_resources(struct nicvf *nic) 1115 { 1116 int err; 1117 union nic_mbx mbx = {}; 1118 1119 mbx.msg.msg = NIC_MBOX_MSG_CFG_DONE; 1120 1121 /* Enable Qset */ 1122 nicvf_qset_config(nic, TRUE); 1123 1124 /* Initialize queues and HW for data transfer */ 1125 err = nicvf_config_data_transfer(nic, TRUE); 1126 if (err) { 1127 device_printf(nic->dev, 1128 "Failed to alloc/config VF's QSet resources\n"); 1129 return (err); 1130 } 1131 1132 /* Send VF config done msg to PF */ 1133 nicvf_write_to_mbx(nic, &mbx); 1134 1135 return (0); 1136 } 1137 1138 static void 1139 nicvf_misc_intr_handler(void *arg) 1140 { 1141 struct nicvf *nic = (struct nicvf *)arg; 1142 uint64_t intr; 1143 1144 intr = nicvf_reg_read(nic, NIC_VF_INT); 1145 /* Check for spurious interrupt */ 1146 if (!(intr & NICVF_INTR_MBOX_MASK)) 1147 return; 1148 1149 nicvf_handle_mbx_intr(nic); 1150 } 1151 1152 static int 1153 nicvf_intr_handler(void *arg) 1154 { 1155 struct nicvf *nic; 1156 struct cmp_queue *cq; 1157 int qidx; 1158 1159 cq = (struct cmp_queue *)arg; 1160 nic = cq->nic; 1161 qidx = cq->idx; 1162 1163 /* Disable interrupts */ 1164 nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx); 1165 1166 taskqueue_enqueue(cq->cmp_taskq, &cq->cmp_task); 1167 1168 /* Clear interrupt */ 1169 nicvf_clear_intr(nic, NICVF_INTR_CQ, qidx); 1170 1171 return (FILTER_HANDLED); 1172 } 1173 1174 static void 1175 nicvf_rbdr_intr_handler(void *arg) 1176 { 1177 struct nicvf *nic; 1178 struct queue_set *qs; 1179 struct rbdr *rbdr; 1180 int qidx; 1181 1182 nic = (struct nicvf *)arg; 1183 1184 /* Disable RBDR interrupt and schedule softirq */ 1185 for (qidx = 0; qidx < nic->qs->rbdr_cnt; qidx++) { 1186 if (!nicvf_is_intr_enabled(nic, NICVF_INTR_RBDR, qidx)) 1187 continue; 1188 nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx); 1189 1190 qs = nic->qs; 1191 rbdr = &qs->rbdr[qidx]; 1192 taskqueue_enqueue(rbdr->rbdr_taskq, &rbdr->rbdr_task_nowait); 1193 /* Clear interrupt */ 1194 nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx); 1195 } 1196 } 1197 1198 static void 1199 nicvf_qs_err_intr_handler(void *arg) 1200 { 1201 struct nicvf *nic = (struct nicvf *)arg; 1202 struct queue_set *qs = nic->qs; 1203 1204 /* Disable Qset err interrupt and schedule softirq */ 1205 nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0); 1206 taskqueue_enqueue(qs->qs_err_taskq, &qs->qs_err_task); 1207 nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0); 1208 1209 } 1210 1211 static int 1212 nicvf_enable_msix(struct nicvf *nic) 1213 { 1214 struct pci_devinfo *dinfo; 1215 int rid, count; 1216 int ret; 1217 1218 dinfo = device_get_ivars(nic->dev); 1219 rid = dinfo->cfg.msix.msix_table_bar; 1220 nic->msix_table_res = 1221 bus_alloc_resource_any(nic->dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); 1222 if (nic->msix_table_res == NULL) { 1223 device_printf(nic->dev, 1224 "Could not allocate memory for MSI-X table\n"); 1225 return (ENXIO); 1226 } 1227 1228 count = nic->num_vec = NIC_VF_MSIX_VECTORS; 1229 1230 ret = pci_alloc_msix(nic->dev, &count); 1231 if ((ret != 0) || (count != nic->num_vec)) { 1232 device_printf(nic->dev, 1233 "Request for #%d msix vectors failed, error: %d\n", 1234 nic->num_vec, ret); 1235 return (ret); 1236 } 1237 1238 nic->msix_enabled = 1; 1239 return (0); 1240 } 1241 1242 static void 1243 nicvf_disable_msix(struct nicvf *nic) 1244 { 1245 1246 if (nic->msix_enabled) { 1247 pci_release_msi(nic->dev); 1248 nic->msix_enabled = 0; 1249 nic->num_vec = 0; 1250 } 1251 } 1252 1253 static void 1254 nicvf_release_all_interrupts(struct nicvf *nic) 1255 { 1256 struct resource *res; 1257 int irq; 1258 int err; 1259 1260 /* Free registered interrupts */ 1261 for (irq = 0; irq < nic->num_vec; irq++) { 1262 res = nic->msix_entries[irq].irq_res; 1263 if (res == NULL) 1264 continue; 1265 /* Teardown interrupt first */ 1266 if (nic->msix_entries[irq].handle != NULL) { 1267 err = bus_teardown_intr(nic->dev, 1268 nic->msix_entries[irq].irq_res, 1269 nic->msix_entries[irq].handle); 1270 KASSERT(err == 0, 1271 ("ERROR: Unable to teardown interrupt %d", irq)); 1272 nic->msix_entries[irq].handle = NULL; 1273 } 1274 1275 bus_release_resource(nic->dev, SYS_RES_IRQ, 1276 rman_get_rid(res), nic->msix_entries[irq].irq_res); 1277 nic->msix_entries[irq].irq_res = NULL; 1278 } 1279 /* Disable MSI-X */ 1280 nicvf_disable_msix(nic); 1281 } 1282 1283 /* 1284 * Initialize MSIX vectors and register MISC interrupt. 1285 * Send READY message to PF to check if its alive 1286 */ 1287 static int 1288 nicvf_allocate_misc_interrupt(struct nicvf *nic) 1289 { 1290 struct resource *res; 1291 int irq, rid; 1292 int ret = 0; 1293 1294 /* Return if mailbox interrupt is already registered */ 1295 if (nic->msix_enabled) 1296 return (0); 1297 1298 /* Enable MSI-X */ 1299 if (nicvf_enable_msix(nic) != 0) 1300 return (ENXIO); 1301 1302 irq = NICVF_INTR_ID_MISC; 1303 rid = irq + 1; 1304 nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev, 1305 SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE)); 1306 if (nic->msix_entries[irq].irq_res == NULL) { 1307 device_printf(nic->dev, 1308 "Could not allocate Mbox interrupt for VF%d\n", 1309 device_get_unit(nic->dev)); 1310 return (ENXIO); 1311 } 1312 1313 ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res, 1314 (INTR_MPSAFE | INTR_TYPE_MISC), NULL, nicvf_misc_intr_handler, nic, 1315 &nic->msix_entries[irq].handle); 1316 if (ret != 0) { 1317 res = nic->msix_entries[irq].irq_res; 1318 bus_release_resource(nic->dev, SYS_RES_IRQ, 1319 rman_get_rid(res), res); 1320 nic->msix_entries[irq].irq_res = NULL; 1321 return (ret); 1322 } 1323 1324 return (0); 1325 } 1326 1327 static int 1328 nicvf_enable_misc_interrupt(struct nicvf *nic) 1329 { 1330 1331 /* Enable mailbox interrupt */ 1332 nicvf_enable_intr(nic, NICVF_INTR_MBOX, 0); 1333 1334 /* Check if VF is able to communicate with PF */ 1335 if (!nicvf_check_pf_ready(nic)) { 1336 nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0); 1337 return (ENXIO); 1338 } 1339 1340 return (0); 1341 } 1342 1343 static void 1344 nicvf_release_net_interrupts(struct nicvf *nic) 1345 { 1346 struct resource *res; 1347 int irq; 1348 int err; 1349 1350 for_each_cq_irq(irq) { 1351 res = nic->msix_entries[irq].irq_res; 1352 if (res == NULL) 1353 continue; 1354 /* Teardown active interrupts first */ 1355 if (nic->msix_entries[irq].handle != NULL) { 1356 err = bus_teardown_intr(nic->dev, 1357 nic->msix_entries[irq].irq_res, 1358 nic->msix_entries[irq].handle); 1359 KASSERT(err == 0, 1360 ("ERROR: Unable to teardown CQ interrupt %d", 1361 (irq - NICVF_INTR_ID_CQ))); 1362 if (err != 0) 1363 continue; 1364 } 1365 1366 /* Release resource */ 1367 bus_release_resource(nic->dev, SYS_RES_IRQ, rman_get_rid(res), 1368 res); 1369 nic->msix_entries[irq].irq_res = NULL; 1370 } 1371 1372 for_each_rbdr_irq(irq) { 1373 res = nic->msix_entries[irq].irq_res; 1374 if (res == NULL) 1375 continue; 1376 /* Teardown active interrupts first */ 1377 if (nic->msix_entries[irq].handle != NULL) { 1378 err = bus_teardown_intr(nic->dev, 1379 nic->msix_entries[irq].irq_res, 1380 nic->msix_entries[irq].handle); 1381 KASSERT(err == 0, 1382 ("ERROR: Unable to teardown RDBR interrupt %d", 1383 (irq - NICVF_INTR_ID_RBDR))); 1384 if (err != 0) 1385 continue; 1386 } 1387 1388 /* Release resource */ 1389 bus_release_resource(nic->dev, SYS_RES_IRQ, rman_get_rid(res), 1390 res); 1391 nic->msix_entries[irq].irq_res = NULL; 1392 } 1393 1394 irq = NICVF_INTR_ID_QS_ERR; 1395 res = nic->msix_entries[irq].irq_res; 1396 if (res != NULL) { 1397 /* Teardown active interrupts first */ 1398 if (nic->msix_entries[irq].handle != NULL) { 1399 err = bus_teardown_intr(nic->dev, 1400 nic->msix_entries[irq].irq_res, 1401 nic->msix_entries[irq].handle); 1402 KASSERT(err == 0, 1403 ("ERROR: Unable to teardown QS Error interrupt %d", 1404 irq)); 1405 if (err != 0) 1406 return; 1407 } 1408 1409 /* Release resource */ 1410 bus_release_resource(nic->dev, SYS_RES_IRQ, rman_get_rid(res), 1411 res); 1412 nic->msix_entries[irq].irq_res = NULL; 1413 } 1414 } 1415 1416 static int 1417 nicvf_allocate_net_interrupts(struct nicvf *nic) 1418 { 1419 u_int cpuid; 1420 int irq, rid; 1421 int qidx; 1422 int ret = 0; 1423 1424 /* MSI-X must be configured by now */ 1425 if (!nic->msix_enabled) { 1426 device_printf(nic->dev, "Cannot alloacte queue interrups. " 1427 "MSI-X interrupts disabled.\n"); 1428 return (ENXIO); 1429 } 1430 1431 /* Register CQ interrupts */ 1432 for_each_cq_irq(irq) { 1433 if (irq >= (NICVF_INTR_ID_CQ + nic->qs->cq_cnt)) 1434 break; 1435 1436 qidx = irq - NICVF_INTR_ID_CQ; 1437 rid = irq + 1; 1438 nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev, 1439 SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE)); 1440 if (nic->msix_entries[irq].irq_res == NULL) { 1441 device_printf(nic->dev, 1442 "Could not allocate CQ interrupt %d for VF%d\n", 1443 (irq - NICVF_INTR_ID_CQ), device_get_unit(nic->dev)); 1444 ret = ENXIO; 1445 goto error; 1446 } 1447 ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res, 1448 (INTR_MPSAFE | INTR_TYPE_NET), nicvf_intr_handler, 1449 NULL, &nic->qs->cq[qidx], &nic->msix_entries[irq].handle); 1450 if (ret != 0) { 1451 device_printf(nic->dev, 1452 "Could not setup CQ interrupt %d for VF%d\n", 1453 (irq - NICVF_INTR_ID_CQ), device_get_unit(nic->dev)); 1454 goto error; 1455 } 1456 cpuid = (device_get_unit(nic->dev) * CMP_QUEUE_CNT) + qidx; 1457 cpuid %= mp_ncpus; 1458 /* 1459 * Save CPU ID for later use when system-wide RSS is enabled. 1460 * It will be used to pit the CQ task to the same CPU that got 1461 * interrupted. 1462 */ 1463 nic->qs->cq[qidx].cmp_cpuid = cpuid; 1464 if (bootverbose) { 1465 device_printf(nic->dev, "bind CQ%d IRQ to CPU%d\n", 1466 qidx, cpuid); 1467 } 1468 /* Bind interrupts to the given CPU */ 1469 bus_bind_intr(nic->dev, nic->msix_entries[irq].irq_res, cpuid); 1470 } 1471 1472 /* Register RBDR interrupt */ 1473 for_each_rbdr_irq(irq) { 1474 if (irq >= (NICVF_INTR_ID_RBDR + nic->qs->rbdr_cnt)) 1475 break; 1476 1477 rid = irq + 1; 1478 nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev, 1479 SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE)); 1480 if (nic->msix_entries[irq].irq_res == NULL) { 1481 device_printf(nic->dev, 1482 "Could not allocate RBDR interrupt %d for VF%d\n", 1483 (irq - NICVF_INTR_ID_RBDR), 1484 device_get_unit(nic->dev)); 1485 ret = ENXIO; 1486 goto error; 1487 } 1488 ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res, 1489 (INTR_MPSAFE | INTR_TYPE_NET), NULL, 1490 nicvf_rbdr_intr_handler, nic, 1491 &nic->msix_entries[irq].handle); 1492 if (ret != 0) { 1493 device_printf(nic->dev, 1494 "Could not setup RBDR interrupt %d for VF%d\n", 1495 (irq - NICVF_INTR_ID_RBDR), 1496 device_get_unit(nic->dev)); 1497 goto error; 1498 } 1499 } 1500 1501 /* Register QS error interrupt */ 1502 irq = NICVF_INTR_ID_QS_ERR; 1503 rid = irq + 1; 1504 nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev, 1505 SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE)); 1506 if (nic->msix_entries[irq].irq_res == NULL) { 1507 device_printf(nic->dev, 1508 "Could not allocate QS Error interrupt for VF%d\n", 1509 device_get_unit(nic->dev)); 1510 ret = ENXIO; 1511 goto error; 1512 } 1513 ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res, 1514 (INTR_MPSAFE | INTR_TYPE_NET), NULL, nicvf_qs_err_intr_handler, 1515 nic, &nic->msix_entries[irq].handle); 1516 if (ret != 0) { 1517 device_printf(nic->dev, 1518 "Could not setup QS Error interrupt for VF%d\n", 1519 device_get_unit(nic->dev)); 1520 goto error; 1521 } 1522 1523 return (0); 1524 error: 1525 nicvf_release_net_interrupts(nic); 1526 return (ret); 1527 } 1528 1529 static int 1530 nicvf_stop_locked(struct nicvf *nic) 1531 { 1532 struct ifnet *ifp; 1533 int qidx; 1534 struct queue_set *qs = nic->qs; 1535 union nic_mbx mbx = {}; 1536 1537 NICVF_CORE_LOCK_ASSERT(nic); 1538 /* Stop callout. Can block here since holding SX lock */ 1539 callout_drain(&nic->stats_callout); 1540 1541 ifp = nic->ifp; 1542 1543 mbx.msg.msg = NIC_MBOX_MSG_SHUTDOWN; 1544 nicvf_send_msg_to_pf(nic, &mbx); 1545 1546 /* Disable RBDR & QS error interrupts */ 1547 for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) { 1548 nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx); 1549 nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx); 1550 } 1551 nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0); 1552 nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0); 1553 1554 /* Deactivate network interface */ 1555 if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); 1556 1557 /* Free resources */ 1558 nicvf_config_data_transfer(nic, FALSE); 1559 1560 /* Disable HW Qset */ 1561 nicvf_qset_config(nic, FALSE); 1562 1563 /* disable mailbox interrupt */ 1564 nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0); 1565 1566 return (0); 1567 } 1568 1569 static void 1570 nicvf_update_stats(struct nicvf *nic) 1571 { 1572 int qidx; 1573 struct nicvf_hw_stats *stats = &nic->hw_stats; 1574 struct nicvf_drv_stats *drv_stats = &nic->drv_stats; 1575 struct queue_set *qs = nic->qs; 1576 1577 #define GET_RX_STATS(reg) \ 1578 nicvf_reg_read(nic, NIC_VNIC_RX_STAT_0_13 | ((reg) << 3)) 1579 #define GET_TX_STATS(reg) \ 1580 nicvf_reg_read(nic, NIC_VNIC_TX_STAT_0_4 | ((reg) << 3)) 1581 1582 stats->rx_bytes = GET_RX_STATS(RX_OCTS); 1583 stats->rx_ucast_frames = GET_RX_STATS(RX_UCAST); 1584 stats->rx_bcast_frames = GET_RX_STATS(RX_BCAST); 1585 stats->rx_mcast_frames = GET_RX_STATS(RX_MCAST); 1586 stats->rx_fcs_errors = GET_RX_STATS(RX_FCS); 1587 stats->rx_l2_errors = GET_RX_STATS(RX_L2ERR); 1588 stats->rx_drop_red = GET_RX_STATS(RX_RED); 1589 stats->rx_drop_red_bytes = GET_RX_STATS(RX_RED_OCTS); 1590 stats->rx_drop_overrun = GET_RX_STATS(RX_ORUN); 1591 stats->rx_drop_overrun_bytes = GET_RX_STATS(RX_ORUN_OCTS); 1592 stats->rx_drop_bcast = GET_RX_STATS(RX_DRP_BCAST); 1593 stats->rx_drop_mcast = GET_RX_STATS(RX_DRP_MCAST); 1594 stats->rx_drop_l3_bcast = GET_RX_STATS(RX_DRP_L3BCAST); 1595 stats->rx_drop_l3_mcast = GET_RX_STATS(RX_DRP_L3MCAST); 1596 1597 stats->tx_bytes_ok = GET_TX_STATS(TX_OCTS); 1598 stats->tx_ucast_frames_ok = GET_TX_STATS(TX_UCAST); 1599 stats->tx_bcast_frames_ok = GET_TX_STATS(TX_BCAST); 1600 stats->tx_mcast_frames_ok = GET_TX_STATS(TX_MCAST); 1601 stats->tx_drops = GET_TX_STATS(TX_DROP); 1602 1603 drv_stats->tx_frames_ok = stats->tx_ucast_frames_ok + 1604 stats->tx_bcast_frames_ok + stats->tx_mcast_frames_ok; 1605 drv_stats->rx_drops = stats->rx_drop_red + stats->rx_drop_overrun; 1606 drv_stats->tx_drops = stats->tx_drops; 1607 1608 /* Update RQ and SQ stats */ 1609 for (qidx = 0; qidx < qs->rq_cnt; qidx++) 1610 nicvf_update_rq_stats(nic, qidx); 1611 for (qidx = 0; qidx < qs->sq_cnt; qidx++) 1612 nicvf_update_sq_stats(nic, qidx); 1613 } 1614 1615 static void 1616 nicvf_tick_stats(void *arg) 1617 { 1618 struct nicvf *nic; 1619 1620 nic = (struct nicvf *)arg; 1621 1622 /* Read the statistics */ 1623 nicvf_update_stats(nic); 1624 1625 callout_reset(&nic->stats_callout, hz, nicvf_tick_stats, nic); 1626 } 1627