1 /* 2 * Copyright (C) 2015 Cavium Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 * 28 */ 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include "opt_inet.h" 33 #include "opt_inet6.h" 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/bitset.h> 38 #include <sys/bitstring.h> 39 #include <sys/bus.h> 40 #include <sys/endian.h> 41 #include <sys/kernel.h> 42 #include <sys/malloc.h> 43 #include <sys/mbuf.h> 44 #include <sys/module.h> 45 #include <sys/rman.h> 46 #include <sys/pciio.h> 47 #include <sys/pcpu.h> 48 #include <sys/proc.h> 49 #include <sys/socket.h> 50 #include <sys/sockio.h> 51 #include <sys/stdatomic.h> 52 #include <sys/cpuset.h> 53 #include <sys/lock.h> 54 #include <sys/mutex.h> 55 #include <sys/smp.h> 56 #include <sys/taskqueue.h> 57 58 #include <net/bpf.h> 59 #include <net/ethernet.h> 60 #include <net/if.h> 61 #include <net/if_var.h> 62 #include <net/if_arp.h> 63 #include <net/if_dl.h> 64 #include <net/if_media.h> 65 #include <net/if_types.h> 66 #include <net/if_vlan_var.h> 67 68 #include <netinet/in.h> 69 #include <netinet/ip.h> 70 #include <netinet/if_ether.h> 71 #include <netinet/tcp_lro.h> 72 73 #include <dev/pci/pcireg.h> 74 #include <dev/pci/pcivar.h> 75 76 #include <sys/dnv.h> 77 #include <sys/nv.h> 78 #include <sys/iov_schema.h> 79 80 #include <machine/bus.h> 81 82 #include "thunder_bgx.h" 83 #include "nic_reg.h" 84 #include "nic.h" 85 #include "nicvf_queues.h" 86 87 #define VNIC_VF_DEVSTR "Cavium Thunder NIC Virtual Function Driver" 88 89 #define VNIC_VF_REG_RID PCIR_BAR(PCI_CFG_REG_BAR_NUM) 90 91 /* Lock for core interface settings */ 92 #define NICVF_CORE_LOCK_INIT(nic) \ 93 sx_init(&(nic)->core_sx, device_get_nameunit((nic)->dev)) 94 95 #define NICVF_CORE_LOCK_DESTROY(nic) \ 96 sx_destroy(&(nic)->core_sx) 97 98 #define NICVF_CORE_LOCK(nic) sx_xlock(&(nic)->core_sx) 99 #define NICVF_CORE_UNLOCK(nic) sx_xunlock(&(nic)->core_sx) 100 101 #define NICVF_CORE_LOCK_ASSERT(nic) sx_assert(&(nic)->core_sx, SA_XLOCKED) 102 103 #define SPEED_10 10 104 #define SPEED_100 100 105 #define SPEED_1000 1000 106 #define SPEED_10000 10000 107 #define SPEED_40000 40000 108 109 MALLOC_DEFINE(M_NICVF, "nicvf", "ThunderX VNIC VF dynamic memory"); 110 111 static int nicvf_probe(device_t); 112 static int nicvf_attach(device_t); 113 static int nicvf_detach(device_t); 114 115 static device_method_t nicvf_methods[] = { 116 /* Device interface */ 117 DEVMETHOD(device_probe, nicvf_probe), 118 DEVMETHOD(device_attach, nicvf_attach), 119 DEVMETHOD(device_detach, nicvf_detach), 120 121 DEVMETHOD_END, 122 }; 123 124 static driver_t nicvf_driver = { 125 "vnic", 126 nicvf_methods, 127 sizeof(struct nicvf), 128 }; 129 130 static devclass_t nicvf_devclass; 131 132 DRIVER_MODULE(vnicvf, pci, nicvf_driver, nicvf_devclass, 0, 0); 133 MODULE_VERSION(vnicvf, 1); 134 MODULE_DEPEND(vnicvf, pci, 1, 1, 1); 135 MODULE_DEPEND(vnicvf, ether, 1, 1, 1); 136 MODULE_DEPEND(vnicvf, vnicpf, 1, 1, 1); 137 138 static int nicvf_allocate_misc_interrupt(struct nicvf *); 139 static int nicvf_enable_misc_interrupt(struct nicvf *); 140 static int nicvf_allocate_net_interrupts(struct nicvf *); 141 static void nicvf_release_all_interrupts(struct nicvf *); 142 static int nicvf_update_hw_max_frs(struct nicvf *, int); 143 static int nicvf_hw_set_mac_addr(struct nicvf *, uint8_t *); 144 static void nicvf_config_cpi(struct nicvf *); 145 static int nicvf_rss_init(struct nicvf *); 146 static int nicvf_init_resources(struct nicvf *); 147 148 static int nicvf_setup_ifnet(struct nicvf *); 149 static int nicvf_setup_ifmedia(struct nicvf *); 150 static void nicvf_hw_addr_random(uint8_t *); 151 152 static int nicvf_if_ioctl(struct ifnet *, u_long, caddr_t); 153 static void nicvf_if_init(void *); 154 static void nicvf_if_init_locked(struct nicvf *); 155 static int nicvf_if_transmit(struct ifnet *, struct mbuf *); 156 static void nicvf_if_qflush(struct ifnet *); 157 static uint64_t nicvf_if_getcounter(struct ifnet *, ift_counter); 158 159 static int nicvf_stop_locked(struct nicvf *); 160 161 static void nicvf_media_status(struct ifnet *, struct ifmediareq *); 162 static int nicvf_media_change(struct ifnet *); 163 164 static void nicvf_tick_stats(void *); 165 166 static int 167 nicvf_probe(device_t dev) 168 { 169 uint16_t vendor_id; 170 uint16_t device_id; 171 172 vendor_id = pci_get_vendor(dev); 173 device_id = pci_get_device(dev); 174 175 if (vendor_id != PCI_VENDOR_ID_CAVIUM) 176 return (ENXIO); 177 178 if (device_id == PCI_DEVICE_ID_THUNDER_NIC_VF || 179 device_id == PCI_DEVICE_ID_THUNDER_PASS1_NIC_VF) { 180 device_set_desc(dev, VNIC_VF_DEVSTR); 181 return (BUS_PROBE_DEFAULT); 182 } 183 184 return (ENXIO); 185 } 186 187 static int 188 nicvf_attach(device_t dev) 189 { 190 struct nicvf *nic; 191 int rid, qcount; 192 int err = 0; 193 uint8_t hwaddr[ETHER_ADDR_LEN]; 194 uint8_t zeromac[] = {[0 ... (ETHER_ADDR_LEN - 1)] = 0}; 195 196 nic = device_get_softc(dev); 197 nic->dev = dev; 198 nic->pnicvf = nic; 199 200 NICVF_CORE_LOCK_INIT(nic); 201 /* Enable HW TSO on Pass2 */ 202 if (!pass1_silicon(dev)) 203 nic->hw_tso = TRUE; 204 205 rid = VNIC_VF_REG_RID; 206 nic->reg_base = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 207 RF_ACTIVE); 208 if (nic->reg_base == NULL) { 209 device_printf(dev, "Could not allocate registers memory\n"); 210 return (ENXIO); 211 } 212 213 qcount = MAX_CMP_QUEUES_PER_QS; 214 nic->max_queues = qcount; 215 216 err = nicvf_set_qset_resources(nic); 217 if (err != 0) 218 goto err_free_res; 219 220 /* Check if PF is alive and get MAC address for this VF */ 221 err = nicvf_allocate_misc_interrupt(nic); 222 if (err != 0) 223 goto err_free_res; 224 225 NICVF_CORE_LOCK(nic); 226 err = nicvf_enable_misc_interrupt(nic); 227 NICVF_CORE_UNLOCK(nic); 228 if (err != 0) 229 goto err_release_intr; 230 231 err = nicvf_allocate_net_interrupts(nic); 232 if (err != 0) { 233 device_printf(dev, 234 "Could not allocate network interface interrupts\n"); 235 goto err_free_ifnet; 236 } 237 238 /* If no MAC address was obtained we generate random one */ 239 if (memcmp(nic->hwaddr, zeromac, ETHER_ADDR_LEN) == 0) { 240 nicvf_hw_addr_random(hwaddr); 241 memcpy(nic->hwaddr, hwaddr, ETHER_ADDR_LEN); 242 NICVF_CORE_LOCK(nic); 243 nicvf_hw_set_mac_addr(nic, hwaddr); 244 NICVF_CORE_UNLOCK(nic); 245 } 246 247 /* Configure CPI alorithm */ 248 nic->cpi_alg = CPI_ALG_NONE; 249 NICVF_CORE_LOCK(nic); 250 nicvf_config_cpi(nic); 251 /* Configure receive side scaling */ 252 if (nic->qs->rq_cnt > 1) 253 nicvf_rss_init(nic); 254 NICVF_CORE_UNLOCK(nic); 255 256 err = nicvf_setup_ifnet(nic); 257 if (err != 0) { 258 device_printf(dev, "Could not set-up ifnet\n"); 259 goto err_release_intr; 260 } 261 262 err = nicvf_setup_ifmedia(nic); 263 if (err != 0) { 264 device_printf(dev, "Could not set-up ifmedia\n"); 265 goto err_free_ifnet; 266 } 267 268 mtx_init(&nic->stats_mtx, "VNIC stats", NULL, MTX_DEF); 269 callout_init_mtx(&nic->stats_callout, &nic->stats_mtx, 0); 270 271 ether_ifattach(nic->ifp, nic->hwaddr); 272 273 return (0); 274 275 err_free_ifnet: 276 if_free(nic->ifp); 277 err_release_intr: 278 nicvf_release_all_interrupts(nic); 279 err_free_res: 280 bus_release_resource(dev, SYS_RES_MEMORY, rman_get_rid(nic->reg_base), 281 nic->reg_base); 282 283 return (err); 284 } 285 286 static int 287 nicvf_detach(device_t dev) 288 { 289 struct nicvf *nic; 290 291 nic = device_get_softc(dev); 292 293 NICVF_CORE_LOCK(nic); 294 /* Shut down the port and release ring resources */ 295 nicvf_stop_locked(nic); 296 /* Release stats lock */ 297 mtx_destroy(&nic->stats_mtx); 298 /* Release interrupts */ 299 nicvf_release_all_interrupts(nic); 300 /* Release memory resource */ 301 if (nic->reg_base != NULL) { 302 bus_release_resource(dev, SYS_RES_MEMORY, 303 rman_get_rid(nic->reg_base), nic->reg_base); 304 } 305 306 /* Remove all ifmedia configurations */ 307 ifmedia_removeall(&nic->if_media); 308 /* Free this ifnet */ 309 if_free(nic->ifp); 310 NICVF_CORE_UNLOCK(nic); 311 /* Finally destroy the lock */ 312 NICVF_CORE_LOCK_DESTROY(nic); 313 314 return (0); 315 } 316 317 static void 318 nicvf_hw_addr_random(uint8_t *hwaddr) 319 { 320 uint32_t rnd; 321 uint8_t addr[ETHER_ADDR_LEN]; 322 323 /* 324 * Create randomized MAC address. 325 * Set 'bsd' + random 24 low-order bits. 326 */ 327 rnd = arc4random() & 0x00ffffff; 328 addr[0] = 'b'; 329 addr[1] = 's'; 330 addr[2] = 'd'; 331 addr[3] = rnd >> 16; 332 addr[4] = rnd >> 8; 333 addr[5] = rnd >> 0; 334 335 memcpy(hwaddr, addr, ETHER_ADDR_LEN); 336 } 337 338 static int 339 nicvf_setup_ifnet(struct nicvf *nic) 340 { 341 struct ifnet *ifp; 342 343 ifp = if_alloc(IFT_ETHER); 344 if (ifp == NULL) { 345 device_printf(nic->dev, "Could not allocate ifnet structure\n"); 346 return (ENOMEM); 347 } 348 349 nic->ifp = ifp; 350 351 if_setsoftc(ifp, nic); 352 if_initname(ifp, device_get_name(nic->dev), device_get_unit(nic->dev)); 353 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX); 354 355 if_settransmitfn(ifp, nicvf_if_transmit); 356 if_setqflushfn(ifp, nicvf_if_qflush); 357 if_setioctlfn(ifp, nicvf_if_ioctl); 358 if_setinitfn(ifp, nicvf_if_init); 359 if_setgetcounterfn(ifp, nicvf_if_getcounter); 360 361 if_setmtu(ifp, ETHERMTU); 362 363 /* Reset caps */ 364 if_setcapabilities(ifp, 0); 365 366 /* Set the default values */ 367 if_setcapabilitiesbit(ifp, IFCAP_VLAN_MTU | IFCAP_JUMBO_MTU, 0); 368 if_setcapabilitiesbit(ifp, IFCAP_LRO, 0); 369 if (nic->hw_tso) { 370 /* TSO */ 371 if_setcapabilitiesbit(ifp, IFCAP_TSO4, 0); 372 /* TSO parameters */ 373 if_sethwtsomax(ifp, NICVF_TSO_MAXSIZE); 374 if_sethwtsomaxsegcount(ifp, NICVF_TSO_NSEGS); 375 if_sethwtsomaxsegsize(ifp, MCLBYTES); 376 } 377 /* IP/TCP/UDP HW checksums */ 378 if_setcapabilitiesbit(ifp, IFCAP_HWCSUM, 0); 379 if_setcapabilitiesbit(ifp, IFCAP_HWSTATS, 0); 380 /* 381 * HW offload enable 382 */ 383 if_clearhwassist(ifp); 384 if_sethwassistbits(ifp, (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_SCTP), 0); 385 if (nic->hw_tso) 386 if_sethwassistbits(ifp, (CSUM_TSO), 0); 387 if_setcapenable(ifp, if_getcapabilities(ifp)); 388 389 return (0); 390 } 391 392 static int 393 nicvf_setup_ifmedia(struct nicvf *nic) 394 { 395 396 ifmedia_init(&nic->if_media, IFM_IMASK, nicvf_media_change, 397 nicvf_media_status); 398 399 /* 400 * Advertise availability of all possible connection types, 401 * even though not all are possible at the same time. 402 */ 403 404 ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_10_T | IFM_FDX), 405 0, NULL); 406 ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_100_TX | IFM_FDX), 407 0, NULL); 408 ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_1000_T | IFM_FDX), 409 0, NULL); 410 ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_10G_SR | IFM_FDX), 411 0, NULL); 412 ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_40G_CR4 | IFM_FDX), 413 0, NULL); 414 ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_AUTO | IFM_FDX), 415 0, NULL); 416 417 ifmedia_set(&nic->if_media, (IFM_ETHER | IFM_AUTO | IFM_FDX)); 418 419 return (0); 420 } 421 422 static int 423 nicvf_if_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 424 { 425 struct nicvf *nic; 426 struct rcv_queue *rq; 427 struct ifreq *ifr; 428 uint32_t flags; 429 int mask, err; 430 int rq_idx; 431 #if defined(INET) || defined(INET6) 432 struct ifaddr *ifa; 433 boolean_t avoid_reset = FALSE; 434 #endif 435 436 nic = if_getsoftc(ifp); 437 ifr = (struct ifreq *)data; 438 #if defined(INET) || defined(INET6) 439 ifa = (struct ifaddr *)data; 440 #endif 441 err = 0; 442 switch (cmd) { 443 case SIOCSIFADDR: 444 #ifdef INET 445 if (ifa->ifa_addr->sa_family == AF_INET) 446 avoid_reset = TRUE; 447 #endif 448 #ifdef INET6 449 if (ifa->ifa_addr->sa_family == AF_INET6) 450 avoid_reset = TRUE; 451 #endif 452 453 #if defined(INET) || defined(INET6) 454 /* Avoid reinitialization unless it's necessary */ 455 if (avoid_reset) { 456 if_setflagbits(ifp, IFF_UP, 0); 457 if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) 458 nicvf_if_init(nic); 459 #ifdef INET 460 if (!(if_getflags(ifp) & IFF_NOARP)) 461 arp_ifinit(ifp, ifa); 462 #endif 463 464 return (0); 465 } 466 #endif 467 err = ether_ioctl(ifp, cmd, data); 468 break; 469 case SIOCSIFMTU: 470 if (ifr->ifr_mtu < NIC_HW_MIN_FRS || 471 ifr->ifr_mtu > NIC_HW_MAX_FRS) { 472 err = EINVAL; 473 } else { 474 NICVF_CORE_LOCK(nic); 475 err = nicvf_update_hw_max_frs(nic, ifr->ifr_mtu); 476 if (err == 0) 477 if_setmtu(ifp, ifr->ifr_mtu); 478 NICVF_CORE_UNLOCK(nic); 479 } 480 break; 481 case SIOCSIFFLAGS: 482 NICVF_CORE_LOCK(nic); 483 if (if_getflags(ifp) & IFF_UP) { 484 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 485 flags = if_getflags(ifp) ^ nic->if_flags; 486 if ((nic->if_flags & if_getflags(ifp)) & 487 IFF_PROMISC) { 488 /* Change promiscous mode */ 489 #if 0 490 /* ARM64TODO */ 491 nicvf_set_promiscous(nic); 492 #endif 493 } 494 495 if ((nic->if_flags ^ if_getflags(ifp)) & 496 IFF_ALLMULTI) { 497 /* Change multicasting settings */ 498 #if 0 499 /* ARM64TODO */ 500 nicvf_set_multicast(nic); 501 #endif 502 } 503 } else { 504 nicvf_if_init_locked(nic); 505 } 506 } else if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) 507 nicvf_stop_locked(nic); 508 509 nic->if_flags = if_getflags(ifp); 510 NICVF_CORE_UNLOCK(nic); 511 break; 512 513 case SIOCADDMULTI: 514 case SIOCDELMULTI: 515 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 516 #if 0 517 NICVF_CORE_LOCK(nic); 518 /* ARM64TODO */ 519 nicvf_set_multicast(nic); 520 NICVF_CORE_UNLOCK(nic); 521 #endif 522 } 523 break; 524 525 case SIOCSIFMEDIA: 526 case SIOCGIFMEDIA: 527 err = ifmedia_ioctl(ifp, ifr, &nic->if_media, cmd); 528 break; 529 530 case SIOCSIFCAP: 531 mask = if_getcapenable(ifp) ^ ifr->ifr_reqcap; 532 if (mask & IFCAP_VLAN_MTU) { 533 /* No work to do except acknowledge the change took. */ 534 if_togglecapenable(ifp, IFCAP_VLAN_MTU); 535 } 536 if (mask & IFCAP_TXCSUM) 537 if_togglecapenable(ifp, IFCAP_TXCSUM); 538 if (mask & IFCAP_RXCSUM) 539 if_togglecapenable(ifp, IFCAP_RXCSUM); 540 if ((mask & IFCAP_TSO4) && nic->hw_tso) 541 if_togglecapenable(ifp, IFCAP_TSO4); 542 if (mask & IFCAP_LRO) { 543 /* 544 * Lock the driver for a moment to avoid 545 * mismatch in per-queue settings. 546 */ 547 NICVF_CORE_LOCK(nic); 548 if_togglecapenable(ifp, IFCAP_LRO); 549 if ((if_getdrvflags(nic->ifp) & IFF_DRV_RUNNING) != 0) { 550 /* 551 * Now disable LRO for subsequent packets. 552 * Atomicity of this change is not necessary 553 * as we don't need precise toggle of this 554 * feature for all threads processing the 555 * completion queue. 556 */ 557 for (rq_idx = 0; 558 rq_idx < nic->qs->rq_cnt; rq_idx++) { 559 rq = &nic->qs->rq[rq_idx]; 560 rq->lro_enabled = !rq->lro_enabled; 561 } 562 } 563 NICVF_CORE_UNLOCK(nic); 564 } 565 566 break; 567 568 default: 569 err = ether_ioctl(ifp, cmd, data); 570 break; 571 } 572 573 return (err); 574 } 575 576 static void 577 nicvf_if_init_locked(struct nicvf *nic) 578 { 579 struct queue_set *qs = nic->qs; 580 struct ifnet *ifp; 581 int qidx; 582 int err; 583 caddr_t if_addr; 584 585 NICVF_CORE_LOCK_ASSERT(nic); 586 ifp = nic->ifp; 587 588 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) 589 nicvf_stop_locked(nic); 590 591 err = nicvf_enable_misc_interrupt(nic); 592 if (err != 0) { 593 if_printf(ifp, "Could not reenable Mbox interrupt\n"); 594 return; 595 } 596 597 /* Get the latest MAC address */ 598 if_addr = if_getlladdr(ifp); 599 /* Update MAC address if changed */ 600 if (memcmp(nic->hwaddr, if_addr, ETHER_ADDR_LEN) != 0) { 601 memcpy(nic->hwaddr, if_addr, ETHER_ADDR_LEN); 602 nicvf_hw_set_mac_addr(nic, if_addr); 603 } 604 605 /* Initialize the queues */ 606 err = nicvf_init_resources(nic); 607 if (err != 0) 608 goto error; 609 610 /* Make sure queue initialization is written */ 611 wmb(); 612 613 nicvf_reg_write(nic, NIC_VF_INT, ~0UL); 614 /* Enable Qset err interrupt */ 615 nicvf_enable_intr(nic, NICVF_INTR_QS_ERR, 0); 616 617 /* Enable completion queue interrupt */ 618 for (qidx = 0; qidx < qs->cq_cnt; qidx++) 619 nicvf_enable_intr(nic, NICVF_INTR_CQ, qidx); 620 621 /* Enable RBDR threshold interrupt */ 622 for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) 623 nicvf_enable_intr(nic, NICVF_INTR_RBDR, qidx); 624 625 nic->drv_stats.txq_stop = 0; 626 nic->drv_stats.txq_wake = 0; 627 628 /* Activate network interface */ 629 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); 630 631 /* Schedule callout to update stats */ 632 callout_reset(&nic->stats_callout, hz, nicvf_tick_stats, nic); 633 634 return; 635 636 error: 637 /* Something went very wrong. Disable this ifnet for good */ 638 if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); 639 } 640 641 static void 642 nicvf_if_init(void *if_softc) 643 { 644 struct nicvf *nic = if_softc; 645 646 NICVF_CORE_LOCK(nic); 647 nicvf_if_init_locked(nic); 648 NICVF_CORE_UNLOCK(nic); 649 } 650 651 static int 652 nicvf_if_transmit(struct ifnet *ifp, struct mbuf *mbuf) 653 { 654 struct nicvf *nic = if_getsoftc(ifp); 655 struct queue_set *qs = nic->qs; 656 struct snd_queue *sq; 657 struct mbuf *mtmp; 658 int qidx; 659 int err = 0; 660 661 662 if (__predict_false(qs == NULL)) { 663 panic("%s: missing queue set for %s", __func__, 664 device_get_nameunit(nic->dev)); 665 } 666 667 /* Select queue */ 668 if (M_HASHTYPE_GET(mbuf) != M_HASHTYPE_NONE) 669 qidx = mbuf->m_pkthdr.flowid % qs->sq_cnt; 670 else 671 qidx = curcpu % qs->sq_cnt; 672 673 sq = &qs->sq[qidx]; 674 675 if (mbuf->m_next != NULL && 676 (mbuf->m_pkthdr.csum_flags & 677 (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_SCTP)) != 0) { 678 if (M_WRITABLE(mbuf) == 0) { 679 mtmp = m_dup(mbuf, M_NOWAIT); 680 m_freem(mbuf); 681 if (mtmp == NULL) 682 return (ENOBUFS); 683 mbuf = mtmp; 684 } 685 } 686 687 err = drbr_enqueue(ifp, sq->br, mbuf); 688 if (((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 689 IFF_DRV_RUNNING) || !nic->link_up || (err != 0)) { 690 /* 691 * Try to enqueue packet to the ring buffer. 692 * If the driver is not active, link down or enqueue operation 693 * failed, return with the appropriate error code. 694 */ 695 return (err); 696 } 697 698 if (NICVF_TX_TRYLOCK(sq) != 0) { 699 err = nicvf_xmit_locked(sq); 700 NICVF_TX_UNLOCK(sq); 701 return (err); 702 } else 703 taskqueue_enqueue(sq->snd_taskq, &sq->snd_task); 704 705 return (0); 706 } 707 708 static void 709 nicvf_if_qflush(struct ifnet *ifp) 710 { 711 struct nicvf *nic; 712 struct queue_set *qs; 713 struct snd_queue *sq; 714 struct mbuf *mbuf; 715 size_t idx; 716 717 nic = if_getsoftc(ifp); 718 qs = nic->qs; 719 720 for (idx = 0; idx < qs->sq_cnt; idx++) { 721 sq = &qs->sq[idx]; 722 NICVF_TX_LOCK(sq); 723 while ((mbuf = buf_ring_dequeue_sc(sq->br)) != NULL) 724 m_freem(mbuf); 725 NICVF_TX_UNLOCK(sq); 726 } 727 if_qflush(ifp); 728 } 729 730 static uint64_t 731 nicvf_if_getcounter(struct ifnet *ifp, ift_counter cnt) 732 { 733 struct nicvf *nic; 734 struct nicvf_hw_stats *hw_stats; 735 struct nicvf_drv_stats *drv_stats; 736 737 nic = if_getsoftc(ifp); 738 hw_stats = &nic->hw_stats; 739 drv_stats = &nic->drv_stats; 740 741 switch (cnt) { 742 case IFCOUNTER_IPACKETS: 743 return (drv_stats->rx_frames_ok); 744 case IFCOUNTER_OPACKETS: 745 return (drv_stats->tx_frames_ok); 746 case IFCOUNTER_IBYTES: 747 return (hw_stats->rx_bytes); 748 case IFCOUNTER_OBYTES: 749 return (hw_stats->tx_bytes_ok); 750 case IFCOUNTER_IMCASTS: 751 return (hw_stats->rx_mcast_frames); 752 case IFCOUNTER_COLLISIONS: 753 return (0); 754 case IFCOUNTER_IQDROPS: 755 return (drv_stats->rx_drops); 756 case IFCOUNTER_OQDROPS: 757 return (drv_stats->tx_drops); 758 default: 759 return (if_get_counter_default(ifp, cnt)); 760 } 761 762 } 763 764 static void 765 nicvf_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 766 { 767 struct nicvf *nic = if_getsoftc(ifp); 768 769 NICVF_CORE_LOCK(nic); 770 771 ifmr->ifm_status = IFM_AVALID; 772 ifmr->ifm_active = IFM_ETHER; 773 774 if (nic->link_up) { 775 /* Device attached to working network */ 776 ifmr->ifm_status |= IFM_ACTIVE; 777 } 778 779 switch (nic->speed) { 780 case SPEED_10: 781 ifmr->ifm_active |= IFM_10_T; 782 break; 783 case SPEED_100: 784 ifmr->ifm_active |= IFM_100_TX; 785 break; 786 case SPEED_1000: 787 ifmr->ifm_active |= IFM_1000_T; 788 break; 789 case SPEED_10000: 790 ifmr->ifm_active |= IFM_10G_SR; 791 break; 792 case SPEED_40000: 793 ifmr->ifm_active |= IFM_40G_CR4; 794 break; 795 default: 796 ifmr->ifm_active |= IFM_AUTO; 797 break; 798 } 799 800 if (nic->duplex) 801 ifmr->ifm_active |= IFM_FDX; 802 else 803 ifmr->ifm_active |= IFM_HDX; 804 805 NICVF_CORE_UNLOCK(nic); 806 } 807 808 static int 809 nicvf_media_change(struct ifnet *ifp __unused) 810 { 811 812 return (0); 813 } 814 815 /* Register read/write APIs */ 816 void 817 nicvf_reg_write(struct nicvf *nic, bus_space_handle_t offset, uint64_t val) 818 { 819 820 bus_write_8(nic->reg_base, offset, val); 821 } 822 823 uint64_t 824 nicvf_reg_read(struct nicvf *nic, uint64_t offset) 825 { 826 827 return (bus_read_8(nic->reg_base, offset)); 828 } 829 830 void 831 nicvf_queue_reg_write(struct nicvf *nic, bus_space_handle_t offset, 832 uint64_t qidx, uint64_t val) 833 { 834 835 bus_write_8(nic->reg_base, offset + (qidx << NIC_Q_NUM_SHIFT), val); 836 } 837 838 uint64_t 839 nicvf_queue_reg_read(struct nicvf *nic, bus_space_handle_t offset, 840 uint64_t qidx) 841 { 842 843 return (bus_read_8(nic->reg_base, offset + (qidx << NIC_Q_NUM_SHIFT))); 844 } 845 846 /* VF -> PF mailbox communication */ 847 static void 848 nicvf_write_to_mbx(struct nicvf *nic, union nic_mbx *mbx) 849 { 850 uint64_t *msg = (uint64_t *)mbx; 851 852 nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 0, msg[0]); 853 nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 8, msg[1]); 854 } 855 856 int 857 nicvf_send_msg_to_pf(struct nicvf *nic, union nic_mbx *mbx) 858 { 859 int timeout = NIC_MBOX_MSG_TIMEOUT * 10; 860 int sleep = 2; 861 862 NICVF_CORE_LOCK_ASSERT(nic); 863 864 nic->pf_acked = FALSE; 865 nic->pf_nacked = FALSE; 866 867 nicvf_write_to_mbx(nic, mbx); 868 869 /* Wait for previous message to be acked, timeout 2sec */ 870 while (!nic->pf_acked) { 871 if (nic->pf_nacked) 872 return (EINVAL); 873 874 DELAY(sleep * 1000); 875 876 if (nic->pf_acked) 877 break; 878 timeout -= sleep; 879 if (!timeout) { 880 device_printf(nic->dev, 881 "PF didn't ack to mbox msg %d from VF%d\n", 882 (mbx->msg.msg & 0xFF), nic->vf_id); 883 884 return (EBUSY); 885 } 886 } 887 return (0); 888 } 889 890 /* 891 * Checks if VF is able to comminicate with PF 892 * and also gets the VNIC number this VF is associated to. 893 */ 894 static int 895 nicvf_check_pf_ready(struct nicvf *nic) 896 { 897 union nic_mbx mbx = {}; 898 899 mbx.msg.msg = NIC_MBOX_MSG_READY; 900 if (nicvf_send_msg_to_pf(nic, &mbx)) { 901 device_printf(nic->dev, 902 "PF didn't respond to READY msg\n"); 903 return 0; 904 } 905 906 return 1; 907 } 908 909 static void 910 nicvf_read_bgx_stats(struct nicvf *nic, struct bgx_stats_msg *bgx) 911 { 912 913 if (bgx->rx) 914 nic->bgx_stats.rx_stats[bgx->idx] = bgx->stats; 915 else 916 nic->bgx_stats.tx_stats[bgx->idx] = bgx->stats; 917 } 918 919 static void 920 nicvf_handle_mbx_intr(struct nicvf *nic) 921 { 922 union nic_mbx mbx = {}; 923 uint64_t *mbx_data; 924 uint64_t mbx_addr; 925 int i; 926 927 mbx_addr = NIC_VF_PF_MAILBOX_0_1; 928 mbx_data = (uint64_t *)&mbx; 929 930 for (i = 0; i < NIC_PF_VF_MAILBOX_SIZE; i++) { 931 *mbx_data = nicvf_reg_read(nic, mbx_addr); 932 mbx_data++; 933 mbx_addr += sizeof(uint64_t); 934 } 935 936 switch (mbx.msg.msg) { 937 case NIC_MBOX_MSG_READY: 938 nic->pf_acked = TRUE; 939 nic->vf_id = mbx.nic_cfg.vf_id & 0x7F; 940 nic->tns_mode = mbx.nic_cfg.tns_mode & 0x7F; 941 nic->node = mbx.nic_cfg.node_id; 942 memcpy(nic->hwaddr, mbx.nic_cfg.mac_addr, ETHER_ADDR_LEN); 943 nic->loopback_supported = mbx.nic_cfg.loopback_supported; 944 nic->link_up = FALSE; 945 nic->duplex = 0; 946 nic->speed = 0; 947 break; 948 case NIC_MBOX_MSG_ACK: 949 nic->pf_acked = TRUE; 950 break; 951 case NIC_MBOX_MSG_NACK: 952 nic->pf_nacked = TRUE; 953 break; 954 case NIC_MBOX_MSG_RSS_SIZE: 955 nic->rss_info.rss_size = mbx.rss_size.ind_tbl_size; 956 nic->pf_acked = TRUE; 957 break; 958 case NIC_MBOX_MSG_BGX_STATS: 959 nicvf_read_bgx_stats(nic, &mbx.bgx_stats); 960 nic->pf_acked = TRUE; 961 break; 962 case NIC_MBOX_MSG_BGX_LINK_CHANGE: 963 nic->pf_acked = TRUE; 964 nic->link_up = mbx.link_status.link_up; 965 nic->duplex = mbx.link_status.duplex; 966 nic->speed = mbx.link_status.speed; 967 if (nic->link_up) { 968 if_setbaudrate(nic->ifp, nic->speed * 1000000); 969 if_link_state_change(nic->ifp, LINK_STATE_UP); 970 } else { 971 if_setbaudrate(nic->ifp, 0); 972 if_link_state_change(nic->ifp, LINK_STATE_DOWN); 973 } 974 break; 975 default: 976 device_printf(nic->dev, 977 "Invalid message from PF, msg 0x%x\n", mbx.msg.msg); 978 break; 979 } 980 nicvf_clear_intr(nic, NICVF_INTR_MBOX, 0); 981 } 982 983 static int 984 nicvf_update_hw_max_frs(struct nicvf *nic, int mtu) 985 { 986 union nic_mbx mbx = {}; 987 988 mbx.frs.msg = NIC_MBOX_MSG_SET_MAX_FRS; 989 mbx.frs.max_frs = mtu; 990 mbx.frs.vf_id = nic->vf_id; 991 992 return nicvf_send_msg_to_pf(nic, &mbx); 993 } 994 995 static int 996 nicvf_hw_set_mac_addr(struct nicvf *nic, uint8_t *hwaddr) 997 { 998 union nic_mbx mbx = {}; 999 1000 mbx.mac.msg = NIC_MBOX_MSG_SET_MAC; 1001 mbx.mac.vf_id = nic->vf_id; 1002 memcpy(mbx.mac.mac_addr, hwaddr, ETHER_ADDR_LEN); 1003 1004 return (nicvf_send_msg_to_pf(nic, &mbx)); 1005 } 1006 1007 static void 1008 nicvf_config_cpi(struct nicvf *nic) 1009 { 1010 union nic_mbx mbx = {}; 1011 1012 mbx.cpi_cfg.msg = NIC_MBOX_MSG_CPI_CFG; 1013 mbx.cpi_cfg.vf_id = nic->vf_id; 1014 mbx.cpi_cfg.cpi_alg = nic->cpi_alg; 1015 mbx.cpi_cfg.rq_cnt = nic->qs->rq_cnt; 1016 1017 nicvf_send_msg_to_pf(nic, &mbx); 1018 } 1019 1020 static void 1021 nicvf_get_rss_size(struct nicvf *nic) 1022 { 1023 union nic_mbx mbx = {}; 1024 1025 mbx.rss_size.msg = NIC_MBOX_MSG_RSS_SIZE; 1026 mbx.rss_size.vf_id = nic->vf_id; 1027 nicvf_send_msg_to_pf(nic, &mbx); 1028 } 1029 1030 static void 1031 nicvf_config_rss(struct nicvf *nic) 1032 { 1033 union nic_mbx mbx = {}; 1034 struct nicvf_rss_info *rss; 1035 int ind_tbl_len; 1036 int i, nextq; 1037 1038 rss = &nic->rss_info; 1039 ind_tbl_len = rss->rss_size; 1040 nextq = 0; 1041 1042 mbx.rss_cfg.vf_id = nic->vf_id; 1043 mbx.rss_cfg.hash_bits = rss->hash_bits; 1044 while (ind_tbl_len != 0) { 1045 mbx.rss_cfg.tbl_offset = nextq; 1046 mbx.rss_cfg.tbl_len = MIN(ind_tbl_len, 1047 RSS_IND_TBL_LEN_PER_MBX_MSG); 1048 mbx.rss_cfg.msg = mbx.rss_cfg.tbl_offset ? 1049 NIC_MBOX_MSG_RSS_CFG_CONT : NIC_MBOX_MSG_RSS_CFG; 1050 1051 for (i = 0; i < mbx.rss_cfg.tbl_len; i++) 1052 mbx.rss_cfg.ind_tbl[i] = rss->ind_tbl[nextq++]; 1053 1054 nicvf_send_msg_to_pf(nic, &mbx); 1055 1056 ind_tbl_len -= mbx.rss_cfg.tbl_len; 1057 } 1058 } 1059 1060 static void 1061 nicvf_set_rss_key(struct nicvf *nic) 1062 { 1063 struct nicvf_rss_info *rss; 1064 uint64_t key_addr; 1065 int idx; 1066 1067 rss = &nic->rss_info; 1068 key_addr = NIC_VNIC_RSS_KEY_0_4; 1069 1070 for (idx = 0; idx < RSS_HASH_KEY_SIZE; idx++) { 1071 nicvf_reg_write(nic, key_addr, rss->key[idx]); 1072 key_addr += sizeof(uint64_t); 1073 } 1074 } 1075 1076 static int 1077 nicvf_rss_init(struct nicvf *nic) 1078 { 1079 struct nicvf_rss_info *rss; 1080 int idx; 1081 1082 nicvf_get_rss_size(nic); 1083 1084 rss = &nic->rss_info; 1085 if (nic->cpi_alg != CPI_ALG_NONE) { 1086 rss->enable = FALSE; 1087 rss->hash_bits = 0; 1088 return (ENXIO); 1089 } 1090 1091 rss->enable = TRUE; 1092 1093 /* Using the HW reset value for now */ 1094 rss->key[0] = 0xFEED0BADFEED0BADUL; 1095 rss->key[1] = 0xFEED0BADFEED0BADUL; 1096 rss->key[2] = 0xFEED0BADFEED0BADUL; 1097 rss->key[3] = 0xFEED0BADFEED0BADUL; 1098 rss->key[4] = 0xFEED0BADFEED0BADUL; 1099 1100 nicvf_set_rss_key(nic); 1101 1102 rss->cfg = RSS_IP_HASH_ENA | RSS_TCP_HASH_ENA | RSS_UDP_HASH_ENA; 1103 nicvf_reg_write(nic, NIC_VNIC_RSS_CFG, rss->cfg); 1104 1105 rss->hash_bits = fls(rss->rss_size) - 1; 1106 for (idx = 0; idx < rss->rss_size; idx++) 1107 rss->ind_tbl[idx] = idx % nic->rx_queues; 1108 1109 nicvf_config_rss(nic); 1110 1111 return (0); 1112 } 1113 1114 static int 1115 nicvf_init_resources(struct nicvf *nic) 1116 { 1117 int err; 1118 union nic_mbx mbx = {}; 1119 1120 mbx.msg.msg = NIC_MBOX_MSG_CFG_DONE; 1121 1122 /* Enable Qset */ 1123 nicvf_qset_config(nic, TRUE); 1124 1125 /* Initialize queues and HW for data transfer */ 1126 err = nicvf_config_data_transfer(nic, TRUE); 1127 if (err) { 1128 device_printf(nic->dev, 1129 "Failed to alloc/config VF's QSet resources\n"); 1130 return (err); 1131 } 1132 1133 /* Send VF config done msg to PF */ 1134 nicvf_write_to_mbx(nic, &mbx); 1135 1136 return (0); 1137 } 1138 1139 static void 1140 nicvf_misc_intr_handler(void *arg) 1141 { 1142 struct nicvf *nic = (struct nicvf *)arg; 1143 uint64_t intr; 1144 1145 intr = nicvf_reg_read(nic, NIC_VF_INT); 1146 /* Check for spurious interrupt */ 1147 if (!(intr & NICVF_INTR_MBOX_MASK)) 1148 return; 1149 1150 nicvf_handle_mbx_intr(nic); 1151 } 1152 1153 static int 1154 nicvf_intr_handler(void *arg) 1155 { 1156 struct nicvf *nic; 1157 struct cmp_queue *cq; 1158 int qidx; 1159 1160 cq = (struct cmp_queue *)arg; 1161 nic = cq->nic; 1162 qidx = cq->idx; 1163 1164 /* Disable interrupts */ 1165 nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx); 1166 1167 taskqueue_enqueue(cq->cmp_taskq, &cq->cmp_task); 1168 1169 /* Clear interrupt */ 1170 nicvf_clear_intr(nic, NICVF_INTR_CQ, qidx); 1171 1172 return (FILTER_HANDLED); 1173 } 1174 1175 static void 1176 nicvf_rbdr_intr_handler(void *arg) 1177 { 1178 struct nicvf *nic; 1179 struct queue_set *qs; 1180 struct rbdr *rbdr; 1181 int qidx; 1182 1183 nic = (struct nicvf *)arg; 1184 1185 /* Disable RBDR interrupt and schedule softirq */ 1186 for (qidx = 0; qidx < nic->qs->rbdr_cnt; qidx++) { 1187 if (!nicvf_is_intr_enabled(nic, NICVF_INTR_RBDR, qidx)) 1188 continue; 1189 nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx); 1190 1191 qs = nic->qs; 1192 rbdr = &qs->rbdr[qidx]; 1193 taskqueue_enqueue(rbdr->rbdr_taskq, &rbdr->rbdr_task_nowait); 1194 /* Clear interrupt */ 1195 nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx); 1196 } 1197 } 1198 1199 static void 1200 nicvf_qs_err_intr_handler(void *arg) 1201 { 1202 struct nicvf *nic = (struct nicvf *)arg; 1203 struct queue_set *qs = nic->qs; 1204 1205 /* Disable Qset err interrupt and schedule softirq */ 1206 nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0); 1207 taskqueue_enqueue(qs->qs_err_taskq, &qs->qs_err_task); 1208 nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0); 1209 1210 } 1211 1212 static int 1213 nicvf_enable_msix(struct nicvf *nic) 1214 { 1215 struct pci_devinfo *dinfo; 1216 int rid, count; 1217 int ret; 1218 1219 dinfo = device_get_ivars(nic->dev); 1220 rid = dinfo->cfg.msix.msix_table_bar; 1221 nic->msix_table_res = 1222 bus_alloc_resource_any(nic->dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); 1223 if (nic->msix_table_res == NULL) { 1224 device_printf(nic->dev, 1225 "Could not allocate memory for MSI-X table\n"); 1226 return (ENXIO); 1227 } 1228 1229 count = nic->num_vec = NIC_VF_MSIX_VECTORS; 1230 1231 ret = pci_alloc_msix(nic->dev, &count); 1232 if ((ret != 0) || (count != nic->num_vec)) { 1233 device_printf(nic->dev, 1234 "Request for #%d msix vectors failed, error: %d\n", 1235 nic->num_vec, ret); 1236 return (ret); 1237 } 1238 1239 nic->msix_enabled = 1; 1240 return (0); 1241 } 1242 1243 static void 1244 nicvf_disable_msix(struct nicvf *nic) 1245 { 1246 1247 if (nic->msix_enabled) { 1248 pci_release_msi(nic->dev); 1249 nic->msix_enabled = 0; 1250 nic->num_vec = 0; 1251 } 1252 } 1253 1254 static void 1255 nicvf_release_all_interrupts(struct nicvf *nic) 1256 { 1257 struct resource *res; 1258 int irq; 1259 int err; 1260 1261 /* Free registered interrupts */ 1262 for (irq = 0; irq < nic->num_vec; irq++) { 1263 res = nic->msix_entries[irq].irq_res; 1264 if (res == NULL) 1265 continue; 1266 /* Teardown interrupt first */ 1267 if (nic->msix_entries[irq].handle != NULL) { 1268 err = bus_teardown_intr(nic->dev, 1269 nic->msix_entries[irq].irq_res, 1270 nic->msix_entries[irq].handle); 1271 KASSERT(err == 0, 1272 ("ERROR: Unable to teardown interrupt %d", irq)); 1273 nic->msix_entries[irq].handle = NULL; 1274 } 1275 1276 bus_release_resource(nic->dev, SYS_RES_IRQ, 1277 rman_get_rid(res), nic->msix_entries[irq].irq_res); 1278 nic->msix_entries[irq].irq_res = NULL; 1279 } 1280 /* Disable MSI-X */ 1281 nicvf_disable_msix(nic); 1282 } 1283 1284 /* 1285 * Initialize MSIX vectors and register MISC interrupt. 1286 * Send READY message to PF to check if its alive 1287 */ 1288 static int 1289 nicvf_allocate_misc_interrupt(struct nicvf *nic) 1290 { 1291 struct resource *res; 1292 int irq, rid; 1293 int ret = 0; 1294 1295 /* Return if mailbox interrupt is already registered */ 1296 if (nic->msix_enabled) 1297 return (0); 1298 1299 /* Enable MSI-X */ 1300 if (nicvf_enable_msix(nic) != 0) 1301 return (ENXIO); 1302 1303 irq = NICVF_INTR_ID_MISC; 1304 rid = irq + 1; 1305 nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev, 1306 SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE)); 1307 if (nic->msix_entries[irq].irq_res == NULL) { 1308 device_printf(nic->dev, 1309 "Could not allocate Mbox interrupt for VF%d\n", 1310 device_get_unit(nic->dev)); 1311 return (ENXIO); 1312 } 1313 1314 ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res, 1315 (INTR_MPSAFE | INTR_TYPE_MISC), NULL, nicvf_misc_intr_handler, nic, 1316 &nic->msix_entries[irq].handle); 1317 if (ret != 0) { 1318 res = nic->msix_entries[irq].irq_res; 1319 bus_release_resource(nic->dev, SYS_RES_IRQ, 1320 rman_get_rid(res), res); 1321 nic->msix_entries[irq].irq_res = NULL; 1322 return (ret); 1323 } 1324 1325 return (0); 1326 } 1327 1328 static int 1329 nicvf_enable_misc_interrupt(struct nicvf *nic) 1330 { 1331 1332 /* Enable mailbox interrupt */ 1333 nicvf_enable_intr(nic, NICVF_INTR_MBOX, 0); 1334 1335 /* Check if VF is able to communicate with PF */ 1336 if (!nicvf_check_pf_ready(nic)) { 1337 nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0); 1338 return (ENXIO); 1339 } 1340 1341 return (0); 1342 } 1343 1344 static void 1345 nicvf_release_net_interrupts(struct nicvf *nic) 1346 { 1347 struct resource *res; 1348 int irq; 1349 int err; 1350 1351 for_each_cq_irq(irq) { 1352 res = nic->msix_entries[irq].irq_res; 1353 if (res == NULL) 1354 continue; 1355 /* Teardown active interrupts first */ 1356 if (nic->msix_entries[irq].handle != NULL) { 1357 err = bus_teardown_intr(nic->dev, 1358 nic->msix_entries[irq].irq_res, 1359 nic->msix_entries[irq].handle); 1360 KASSERT(err == 0, 1361 ("ERROR: Unable to teardown CQ interrupt %d", 1362 (irq - NICVF_INTR_ID_CQ))); 1363 if (err != 0) 1364 continue; 1365 } 1366 1367 /* Release resource */ 1368 bus_release_resource(nic->dev, SYS_RES_IRQ, rman_get_rid(res), 1369 res); 1370 nic->msix_entries[irq].irq_res = NULL; 1371 } 1372 1373 for_each_rbdr_irq(irq) { 1374 res = nic->msix_entries[irq].irq_res; 1375 if (res == NULL) 1376 continue; 1377 /* Teardown active interrupts first */ 1378 if (nic->msix_entries[irq].handle != NULL) { 1379 err = bus_teardown_intr(nic->dev, 1380 nic->msix_entries[irq].irq_res, 1381 nic->msix_entries[irq].handle); 1382 KASSERT(err == 0, 1383 ("ERROR: Unable to teardown RDBR interrupt %d", 1384 (irq - NICVF_INTR_ID_RBDR))); 1385 if (err != 0) 1386 continue; 1387 } 1388 1389 /* Release resource */ 1390 bus_release_resource(nic->dev, SYS_RES_IRQ, rman_get_rid(res), 1391 res); 1392 nic->msix_entries[irq].irq_res = NULL; 1393 } 1394 1395 irq = NICVF_INTR_ID_QS_ERR; 1396 res = nic->msix_entries[irq].irq_res; 1397 if (res != NULL) { 1398 /* Teardown active interrupts first */ 1399 if (nic->msix_entries[irq].handle != NULL) { 1400 err = bus_teardown_intr(nic->dev, 1401 nic->msix_entries[irq].irq_res, 1402 nic->msix_entries[irq].handle); 1403 KASSERT(err == 0, 1404 ("ERROR: Unable to teardown QS Error interrupt %d", 1405 irq)); 1406 if (err != 0) 1407 return; 1408 } 1409 1410 /* Release resource */ 1411 bus_release_resource(nic->dev, SYS_RES_IRQ, rman_get_rid(res), 1412 res); 1413 nic->msix_entries[irq].irq_res = NULL; 1414 } 1415 } 1416 1417 static int 1418 nicvf_allocate_net_interrupts(struct nicvf *nic) 1419 { 1420 u_int cpuid; 1421 int irq, rid; 1422 int qidx; 1423 int ret = 0; 1424 1425 /* MSI-X must be configured by now */ 1426 if (!nic->msix_enabled) { 1427 device_printf(nic->dev, "Cannot alloacte queue interrups. " 1428 "MSI-X interrupts disabled.\n"); 1429 return (ENXIO); 1430 } 1431 1432 /* Register CQ interrupts */ 1433 for_each_cq_irq(irq) { 1434 if (irq >= (NICVF_INTR_ID_CQ + nic->qs->cq_cnt)) 1435 break; 1436 1437 qidx = irq - NICVF_INTR_ID_CQ; 1438 rid = irq + 1; 1439 nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev, 1440 SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE)); 1441 if (nic->msix_entries[irq].irq_res == NULL) { 1442 device_printf(nic->dev, 1443 "Could not allocate CQ interrupt %d for VF%d\n", 1444 (irq - NICVF_INTR_ID_CQ), device_get_unit(nic->dev)); 1445 ret = ENXIO; 1446 goto error; 1447 } 1448 ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res, 1449 (INTR_MPSAFE | INTR_TYPE_NET), nicvf_intr_handler, 1450 NULL, &nic->qs->cq[qidx], &nic->msix_entries[irq].handle); 1451 if (ret != 0) { 1452 device_printf(nic->dev, 1453 "Could not setup CQ interrupt %d for VF%d\n", 1454 (irq - NICVF_INTR_ID_CQ), device_get_unit(nic->dev)); 1455 goto error; 1456 } 1457 cpuid = (device_get_unit(nic->dev) * CMP_QUEUE_CNT) + qidx; 1458 cpuid %= mp_ncpus; 1459 /* 1460 * Save CPU ID for later use when system-wide RSS is enabled. 1461 * It will be used to pit the CQ task to the same CPU that got 1462 * interrupted. 1463 */ 1464 nic->qs->cq[qidx].cmp_cpuid = cpuid; 1465 if (bootverbose) { 1466 device_printf(nic->dev, "bind CQ%d IRQ to CPU%d\n", 1467 qidx, cpuid); 1468 } 1469 /* Bind interrupts to the given CPU */ 1470 bus_bind_intr(nic->dev, nic->msix_entries[irq].irq_res, cpuid); 1471 } 1472 1473 /* Register RBDR interrupt */ 1474 for_each_rbdr_irq(irq) { 1475 if (irq >= (NICVF_INTR_ID_RBDR + nic->qs->rbdr_cnt)) 1476 break; 1477 1478 rid = irq + 1; 1479 nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev, 1480 SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE)); 1481 if (nic->msix_entries[irq].irq_res == NULL) { 1482 device_printf(nic->dev, 1483 "Could not allocate RBDR interrupt %d for VF%d\n", 1484 (irq - NICVF_INTR_ID_RBDR), 1485 device_get_unit(nic->dev)); 1486 ret = ENXIO; 1487 goto error; 1488 } 1489 ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res, 1490 (INTR_MPSAFE | INTR_TYPE_NET), NULL, 1491 nicvf_rbdr_intr_handler, nic, 1492 &nic->msix_entries[irq].handle); 1493 if (ret != 0) { 1494 device_printf(nic->dev, 1495 "Could not setup RBDR interrupt %d for VF%d\n", 1496 (irq - NICVF_INTR_ID_RBDR), 1497 device_get_unit(nic->dev)); 1498 goto error; 1499 } 1500 } 1501 1502 /* Register QS error interrupt */ 1503 irq = NICVF_INTR_ID_QS_ERR; 1504 rid = irq + 1; 1505 nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev, 1506 SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE)); 1507 if (nic->msix_entries[irq].irq_res == NULL) { 1508 device_printf(nic->dev, 1509 "Could not allocate QS Error interrupt for VF%d\n", 1510 device_get_unit(nic->dev)); 1511 ret = ENXIO; 1512 goto error; 1513 } 1514 ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res, 1515 (INTR_MPSAFE | INTR_TYPE_NET), NULL, nicvf_qs_err_intr_handler, 1516 nic, &nic->msix_entries[irq].handle); 1517 if (ret != 0) { 1518 device_printf(nic->dev, 1519 "Could not setup QS Error interrupt for VF%d\n", 1520 device_get_unit(nic->dev)); 1521 goto error; 1522 } 1523 1524 return (0); 1525 error: 1526 nicvf_release_net_interrupts(nic); 1527 return (ret); 1528 } 1529 1530 static int 1531 nicvf_stop_locked(struct nicvf *nic) 1532 { 1533 struct ifnet *ifp; 1534 int qidx; 1535 struct queue_set *qs = nic->qs; 1536 union nic_mbx mbx = {}; 1537 1538 NICVF_CORE_LOCK_ASSERT(nic); 1539 /* Stop callout. Can block here since holding SX lock */ 1540 callout_drain(&nic->stats_callout); 1541 1542 ifp = nic->ifp; 1543 1544 mbx.msg.msg = NIC_MBOX_MSG_SHUTDOWN; 1545 nicvf_send_msg_to_pf(nic, &mbx); 1546 1547 /* Disable RBDR & QS error interrupts */ 1548 for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) { 1549 nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx); 1550 nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx); 1551 } 1552 nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0); 1553 nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0); 1554 1555 /* Deactivate network interface */ 1556 if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); 1557 1558 /* Free resources */ 1559 nicvf_config_data_transfer(nic, FALSE); 1560 1561 /* Disable HW Qset */ 1562 nicvf_qset_config(nic, FALSE); 1563 1564 /* disable mailbox interrupt */ 1565 nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0); 1566 1567 return (0); 1568 } 1569 1570 static void 1571 nicvf_update_stats(struct nicvf *nic) 1572 { 1573 int qidx; 1574 struct nicvf_hw_stats *stats = &nic->hw_stats; 1575 struct nicvf_drv_stats *drv_stats = &nic->drv_stats; 1576 struct queue_set *qs = nic->qs; 1577 1578 #define GET_RX_STATS(reg) \ 1579 nicvf_reg_read(nic, NIC_VNIC_RX_STAT_0_13 | ((reg) << 3)) 1580 #define GET_TX_STATS(reg) \ 1581 nicvf_reg_read(nic, NIC_VNIC_TX_STAT_0_4 | ((reg) << 3)) 1582 1583 stats->rx_bytes = GET_RX_STATS(RX_OCTS); 1584 stats->rx_ucast_frames = GET_RX_STATS(RX_UCAST); 1585 stats->rx_bcast_frames = GET_RX_STATS(RX_BCAST); 1586 stats->rx_mcast_frames = GET_RX_STATS(RX_MCAST); 1587 stats->rx_fcs_errors = GET_RX_STATS(RX_FCS); 1588 stats->rx_l2_errors = GET_RX_STATS(RX_L2ERR); 1589 stats->rx_drop_red = GET_RX_STATS(RX_RED); 1590 stats->rx_drop_red_bytes = GET_RX_STATS(RX_RED_OCTS); 1591 stats->rx_drop_overrun = GET_RX_STATS(RX_ORUN); 1592 stats->rx_drop_overrun_bytes = GET_RX_STATS(RX_ORUN_OCTS); 1593 stats->rx_drop_bcast = GET_RX_STATS(RX_DRP_BCAST); 1594 stats->rx_drop_mcast = GET_RX_STATS(RX_DRP_MCAST); 1595 stats->rx_drop_l3_bcast = GET_RX_STATS(RX_DRP_L3BCAST); 1596 stats->rx_drop_l3_mcast = GET_RX_STATS(RX_DRP_L3MCAST); 1597 1598 stats->tx_bytes_ok = GET_TX_STATS(TX_OCTS); 1599 stats->tx_ucast_frames_ok = GET_TX_STATS(TX_UCAST); 1600 stats->tx_bcast_frames_ok = GET_TX_STATS(TX_BCAST); 1601 stats->tx_mcast_frames_ok = GET_TX_STATS(TX_MCAST); 1602 stats->tx_drops = GET_TX_STATS(TX_DROP); 1603 1604 drv_stats->tx_frames_ok = stats->tx_ucast_frames_ok + 1605 stats->tx_bcast_frames_ok + stats->tx_mcast_frames_ok; 1606 drv_stats->rx_drops = stats->rx_drop_red + stats->rx_drop_overrun; 1607 drv_stats->tx_drops = stats->tx_drops; 1608 1609 /* Update RQ and SQ stats */ 1610 for (qidx = 0; qidx < qs->rq_cnt; qidx++) 1611 nicvf_update_rq_stats(nic, qidx); 1612 for (qidx = 0; qidx < qs->sq_cnt; qidx++) 1613 nicvf_update_sq_stats(nic, qidx); 1614 } 1615 1616 static void 1617 nicvf_tick_stats(void *arg) 1618 { 1619 struct nicvf *nic; 1620 1621 nic = (struct nicvf *)arg; 1622 1623 /* Read the statistics */ 1624 nicvf_update_stats(nic); 1625 1626 callout_reset(&nic->stats_callout, hz, nicvf_tick_stats, nic); 1627 } 1628