1 /* 2 * Copyright (C) 2015 Cavium Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 */ 27 #include <sys/cdefs.h> 28 #include "opt_inet.h" 29 #include "opt_inet6.h" 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/bitset.h> 34 #include <sys/bitstring.h> 35 #include <sys/bus.h> 36 #include <sys/endian.h> 37 #include <sys/kernel.h> 38 #include <sys/malloc.h> 39 #include <sys/mbuf.h> 40 #include <sys/module.h> 41 #include <sys/rman.h> 42 #include <sys/pciio.h> 43 #include <sys/pcpu.h> 44 #include <sys/proc.h> 45 #include <sys/socket.h> 46 #include <sys/sockio.h> 47 #include <sys/stdatomic.h> 48 #include <sys/cpuset.h> 49 #include <sys/lock.h> 50 #include <sys/mutex.h> 51 #include <sys/smp.h> 52 #include <sys/taskqueue.h> 53 54 #include <net/bpf.h> 55 #include <net/ethernet.h> 56 #include <net/if.h> 57 #include <net/if_var.h> 58 #include <net/if_arp.h> 59 #include <net/if_dl.h> 60 #include <net/if_media.h> 61 #include <net/if_types.h> 62 #include <net/if_vlan_var.h> 63 64 #include <netinet/in.h> 65 #include <netinet/ip.h> 66 #include <netinet/if_ether.h> 67 #include <netinet/tcp_lro.h> 68 69 #include <dev/pci/pcireg.h> 70 #include <dev/pci/pcivar.h> 71 72 #include <sys/dnv.h> 73 #include <sys/nv.h> 74 #include <sys/iov_schema.h> 75 76 #include <machine/bus.h> 77 78 #include "thunder_bgx.h" 79 #include "nic_reg.h" 80 #include "nic.h" 81 #include "nicvf_queues.h" 82 83 #define VNIC_VF_DEVSTR "Cavium Thunder NIC Virtual Function Driver" 84 85 #define VNIC_VF_REG_RID PCIR_BAR(PCI_CFG_REG_BAR_NUM) 86 87 /* Lock for core interface settings */ 88 #define NICVF_CORE_LOCK_INIT(nic) \ 89 sx_init(&(nic)->core_sx, device_get_nameunit((nic)->dev)) 90 91 #define NICVF_CORE_LOCK_DESTROY(nic) \ 92 sx_destroy(&(nic)->core_sx) 93 94 #define NICVF_CORE_LOCK(nic) sx_xlock(&(nic)->core_sx) 95 #define NICVF_CORE_UNLOCK(nic) sx_xunlock(&(nic)->core_sx) 96 97 #define NICVF_CORE_LOCK_ASSERT(nic) sx_assert(&(nic)->core_sx, SA_XLOCKED) 98 99 #define SPEED_10 10 100 #define SPEED_100 100 101 #define SPEED_1000 1000 102 #define SPEED_10000 10000 103 #define SPEED_40000 40000 104 105 MALLOC_DEFINE(M_NICVF, "nicvf", "ThunderX VNIC VF dynamic memory"); 106 107 static int nicvf_probe(device_t); 108 static int nicvf_attach(device_t); 109 static int nicvf_detach(device_t); 110 111 static device_method_t nicvf_methods[] = { 112 /* Device interface */ 113 DEVMETHOD(device_probe, nicvf_probe), 114 DEVMETHOD(device_attach, nicvf_attach), 115 DEVMETHOD(device_detach, nicvf_detach), 116 117 DEVMETHOD_END, 118 }; 119 120 static driver_t nicvf_driver = { 121 "vnic", 122 nicvf_methods, 123 sizeof(struct nicvf), 124 }; 125 126 DRIVER_MODULE(vnicvf, pci, nicvf_driver, 0, 0); 127 MODULE_VERSION(vnicvf, 1); 128 MODULE_DEPEND(vnicvf, pci, 1, 1, 1); 129 MODULE_DEPEND(vnicvf, ether, 1, 1, 1); 130 MODULE_DEPEND(vnicvf, vnicpf, 1, 1, 1); 131 132 static int nicvf_allocate_misc_interrupt(struct nicvf *); 133 static int nicvf_enable_misc_interrupt(struct nicvf *); 134 static int nicvf_allocate_net_interrupts(struct nicvf *); 135 static void nicvf_release_all_interrupts(struct nicvf *); 136 static int nicvf_update_hw_max_frs(struct nicvf *, int); 137 static int nicvf_hw_set_mac_addr(struct nicvf *, uint8_t *); 138 static void nicvf_config_cpi(struct nicvf *); 139 static int nicvf_rss_init(struct nicvf *); 140 static int nicvf_init_resources(struct nicvf *); 141 142 static void nicvf_setup_ifnet(struct nicvf *); 143 static int nicvf_setup_ifmedia(struct nicvf *); 144 static void nicvf_hw_addr_random(uint8_t *); 145 146 static int nicvf_if_ioctl(if_t, u_long, caddr_t); 147 static void nicvf_if_init(void *); 148 static void nicvf_if_init_locked(struct nicvf *); 149 static int nicvf_if_transmit(if_t, struct mbuf *); 150 static void nicvf_if_qflush(if_t); 151 static uint64_t nicvf_if_getcounter(if_t, ift_counter); 152 153 static int nicvf_stop_locked(struct nicvf *); 154 155 static void nicvf_media_status(if_t, struct ifmediareq *); 156 static int nicvf_media_change(if_t); 157 158 static void nicvf_tick_stats(void *); 159 160 static int 161 nicvf_probe(device_t dev) 162 { 163 uint16_t vendor_id; 164 uint16_t device_id; 165 166 vendor_id = pci_get_vendor(dev); 167 device_id = pci_get_device(dev); 168 169 if (vendor_id != PCI_VENDOR_ID_CAVIUM) 170 return (ENXIO); 171 172 if (device_id == PCI_DEVICE_ID_THUNDER_NIC_VF || 173 device_id == PCI_DEVICE_ID_THUNDER_PASS1_NIC_VF) { 174 device_set_desc(dev, VNIC_VF_DEVSTR); 175 return (BUS_PROBE_DEFAULT); 176 } 177 178 return (ENXIO); 179 } 180 181 static int 182 nicvf_attach(device_t dev) 183 { 184 struct nicvf *nic; 185 int rid, qcount; 186 int err = 0; 187 uint8_t hwaddr[ETHER_ADDR_LEN]; 188 uint8_t zeromac[] = {[0 ... (ETHER_ADDR_LEN - 1)] = 0}; 189 190 nic = device_get_softc(dev); 191 nic->dev = dev; 192 nic->pnicvf = nic; 193 194 NICVF_CORE_LOCK_INIT(nic); 195 /* Enable HW TSO on Pass2 */ 196 if (!pass1_silicon(dev)) 197 nic->hw_tso = TRUE; 198 199 rid = VNIC_VF_REG_RID; 200 nic->reg_base = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 201 RF_ACTIVE); 202 if (nic->reg_base == NULL) { 203 device_printf(dev, "Could not allocate registers memory\n"); 204 return (ENXIO); 205 } 206 207 qcount = MAX_CMP_QUEUES_PER_QS; 208 nic->max_queues = qcount; 209 210 err = nicvf_set_qset_resources(nic); 211 if (err != 0) 212 goto err_free_res; 213 214 /* Check if PF is alive and get MAC address for this VF */ 215 err = nicvf_allocate_misc_interrupt(nic); 216 if (err != 0) 217 goto err_free_res; 218 219 NICVF_CORE_LOCK(nic); 220 err = nicvf_enable_misc_interrupt(nic); 221 NICVF_CORE_UNLOCK(nic); 222 if (err != 0) 223 goto err_release_intr; 224 225 err = nicvf_allocate_net_interrupts(nic); 226 if (err != 0) { 227 device_printf(dev, 228 "Could not allocate network interface interrupts\n"); 229 goto err_free_ifnet; 230 } 231 232 /* If no MAC address was obtained we generate random one */ 233 if (memcmp(nic->hwaddr, zeromac, ETHER_ADDR_LEN) == 0) { 234 nicvf_hw_addr_random(hwaddr); 235 memcpy(nic->hwaddr, hwaddr, ETHER_ADDR_LEN); 236 NICVF_CORE_LOCK(nic); 237 nicvf_hw_set_mac_addr(nic, hwaddr); 238 NICVF_CORE_UNLOCK(nic); 239 } 240 241 /* Configure CPI alorithm */ 242 nic->cpi_alg = CPI_ALG_NONE; 243 NICVF_CORE_LOCK(nic); 244 nicvf_config_cpi(nic); 245 /* Configure receive side scaling */ 246 if (nic->qs->rq_cnt > 1) 247 nicvf_rss_init(nic); 248 NICVF_CORE_UNLOCK(nic); 249 250 nicvf_setup_ifnet(nic); 251 252 err = nicvf_setup_ifmedia(nic); 253 if (err != 0) { 254 device_printf(dev, "Could not set-up ifmedia\n"); 255 goto err_free_ifnet; 256 } 257 258 mtx_init(&nic->stats_mtx, "VNIC stats", NULL, MTX_DEF); 259 callout_init_mtx(&nic->stats_callout, &nic->stats_mtx, 0); 260 261 ether_ifattach(nic->ifp, nic->hwaddr); 262 263 return (0); 264 265 err_free_ifnet: 266 if_free(nic->ifp); 267 err_release_intr: 268 nicvf_release_all_interrupts(nic); 269 err_free_res: 270 bus_release_resource(dev, SYS_RES_MEMORY, rman_get_rid(nic->reg_base), 271 nic->reg_base); 272 273 return (err); 274 } 275 276 static int 277 nicvf_detach(device_t dev) 278 { 279 struct nicvf *nic; 280 281 nic = device_get_softc(dev); 282 283 NICVF_CORE_LOCK(nic); 284 /* Shut down the port and release ring resources */ 285 nicvf_stop_locked(nic); 286 /* Release stats lock */ 287 mtx_destroy(&nic->stats_mtx); 288 /* Release interrupts */ 289 nicvf_release_all_interrupts(nic); 290 /* Release memory resource */ 291 if (nic->reg_base != NULL) { 292 bus_release_resource(dev, SYS_RES_MEMORY, 293 rman_get_rid(nic->reg_base), nic->reg_base); 294 } 295 296 /* Remove all ifmedia configurations */ 297 ifmedia_removeall(&nic->if_media); 298 /* Free this ifnet */ 299 if_free(nic->ifp); 300 NICVF_CORE_UNLOCK(nic); 301 /* Finally destroy the lock */ 302 NICVF_CORE_LOCK_DESTROY(nic); 303 304 return (0); 305 } 306 307 static void 308 nicvf_hw_addr_random(uint8_t *hwaddr) 309 { 310 uint32_t rnd; 311 uint8_t addr[ETHER_ADDR_LEN]; 312 313 /* 314 * Create randomized MAC address. 315 * Set 'bsd' + random 24 low-order bits. 316 */ 317 rnd = arc4random() & 0x00ffffff; 318 addr[0] = 'b'; 319 addr[1] = 's'; 320 addr[2] = 'd'; 321 addr[3] = rnd >> 16; 322 addr[4] = rnd >> 8; 323 addr[5] = rnd >> 0; 324 325 memcpy(hwaddr, addr, ETHER_ADDR_LEN); 326 } 327 328 static void 329 nicvf_setup_ifnet(struct nicvf *nic) 330 { 331 if_t ifp; 332 333 ifp = if_alloc(IFT_ETHER); 334 nic->ifp = ifp; 335 336 if_setsoftc(ifp, nic); 337 if_initname(ifp, device_get_name(nic->dev), device_get_unit(nic->dev)); 338 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); 339 340 if_settransmitfn(ifp, nicvf_if_transmit); 341 if_setqflushfn(ifp, nicvf_if_qflush); 342 if_setioctlfn(ifp, nicvf_if_ioctl); 343 if_setinitfn(ifp, nicvf_if_init); 344 if_setgetcounterfn(ifp, nicvf_if_getcounter); 345 346 if_setmtu(ifp, ETHERMTU); 347 348 /* Reset caps */ 349 if_setcapabilities(ifp, 0); 350 351 /* Set the default values */ 352 if_setcapabilitiesbit(ifp, IFCAP_VLAN_MTU | IFCAP_JUMBO_MTU, 0); 353 if_setcapabilitiesbit(ifp, IFCAP_LRO, 0); 354 if (nic->hw_tso) { 355 /* TSO */ 356 if_setcapabilitiesbit(ifp, IFCAP_TSO4, 0); 357 /* TSO parameters */ 358 if_sethwtsomax(ifp, NICVF_TSO_MAXSIZE); 359 if_sethwtsomaxsegcount(ifp, NICVF_TSO_NSEGS); 360 if_sethwtsomaxsegsize(ifp, MCLBYTES); 361 } 362 /* IP/TCP/UDP HW checksums */ 363 if_setcapabilitiesbit(ifp, IFCAP_HWCSUM, 0); 364 if_setcapabilitiesbit(ifp, IFCAP_HWSTATS, 0); 365 /* 366 * HW offload enable 367 */ 368 if_clearhwassist(ifp); 369 if_sethwassistbits(ifp, (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_SCTP), 0); 370 if (nic->hw_tso) 371 if_sethwassistbits(ifp, (CSUM_TSO), 0); 372 if_setcapenable(ifp, if_getcapabilities(ifp)); 373 } 374 375 static int 376 nicvf_setup_ifmedia(struct nicvf *nic) 377 { 378 379 ifmedia_init(&nic->if_media, IFM_IMASK, nicvf_media_change, 380 nicvf_media_status); 381 382 /* 383 * Advertise availability of all possible connection types, 384 * even though not all are possible at the same time. 385 */ 386 387 ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_10_T | IFM_FDX), 388 0, NULL); 389 ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_100_TX | IFM_FDX), 390 0, NULL); 391 ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_1000_T | IFM_FDX), 392 0, NULL); 393 ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_10G_SR | IFM_FDX), 394 0, NULL); 395 ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_40G_CR4 | IFM_FDX), 396 0, NULL); 397 ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_AUTO | IFM_FDX), 398 0, NULL); 399 400 ifmedia_set(&nic->if_media, (IFM_ETHER | IFM_AUTO | IFM_FDX)); 401 402 return (0); 403 } 404 405 static int 406 nicvf_if_ioctl(if_t ifp, u_long cmd, caddr_t data) 407 { 408 struct nicvf *nic; 409 struct rcv_queue *rq; 410 struct ifreq *ifr; 411 uint32_t flags; 412 int mask, err; 413 int rq_idx; 414 #if defined(INET) || defined(INET6) 415 struct ifaddr *ifa; 416 boolean_t avoid_reset = FALSE; 417 #endif 418 419 nic = if_getsoftc(ifp); 420 ifr = (struct ifreq *)data; 421 #if defined(INET) || defined(INET6) 422 ifa = (struct ifaddr *)data; 423 #endif 424 err = 0; 425 switch (cmd) { 426 case SIOCSIFADDR: 427 #ifdef INET 428 if (ifa->ifa_addr->sa_family == AF_INET) 429 avoid_reset = TRUE; 430 #endif 431 #ifdef INET6 432 if (ifa->ifa_addr->sa_family == AF_INET6) 433 avoid_reset = TRUE; 434 #endif 435 436 #if defined(INET) || defined(INET6) 437 /* Avoid reinitialization unless it's necessary */ 438 if (avoid_reset) { 439 if_setflagbits(ifp, IFF_UP, 0); 440 if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) 441 nicvf_if_init(nic); 442 #ifdef INET 443 if (!(if_getflags(ifp) & IFF_NOARP)) 444 arp_ifinit(ifp, ifa); 445 #endif 446 447 return (0); 448 } 449 #endif 450 err = ether_ioctl(ifp, cmd, data); 451 break; 452 case SIOCSIFMTU: 453 if (ifr->ifr_mtu < NIC_HW_MIN_FRS || 454 ifr->ifr_mtu > NIC_HW_MAX_FRS) { 455 err = EINVAL; 456 } else { 457 NICVF_CORE_LOCK(nic); 458 err = nicvf_update_hw_max_frs(nic, ifr->ifr_mtu); 459 if (err == 0) 460 if_setmtu(ifp, ifr->ifr_mtu); 461 NICVF_CORE_UNLOCK(nic); 462 } 463 break; 464 case SIOCSIFFLAGS: 465 NICVF_CORE_LOCK(nic); 466 flags = if_getflags(ifp); 467 if (flags & IFF_UP) { 468 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 469 if ((flags ^ nic->if_flags) & IFF_PROMISC) { 470 /* Change promiscous mode */ 471 #if 0 /* XXX */ 472 nicvf_set_promiscous(nic); 473 #endif 474 } 475 476 if ((flags ^ nic->if_flags) & IFF_ALLMULTI) { 477 /* Change multicasting settings */ 478 #if 0 /* XXX */ 479 nicvf_set_multicast(nic); 480 #endif 481 } 482 } else { 483 nicvf_if_init_locked(nic); 484 } 485 } else if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) 486 nicvf_stop_locked(nic); 487 488 nic->if_flags = flags; 489 NICVF_CORE_UNLOCK(nic); 490 break; 491 492 case SIOCADDMULTI: 493 case SIOCDELMULTI: 494 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 495 #if 0 496 NICVF_CORE_LOCK(nic); 497 /* ARM64TODO */ 498 nicvf_set_multicast(nic); 499 NICVF_CORE_UNLOCK(nic); 500 #endif 501 } 502 break; 503 504 case SIOCSIFMEDIA: 505 case SIOCGIFMEDIA: 506 err = ifmedia_ioctl(ifp, ifr, &nic->if_media, cmd); 507 break; 508 509 case SIOCSIFCAP: 510 mask = if_getcapenable(ifp) ^ ifr->ifr_reqcap; 511 if (mask & IFCAP_VLAN_MTU) { 512 /* No work to do except acknowledge the change took. */ 513 if_togglecapenable(ifp, IFCAP_VLAN_MTU); 514 } 515 if (mask & IFCAP_TXCSUM) 516 if_togglecapenable(ifp, IFCAP_TXCSUM); 517 if (mask & IFCAP_RXCSUM) 518 if_togglecapenable(ifp, IFCAP_RXCSUM); 519 if ((mask & IFCAP_TSO4) && nic->hw_tso) 520 if_togglecapenable(ifp, IFCAP_TSO4); 521 if (mask & IFCAP_LRO) { 522 /* 523 * Lock the driver for a moment to avoid 524 * mismatch in per-queue settings. 525 */ 526 NICVF_CORE_LOCK(nic); 527 if_togglecapenable(ifp, IFCAP_LRO); 528 if ((if_getdrvflags(nic->ifp) & IFF_DRV_RUNNING) != 0) { 529 /* 530 * Now disable LRO for subsequent packets. 531 * Atomicity of this change is not necessary 532 * as we don't need precise toggle of this 533 * feature for all threads processing the 534 * completion queue. 535 */ 536 for (rq_idx = 0; 537 rq_idx < nic->qs->rq_cnt; rq_idx++) { 538 rq = &nic->qs->rq[rq_idx]; 539 rq->lro_enabled = !rq->lro_enabled; 540 } 541 } 542 NICVF_CORE_UNLOCK(nic); 543 } 544 545 break; 546 547 default: 548 err = ether_ioctl(ifp, cmd, data); 549 break; 550 } 551 552 return (err); 553 } 554 555 static void 556 nicvf_if_init_locked(struct nicvf *nic) 557 { 558 struct queue_set *qs = nic->qs; 559 if_t ifp; 560 int qidx; 561 int err; 562 caddr_t if_addr; 563 564 NICVF_CORE_LOCK_ASSERT(nic); 565 ifp = nic->ifp; 566 567 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) 568 nicvf_stop_locked(nic); 569 570 err = nicvf_enable_misc_interrupt(nic); 571 if (err != 0) { 572 if_printf(ifp, "Could not reenable Mbox interrupt\n"); 573 return; 574 } 575 576 /* Get the latest MAC address */ 577 if_addr = if_getlladdr(ifp); 578 /* Update MAC address if changed */ 579 if (memcmp(nic->hwaddr, if_addr, ETHER_ADDR_LEN) != 0) { 580 memcpy(nic->hwaddr, if_addr, ETHER_ADDR_LEN); 581 nicvf_hw_set_mac_addr(nic, if_addr); 582 } 583 584 /* Initialize the queues */ 585 err = nicvf_init_resources(nic); 586 if (err != 0) 587 goto error; 588 589 /* Make sure queue initialization is written */ 590 wmb(); 591 592 nicvf_reg_write(nic, NIC_VF_INT, ~0UL); 593 /* Enable Qset err interrupt */ 594 nicvf_enable_intr(nic, NICVF_INTR_QS_ERR, 0); 595 596 /* Enable completion queue interrupt */ 597 for (qidx = 0; qidx < qs->cq_cnt; qidx++) 598 nicvf_enable_intr(nic, NICVF_INTR_CQ, qidx); 599 600 /* Enable RBDR threshold interrupt */ 601 for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) 602 nicvf_enable_intr(nic, NICVF_INTR_RBDR, qidx); 603 604 nic->drv_stats.txq_stop = 0; 605 nic->drv_stats.txq_wake = 0; 606 607 /* Activate network interface */ 608 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); 609 610 /* Schedule callout to update stats */ 611 callout_reset(&nic->stats_callout, hz, nicvf_tick_stats, nic); 612 613 return; 614 615 error: 616 /* Something went very wrong. Disable this ifnet for good */ 617 if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); 618 } 619 620 static void 621 nicvf_if_init(void *if_softc) 622 { 623 struct nicvf *nic = if_softc; 624 625 NICVF_CORE_LOCK(nic); 626 nicvf_if_init_locked(nic); 627 NICVF_CORE_UNLOCK(nic); 628 } 629 630 static int 631 nicvf_if_transmit(if_t ifp, struct mbuf *mbuf) 632 { 633 struct nicvf *nic = if_getsoftc(ifp); 634 struct queue_set *qs = nic->qs; 635 struct snd_queue *sq; 636 struct mbuf *mtmp; 637 int qidx; 638 int err = 0; 639 640 if (__predict_false(qs == NULL)) { 641 panic("%s: missing queue set for %s", __func__, 642 device_get_nameunit(nic->dev)); 643 } 644 645 /* Select queue */ 646 if (M_HASHTYPE_GET(mbuf) != M_HASHTYPE_NONE) 647 qidx = mbuf->m_pkthdr.flowid % qs->sq_cnt; 648 else 649 qidx = curcpu % qs->sq_cnt; 650 651 sq = &qs->sq[qidx]; 652 653 if (mbuf->m_next != NULL && 654 (mbuf->m_pkthdr.csum_flags & 655 (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_SCTP)) != 0) { 656 if (M_WRITABLE(mbuf) == 0) { 657 mtmp = m_dup(mbuf, M_NOWAIT); 658 m_freem(mbuf); 659 if (mtmp == NULL) 660 return (ENOBUFS); 661 mbuf = mtmp; 662 } 663 } 664 665 err = drbr_enqueue(ifp, sq->br, mbuf); 666 if (((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 667 IFF_DRV_RUNNING) || !nic->link_up || (err != 0)) { 668 /* 669 * Try to enqueue packet to the ring buffer. 670 * If the driver is not active, link down or enqueue operation 671 * failed, return with the appropriate error code. 672 */ 673 return (err); 674 } 675 676 if (NICVF_TX_TRYLOCK(sq) != 0) { 677 err = nicvf_xmit_locked(sq); 678 NICVF_TX_UNLOCK(sq); 679 return (err); 680 } else 681 taskqueue_enqueue(sq->snd_taskq, &sq->snd_task); 682 683 return (0); 684 } 685 686 static void 687 nicvf_if_qflush(if_t ifp) 688 { 689 struct nicvf *nic; 690 struct queue_set *qs; 691 struct snd_queue *sq; 692 struct mbuf *mbuf; 693 size_t idx; 694 695 nic = if_getsoftc(ifp); 696 qs = nic->qs; 697 698 for (idx = 0; idx < qs->sq_cnt; idx++) { 699 sq = &qs->sq[idx]; 700 NICVF_TX_LOCK(sq); 701 while ((mbuf = buf_ring_dequeue_sc(sq->br)) != NULL) 702 m_freem(mbuf); 703 NICVF_TX_UNLOCK(sq); 704 } 705 if_qflush(ifp); 706 } 707 708 static uint64_t 709 nicvf_if_getcounter(if_t ifp, ift_counter cnt) 710 { 711 struct nicvf *nic; 712 struct nicvf_hw_stats *hw_stats; 713 struct nicvf_drv_stats *drv_stats; 714 715 nic = if_getsoftc(ifp); 716 hw_stats = &nic->hw_stats; 717 drv_stats = &nic->drv_stats; 718 719 switch (cnt) { 720 case IFCOUNTER_IPACKETS: 721 return (drv_stats->rx_frames_ok); 722 case IFCOUNTER_OPACKETS: 723 return (drv_stats->tx_frames_ok); 724 case IFCOUNTER_IBYTES: 725 return (hw_stats->rx_bytes); 726 case IFCOUNTER_OBYTES: 727 return (hw_stats->tx_bytes_ok); 728 case IFCOUNTER_IMCASTS: 729 return (hw_stats->rx_mcast_frames); 730 case IFCOUNTER_COLLISIONS: 731 return (0); 732 case IFCOUNTER_IQDROPS: 733 return (drv_stats->rx_drops); 734 case IFCOUNTER_OQDROPS: 735 return (drv_stats->tx_drops); 736 default: 737 return (if_get_counter_default(ifp, cnt)); 738 } 739 740 } 741 742 static void 743 nicvf_media_status(if_t ifp, struct ifmediareq *ifmr) 744 { 745 struct nicvf *nic = if_getsoftc(ifp); 746 747 NICVF_CORE_LOCK(nic); 748 749 ifmr->ifm_status = IFM_AVALID; 750 ifmr->ifm_active = IFM_ETHER; 751 752 if (nic->link_up) { 753 /* Device attached to working network */ 754 ifmr->ifm_status |= IFM_ACTIVE; 755 } 756 757 switch (nic->speed) { 758 case SPEED_10: 759 ifmr->ifm_active |= IFM_10_T; 760 break; 761 case SPEED_100: 762 ifmr->ifm_active |= IFM_100_TX; 763 break; 764 case SPEED_1000: 765 ifmr->ifm_active |= IFM_1000_T; 766 break; 767 case SPEED_10000: 768 ifmr->ifm_active |= IFM_10G_SR; 769 break; 770 case SPEED_40000: 771 ifmr->ifm_active |= IFM_40G_CR4; 772 break; 773 default: 774 ifmr->ifm_active |= IFM_AUTO; 775 break; 776 } 777 778 if (nic->duplex) 779 ifmr->ifm_active |= IFM_FDX; 780 else 781 ifmr->ifm_active |= IFM_HDX; 782 783 NICVF_CORE_UNLOCK(nic); 784 } 785 786 static int 787 nicvf_media_change(if_t ifp __unused) 788 { 789 790 return (0); 791 } 792 793 /* Register read/write APIs */ 794 void 795 nicvf_reg_write(struct nicvf *nic, bus_space_handle_t offset, uint64_t val) 796 { 797 798 bus_write_8(nic->reg_base, offset, val); 799 } 800 801 uint64_t 802 nicvf_reg_read(struct nicvf *nic, uint64_t offset) 803 { 804 805 return (bus_read_8(nic->reg_base, offset)); 806 } 807 808 void 809 nicvf_queue_reg_write(struct nicvf *nic, bus_space_handle_t offset, 810 uint64_t qidx, uint64_t val) 811 { 812 813 bus_write_8(nic->reg_base, offset + (qidx << NIC_Q_NUM_SHIFT), val); 814 } 815 816 uint64_t 817 nicvf_queue_reg_read(struct nicvf *nic, bus_space_handle_t offset, 818 uint64_t qidx) 819 { 820 821 return (bus_read_8(nic->reg_base, offset + (qidx << NIC_Q_NUM_SHIFT))); 822 } 823 824 /* VF -> PF mailbox communication */ 825 static void 826 nicvf_write_to_mbx(struct nicvf *nic, union nic_mbx *mbx) 827 { 828 uint64_t *msg = (uint64_t *)mbx; 829 830 nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 0, msg[0]); 831 nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 8, msg[1]); 832 } 833 834 int 835 nicvf_send_msg_to_pf(struct nicvf *nic, union nic_mbx *mbx) 836 { 837 int timeout = NIC_MBOX_MSG_TIMEOUT * 10; 838 int sleep = 2; 839 840 NICVF_CORE_LOCK_ASSERT(nic); 841 842 nic->pf_acked = FALSE; 843 nic->pf_nacked = FALSE; 844 845 nicvf_write_to_mbx(nic, mbx); 846 847 /* Wait for previous message to be acked, timeout 2sec */ 848 while (!nic->pf_acked) { 849 if (nic->pf_nacked) 850 return (EINVAL); 851 852 DELAY(sleep * 1000); 853 854 if (nic->pf_acked) 855 break; 856 timeout -= sleep; 857 if (!timeout) { 858 device_printf(nic->dev, 859 "PF didn't ack to mbox msg %d from VF%d\n", 860 (mbx->msg.msg & 0xFF), nic->vf_id); 861 862 return (EBUSY); 863 } 864 } 865 return (0); 866 } 867 868 /* 869 * Checks if VF is able to comminicate with PF 870 * and also gets the VNIC number this VF is associated to. 871 */ 872 static int 873 nicvf_check_pf_ready(struct nicvf *nic) 874 { 875 union nic_mbx mbx = {}; 876 877 mbx.msg.msg = NIC_MBOX_MSG_READY; 878 if (nicvf_send_msg_to_pf(nic, &mbx)) { 879 device_printf(nic->dev, 880 "PF didn't respond to READY msg\n"); 881 return 0; 882 } 883 884 return 1; 885 } 886 887 static void 888 nicvf_read_bgx_stats(struct nicvf *nic, struct bgx_stats_msg *bgx) 889 { 890 891 if (bgx->rx) 892 nic->bgx_stats.rx_stats[bgx->idx] = bgx->stats; 893 else 894 nic->bgx_stats.tx_stats[bgx->idx] = bgx->stats; 895 } 896 897 static void 898 nicvf_handle_mbx_intr(struct nicvf *nic) 899 { 900 union nic_mbx mbx = {}; 901 uint64_t *mbx_data; 902 uint64_t mbx_addr; 903 int i; 904 905 mbx_addr = NIC_VF_PF_MAILBOX_0_1; 906 mbx_data = (uint64_t *)&mbx; 907 908 for (i = 0; i < NIC_PF_VF_MAILBOX_SIZE; i++) { 909 *mbx_data = nicvf_reg_read(nic, mbx_addr); 910 mbx_data++; 911 mbx_addr += sizeof(uint64_t); 912 } 913 914 switch (mbx.msg.msg) { 915 case NIC_MBOX_MSG_READY: 916 nic->pf_acked = TRUE; 917 nic->vf_id = mbx.nic_cfg.vf_id & 0x7F; 918 nic->tns_mode = mbx.nic_cfg.tns_mode & 0x7F; 919 nic->node = mbx.nic_cfg.node_id; 920 memcpy(nic->hwaddr, mbx.nic_cfg.mac_addr, ETHER_ADDR_LEN); 921 nic->loopback_supported = mbx.nic_cfg.loopback_supported; 922 nic->link_up = FALSE; 923 nic->duplex = 0; 924 nic->speed = 0; 925 break; 926 case NIC_MBOX_MSG_ACK: 927 nic->pf_acked = TRUE; 928 break; 929 case NIC_MBOX_MSG_NACK: 930 nic->pf_nacked = TRUE; 931 break; 932 case NIC_MBOX_MSG_RSS_SIZE: 933 nic->rss_info.rss_size = mbx.rss_size.ind_tbl_size; 934 nic->pf_acked = TRUE; 935 break; 936 case NIC_MBOX_MSG_BGX_STATS: 937 nicvf_read_bgx_stats(nic, &mbx.bgx_stats); 938 nic->pf_acked = TRUE; 939 break; 940 case NIC_MBOX_MSG_BGX_LINK_CHANGE: 941 nic->pf_acked = TRUE; 942 nic->link_up = mbx.link_status.link_up; 943 nic->duplex = mbx.link_status.duplex; 944 nic->speed = mbx.link_status.speed; 945 if (nic->link_up) { 946 if_setbaudrate(nic->ifp, nic->speed * 1000000); 947 if_link_state_change(nic->ifp, LINK_STATE_UP); 948 } else { 949 if_setbaudrate(nic->ifp, 0); 950 if_link_state_change(nic->ifp, LINK_STATE_DOWN); 951 } 952 break; 953 default: 954 device_printf(nic->dev, 955 "Invalid message from PF, msg 0x%x\n", mbx.msg.msg); 956 break; 957 } 958 nicvf_clear_intr(nic, NICVF_INTR_MBOX, 0); 959 } 960 961 static int 962 nicvf_update_hw_max_frs(struct nicvf *nic, int mtu) 963 { 964 union nic_mbx mbx = {}; 965 966 mbx.frs.msg = NIC_MBOX_MSG_SET_MAX_FRS; 967 mbx.frs.max_frs = mtu; 968 mbx.frs.vf_id = nic->vf_id; 969 970 return nicvf_send_msg_to_pf(nic, &mbx); 971 } 972 973 static int 974 nicvf_hw_set_mac_addr(struct nicvf *nic, uint8_t *hwaddr) 975 { 976 union nic_mbx mbx = {}; 977 978 mbx.mac.msg = NIC_MBOX_MSG_SET_MAC; 979 mbx.mac.vf_id = nic->vf_id; 980 memcpy(mbx.mac.mac_addr, hwaddr, ETHER_ADDR_LEN); 981 982 return (nicvf_send_msg_to_pf(nic, &mbx)); 983 } 984 985 static void 986 nicvf_config_cpi(struct nicvf *nic) 987 { 988 union nic_mbx mbx = {}; 989 990 mbx.cpi_cfg.msg = NIC_MBOX_MSG_CPI_CFG; 991 mbx.cpi_cfg.vf_id = nic->vf_id; 992 mbx.cpi_cfg.cpi_alg = nic->cpi_alg; 993 mbx.cpi_cfg.rq_cnt = nic->qs->rq_cnt; 994 995 nicvf_send_msg_to_pf(nic, &mbx); 996 } 997 998 static void 999 nicvf_get_rss_size(struct nicvf *nic) 1000 { 1001 union nic_mbx mbx = {}; 1002 1003 mbx.rss_size.msg = NIC_MBOX_MSG_RSS_SIZE; 1004 mbx.rss_size.vf_id = nic->vf_id; 1005 nicvf_send_msg_to_pf(nic, &mbx); 1006 } 1007 1008 static void 1009 nicvf_config_rss(struct nicvf *nic) 1010 { 1011 union nic_mbx mbx = {}; 1012 struct nicvf_rss_info *rss; 1013 int ind_tbl_len; 1014 int i, nextq; 1015 1016 rss = &nic->rss_info; 1017 ind_tbl_len = rss->rss_size; 1018 nextq = 0; 1019 1020 mbx.rss_cfg.vf_id = nic->vf_id; 1021 mbx.rss_cfg.hash_bits = rss->hash_bits; 1022 while (ind_tbl_len != 0) { 1023 mbx.rss_cfg.tbl_offset = nextq; 1024 mbx.rss_cfg.tbl_len = MIN(ind_tbl_len, 1025 RSS_IND_TBL_LEN_PER_MBX_MSG); 1026 mbx.rss_cfg.msg = mbx.rss_cfg.tbl_offset ? 1027 NIC_MBOX_MSG_RSS_CFG_CONT : NIC_MBOX_MSG_RSS_CFG; 1028 1029 for (i = 0; i < mbx.rss_cfg.tbl_len; i++) 1030 mbx.rss_cfg.ind_tbl[i] = rss->ind_tbl[nextq++]; 1031 1032 nicvf_send_msg_to_pf(nic, &mbx); 1033 1034 ind_tbl_len -= mbx.rss_cfg.tbl_len; 1035 } 1036 } 1037 1038 static void 1039 nicvf_set_rss_key(struct nicvf *nic) 1040 { 1041 struct nicvf_rss_info *rss; 1042 uint64_t key_addr; 1043 int idx; 1044 1045 rss = &nic->rss_info; 1046 key_addr = NIC_VNIC_RSS_KEY_0_4; 1047 1048 for (idx = 0; idx < RSS_HASH_KEY_SIZE; idx++) { 1049 nicvf_reg_write(nic, key_addr, rss->key[idx]); 1050 key_addr += sizeof(uint64_t); 1051 } 1052 } 1053 1054 static int 1055 nicvf_rss_init(struct nicvf *nic) 1056 { 1057 struct nicvf_rss_info *rss; 1058 int idx; 1059 1060 nicvf_get_rss_size(nic); 1061 1062 rss = &nic->rss_info; 1063 if (nic->cpi_alg != CPI_ALG_NONE) { 1064 rss->enable = FALSE; 1065 rss->hash_bits = 0; 1066 return (ENXIO); 1067 } 1068 1069 rss->enable = TRUE; 1070 1071 /* Using the HW reset value for now */ 1072 rss->key[0] = 0xFEED0BADFEED0BADUL; 1073 rss->key[1] = 0xFEED0BADFEED0BADUL; 1074 rss->key[2] = 0xFEED0BADFEED0BADUL; 1075 rss->key[3] = 0xFEED0BADFEED0BADUL; 1076 rss->key[4] = 0xFEED0BADFEED0BADUL; 1077 1078 nicvf_set_rss_key(nic); 1079 1080 rss->cfg = RSS_IP_HASH_ENA | RSS_TCP_HASH_ENA | RSS_UDP_HASH_ENA; 1081 nicvf_reg_write(nic, NIC_VNIC_RSS_CFG, rss->cfg); 1082 1083 rss->hash_bits = fls(rss->rss_size) - 1; 1084 for (idx = 0; idx < rss->rss_size; idx++) 1085 rss->ind_tbl[idx] = idx % nic->rx_queues; 1086 1087 nicvf_config_rss(nic); 1088 1089 return (0); 1090 } 1091 1092 static int 1093 nicvf_init_resources(struct nicvf *nic) 1094 { 1095 int err; 1096 union nic_mbx mbx = {}; 1097 1098 mbx.msg.msg = NIC_MBOX_MSG_CFG_DONE; 1099 1100 /* Enable Qset */ 1101 nicvf_qset_config(nic, TRUE); 1102 1103 /* Initialize queues and HW for data transfer */ 1104 err = nicvf_config_data_transfer(nic, TRUE); 1105 if (err) { 1106 device_printf(nic->dev, 1107 "Failed to alloc/config VF's QSet resources\n"); 1108 return (err); 1109 } 1110 1111 /* Send VF config done msg to PF */ 1112 nicvf_write_to_mbx(nic, &mbx); 1113 1114 return (0); 1115 } 1116 1117 static void 1118 nicvf_misc_intr_handler(void *arg) 1119 { 1120 struct nicvf *nic = (struct nicvf *)arg; 1121 uint64_t intr; 1122 1123 intr = nicvf_reg_read(nic, NIC_VF_INT); 1124 /* Check for spurious interrupt */ 1125 if (!(intr & NICVF_INTR_MBOX_MASK)) 1126 return; 1127 1128 nicvf_handle_mbx_intr(nic); 1129 } 1130 1131 static int 1132 nicvf_intr_handler(void *arg) 1133 { 1134 struct nicvf *nic; 1135 struct cmp_queue *cq; 1136 int qidx; 1137 1138 cq = (struct cmp_queue *)arg; 1139 nic = cq->nic; 1140 qidx = cq->idx; 1141 1142 /* Disable interrupts */ 1143 nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx); 1144 1145 taskqueue_enqueue(cq->cmp_taskq, &cq->cmp_task); 1146 1147 /* Clear interrupt */ 1148 nicvf_clear_intr(nic, NICVF_INTR_CQ, qidx); 1149 1150 return (FILTER_HANDLED); 1151 } 1152 1153 static void 1154 nicvf_rbdr_intr_handler(void *arg) 1155 { 1156 struct nicvf *nic; 1157 struct queue_set *qs; 1158 struct rbdr *rbdr; 1159 int qidx; 1160 1161 nic = (struct nicvf *)arg; 1162 1163 /* Disable RBDR interrupt and schedule softirq */ 1164 for (qidx = 0; qidx < nic->qs->rbdr_cnt; qidx++) { 1165 if (!nicvf_is_intr_enabled(nic, NICVF_INTR_RBDR, qidx)) 1166 continue; 1167 nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx); 1168 1169 qs = nic->qs; 1170 rbdr = &qs->rbdr[qidx]; 1171 taskqueue_enqueue(rbdr->rbdr_taskq, &rbdr->rbdr_task_nowait); 1172 /* Clear interrupt */ 1173 nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx); 1174 } 1175 } 1176 1177 static void 1178 nicvf_qs_err_intr_handler(void *arg) 1179 { 1180 struct nicvf *nic = (struct nicvf *)arg; 1181 struct queue_set *qs = nic->qs; 1182 1183 /* Disable Qset err interrupt and schedule softirq */ 1184 nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0); 1185 taskqueue_enqueue(qs->qs_err_taskq, &qs->qs_err_task); 1186 nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0); 1187 1188 } 1189 1190 static int 1191 nicvf_enable_msix(struct nicvf *nic) 1192 { 1193 struct pci_devinfo *dinfo; 1194 int rid, count; 1195 int ret; 1196 1197 dinfo = device_get_ivars(nic->dev); 1198 rid = dinfo->cfg.msix.msix_table_bar; 1199 nic->msix_table_res = 1200 bus_alloc_resource_any(nic->dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); 1201 if (nic->msix_table_res == NULL) { 1202 device_printf(nic->dev, 1203 "Could not allocate memory for MSI-X table\n"); 1204 return (ENXIO); 1205 } 1206 1207 count = nic->num_vec = NIC_VF_MSIX_VECTORS; 1208 1209 ret = pci_alloc_msix(nic->dev, &count); 1210 if ((ret != 0) || (count != nic->num_vec)) { 1211 device_printf(nic->dev, 1212 "Request for #%d msix vectors failed, error: %d\n", 1213 nic->num_vec, ret); 1214 return (ret); 1215 } 1216 1217 nic->msix_enabled = 1; 1218 return (0); 1219 } 1220 1221 static void 1222 nicvf_disable_msix(struct nicvf *nic) 1223 { 1224 1225 if (nic->msix_enabled) { 1226 pci_release_msi(nic->dev); 1227 nic->msix_enabled = 0; 1228 nic->num_vec = 0; 1229 } 1230 } 1231 1232 static void 1233 nicvf_release_all_interrupts(struct nicvf *nic) 1234 { 1235 struct resource *res; 1236 int irq; 1237 int err __diagused; 1238 1239 /* Free registered interrupts */ 1240 for (irq = 0; irq < nic->num_vec; irq++) { 1241 res = nic->msix_entries[irq].irq_res; 1242 if (res == NULL) 1243 continue; 1244 /* Teardown interrupt first */ 1245 if (nic->msix_entries[irq].handle != NULL) { 1246 err = bus_teardown_intr(nic->dev, 1247 nic->msix_entries[irq].irq_res, 1248 nic->msix_entries[irq].handle); 1249 KASSERT(err == 0, 1250 ("ERROR: Unable to teardown interrupt %d", irq)); 1251 nic->msix_entries[irq].handle = NULL; 1252 } 1253 1254 bus_release_resource(nic->dev, SYS_RES_IRQ, 1255 rman_get_rid(res), nic->msix_entries[irq].irq_res); 1256 nic->msix_entries[irq].irq_res = NULL; 1257 } 1258 /* Disable MSI-X */ 1259 nicvf_disable_msix(nic); 1260 } 1261 1262 /* 1263 * Initialize MSIX vectors and register MISC interrupt. 1264 * Send READY message to PF to check if its alive 1265 */ 1266 static int 1267 nicvf_allocate_misc_interrupt(struct nicvf *nic) 1268 { 1269 struct resource *res; 1270 int irq, rid; 1271 int ret = 0; 1272 1273 /* Return if mailbox interrupt is already registered */ 1274 if (nic->msix_enabled) 1275 return (0); 1276 1277 /* Enable MSI-X */ 1278 if (nicvf_enable_msix(nic) != 0) 1279 return (ENXIO); 1280 1281 irq = NICVF_INTR_ID_MISC; 1282 rid = irq + 1; 1283 nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev, 1284 SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE)); 1285 if (nic->msix_entries[irq].irq_res == NULL) { 1286 device_printf(nic->dev, 1287 "Could not allocate Mbox interrupt for VF%d\n", 1288 device_get_unit(nic->dev)); 1289 return (ENXIO); 1290 } 1291 1292 ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res, 1293 (INTR_MPSAFE | INTR_TYPE_MISC), NULL, nicvf_misc_intr_handler, nic, 1294 &nic->msix_entries[irq].handle); 1295 if (ret != 0) { 1296 res = nic->msix_entries[irq].irq_res; 1297 bus_release_resource(nic->dev, SYS_RES_IRQ, 1298 rman_get_rid(res), res); 1299 nic->msix_entries[irq].irq_res = NULL; 1300 return (ret); 1301 } 1302 1303 return (0); 1304 } 1305 1306 static int 1307 nicvf_enable_misc_interrupt(struct nicvf *nic) 1308 { 1309 1310 /* Enable mailbox interrupt */ 1311 nicvf_enable_intr(nic, NICVF_INTR_MBOX, 0); 1312 1313 /* Check if VF is able to communicate with PF */ 1314 if (!nicvf_check_pf_ready(nic)) { 1315 nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0); 1316 return (ENXIO); 1317 } 1318 1319 return (0); 1320 } 1321 1322 static void 1323 nicvf_release_net_interrupts(struct nicvf *nic) 1324 { 1325 struct resource *res; 1326 int irq; 1327 int err; 1328 1329 for_each_cq_irq(irq) { 1330 res = nic->msix_entries[irq].irq_res; 1331 if (res == NULL) 1332 continue; 1333 /* Teardown active interrupts first */ 1334 if (nic->msix_entries[irq].handle != NULL) { 1335 err = bus_teardown_intr(nic->dev, 1336 nic->msix_entries[irq].irq_res, 1337 nic->msix_entries[irq].handle); 1338 KASSERT(err == 0, 1339 ("ERROR: Unable to teardown CQ interrupt %d", 1340 (irq - NICVF_INTR_ID_CQ))); 1341 if (err != 0) 1342 continue; 1343 } 1344 1345 /* Release resource */ 1346 bus_release_resource(nic->dev, SYS_RES_IRQ, rman_get_rid(res), 1347 res); 1348 nic->msix_entries[irq].irq_res = NULL; 1349 } 1350 1351 for_each_rbdr_irq(irq) { 1352 res = nic->msix_entries[irq].irq_res; 1353 if (res == NULL) 1354 continue; 1355 /* Teardown active interrupts first */ 1356 if (nic->msix_entries[irq].handle != NULL) { 1357 err = bus_teardown_intr(nic->dev, 1358 nic->msix_entries[irq].irq_res, 1359 nic->msix_entries[irq].handle); 1360 KASSERT(err == 0, 1361 ("ERROR: Unable to teardown RDBR interrupt %d", 1362 (irq - NICVF_INTR_ID_RBDR))); 1363 if (err != 0) 1364 continue; 1365 } 1366 1367 /* Release resource */ 1368 bus_release_resource(nic->dev, SYS_RES_IRQ, rman_get_rid(res), 1369 res); 1370 nic->msix_entries[irq].irq_res = NULL; 1371 } 1372 1373 irq = NICVF_INTR_ID_QS_ERR; 1374 res = nic->msix_entries[irq].irq_res; 1375 if (res != NULL) { 1376 /* Teardown active interrupts first */ 1377 if (nic->msix_entries[irq].handle != NULL) { 1378 err = bus_teardown_intr(nic->dev, 1379 nic->msix_entries[irq].irq_res, 1380 nic->msix_entries[irq].handle); 1381 KASSERT(err == 0, 1382 ("ERROR: Unable to teardown QS Error interrupt %d", 1383 irq)); 1384 if (err != 0) 1385 return; 1386 } 1387 1388 /* Release resource */ 1389 bus_release_resource(nic->dev, SYS_RES_IRQ, rman_get_rid(res), 1390 res); 1391 nic->msix_entries[irq].irq_res = NULL; 1392 } 1393 } 1394 1395 static int 1396 nicvf_allocate_net_interrupts(struct nicvf *nic) 1397 { 1398 u_int cpuid; 1399 int irq, rid; 1400 int qidx; 1401 int ret = 0; 1402 1403 /* MSI-X must be configured by now */ 1404 if (!nic->msix_enabled) { 1405 device_printf(nic->dev, "Cannot alloacte queue interrups. " 1406 "MSI-X interrupts disabled.\n"); 1407 return (ENXIO); 1408 } 1409 1410 /* Register CQ interrupts */ 1411 for_each_cq_irq(irq) { 1412 if (irq >= (NICVF_INTR_ID_CQ + nic->qs->cq_cnt)) 1413 break; 1414 1415 qidx = irq - NICVF_INTR_ID_CQ; 1416 rid = irq + 1; 1417 nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev, 1418 SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE)); 1419 if (nic->msix_entries[irq].irq_res == NULL) { 1420 device_printf(nic->dev, 1421 "Could not allocate CQ interrupt %d for VF%d\n", 1422 (irq - NICVF_INTR_ID_CQ), device_get_unit(nic->dev)); 1423 ret = ENXIO; 1424 goto error; 1425 } 1426 ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res, 1427 (INTR_MPSAFE | INTR_TYPE_NET), nicvf_intr_handler, 1428 NULL, &nic->qs->cq[qidx], &nic->msix_entries[irq].handle); 1429 if (ret != 0) { 1430 device_printf(nic->dev, 1431 "Could not setup CQ interrupt %d for VF%d\n", 1432 (irq - NICVF_INTR_ID_CQ), device_get_unit(nic->dev)); 1433 goto error; 1434 } 1435 cpuid = (device_get_unit(nic->dev) * CMP_QUEUE_CNT) + qidx; 1436 cpuid %= mp_ncpus; 1437 /* 1438 * Save CPU ID for later use when system-wide RSS is enabled. 1439 * It will be used to pit the CQ task to the same CPU that got 1440 * interrupted. 1441 */ 1442 nic->qs->cq[qidx].cmp_cpuid = cpuid; 1443 if (bootverbose) { 1444 device_printf(nic->dev, "bind CQ%d IRQ to CPU%d\n", 1445 qidx, cpuid); 1446 } 1447 /* Bind interrupts to the given CPU */ 1448 bus_bind_intr(nic->dev, nic->msix_entries[irq].irq_res, cpuid); 1449 } 1450 1451 /* Register RBDR interrupt */ 1452 for_each_rbdr_irq(irq) { 1453 if (irq >= (NICVF_INTR_ID_RBDR + nic->qs->rbdr_cnt)) 1454 break; 1455 1456 rid = irq + 1; 1457 nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev, 1458 SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE)); 1459 if (nic->msix_entries[irq].irq_res == NULL) { 1460 device_printf(nic->dev, 1461 "Could not allocate RBDR interrupt %d for VF%d\n", 1462 (irq - NICVF_INTR_ID_RBDR), 1463 device_get_unit(nic->dev)); 1464 ret = ENXIO; 1465 goto error; 1466 } 1467 ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res, 1468 (INTR_MPSAFE | INTR_TYPE_NET), NULL, 1469 nicvf_rbdr_intr_handler, nic, 1470 &nic->msix_entries[irq].handle); 1471 if (ret != 0) { 1472 device_printf(nic->dev, 1473 "Could not setup RBDR interrupt %d for VF%d\n", 1474 (irq - NICVF_INTR_ID_RBDR), 1475 device_get_unit(nic->dev)); 1476 goto error; 1477 } 1478 } 1479 1480 /* Register QS error interrupt */ 1481 irq = NICVF_INTR_ID_QS_ERR; 1482 rid = irq + 1; 1483 nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev, 1484 SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE)); 1485 if (nic->msix_entries[irq].irq_res == NULL) { 1486 device_printf(nic->dev, 1487 "Could not allocate QS Error interrupt for VF%d\n", 1488 device_get_unit(nic->dev)); 1489 ret = ENXIO; 1490 goto error; 1491 } 1492 ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res, 1493 (INTR_MPSAFE | INTR_TYPE_NET), NULL, nicvf_qs_err_intr_handler, 1494 nic, &nic->msix_entries[irq].handle); 1495 if (ret != 0) { 1496 device_printf(nic->dev, 1497 "Could not setup QS Error interrupt for VF%d\n", 1498 device_get_unit(nic->dev)); 1499 goto error; 1500 } 1501 1502 return (0); 1503 error: 1504 nicvf_release_net_interrupts(nic); 1505 return (ret); 1506 } 1507 1508 static int 1509 nicvf_stop_locked(struct nicvf *nic) 1510 { 1511 if_t ifp; 1512 int qidx; 1513 struct queue_set *qs = nic->qs; 1514 union nic_mbx mbx = {}; 1515 1516 NICVF_CORE_LOCK_ASSERT(nic); 1517 /* Stop callout. Can block here since holding SX lock */ 1518 callout_drain(&nic->stats_callout); 1519 1520 ifp = nic->ifp; 1521 1522 mbx.msg.msg = NIC_MBOX_MSG_SHUTDOWN; 1523 nicvf_send_msg_to_pf(nic, &mbx); 1524 1525 /* Disable RBDR & QS error interrupts */ 1526 for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) { 1527 nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx); 1528 nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx); 1529 } 1530 nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0); 1531 nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0); 1532 1533 /* Deactivate network interface */ 1534 if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); 1535 1536 /* Free resources */ 1537 nicvf_config_data_transfer(nic, FALSE); 1538 1539 /* Disable HW Qset */ 1540 nicvf_qset_config(nic, FALSE); 1541 1542 /* disable mailbox interrupt */ 1543 nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0); 1544 1545 return (0); 1546 } 1547 1548 static void 1549 nicvf_update_stats(struct nicvf *nic) 1550 { 1551 int qidx; 1552 struct nicvf_hw_stats *stats = &nic->hw_stats; 1553 struct nicvf_drv_stats *drv_stats = &nic->drv_stats; 1554 struct queue_set *qs = nic->qs; 1555 1556 #define GET_RX_STATS(reg) \ 1557 nicvf_reg_read(nic, NIC_VNIC_RX_STAT_0_13 | ((reg) << 3)) 1558 #define GET_TX_STATS(reg) \ 1559 nicvf_reg_read(nic, NIC_VNIC_TX_STAT_0_4 | ((reg) << 3)) 1560 1561 stats->rx_bytes = GET_RX_STATS(RX_OCTS); 1562 stats->rx_ucast_frames = GET_RX_STATS(RX_UCAST); 1563 stats->rx_bcast_frames = GET_RX_STATS(RX_BCAST); 1564 stats->rx_mcast_frames = GET_RX_STATS(RX_MCAST); 1565 stats->rx_fcs_errors = GET_RX_STATS(RX_FCS); 1566 stats->rx_l2_errors = GET_RX_STATS(RX_L2ERR); 1567 stats->rx_drop_red = GET_RX_STATS(RX_RED); 1568 stats->rx_drop_red_bytes = GET_RX_STATS(RX_RED_OCTS); 1569 stats->rx_drop_overrun = GET_RX_STATS(RX_ORUN); 1570 stats->rx_drop_overrun_bytes = GET_RX_STATS(RX_ORUN_OCTS); 1571 stats->rx_drop_bcast = GET_RX_STATS(RX_DRP_BCAST); 1572 stats->rx_drop_mcast = GET_RX_STATS(RX_DRP_MCAST); 1573 stats->rx_drop_l3_bcast = GET_RX_STATS(RX_DRP_L3BCAST); 1574 stats->rx_drop_l3_mcast = GET_RX_STATS(RX_DRP_L3MCAST); 1575 1576 stats->tx_bytes_ok = GET_TX_STATS(TX_OCTS); 1577 stats->tx_ucast_frames_ok = GET_TX_STATS(TX_UCAST); 1578 stats->tx_bcast_frames_ok = GET_TX_STATS(TX_BCAST); 1579 stats->tx_mcast_frames_ok = GET_TX_STATS(TX_MCAST); 1580 stats->tx_drops = GET_TX_STATS(TX_DROP); 1581 1582 drv_stats->tx_frames_ok = stats->tx_ucast_frames_ok + 1583 stats->tx_bcast_frames_ok + stats->tx_mcast_frames_ok; 1584 drv_stats->rx_drops = stats->rx_drop_red + stats->rx_drop_overrun; 1585 drv_stats->tx_drops = stats->tx_drops; 1586 1587 /* Update RQ and SQ stats */ 1588 for (qidx = 0; qidx < qs->rq_cnt; qidx++) 1589 nicvf_update_rq_stats(nic, qidx); 1590 for (qidx = 0; qidx < qs->sq_cnt; qidx++) 1591 nicvf_update_sq_stats(nic, qidx); 1592 } 1593 1594 static void 1595 nicvf_tick_stats(void *arg) 1596 { 1597 struct nicvf *nic; 1598 1599 nic = (struct nicvf *)arg; 1600 1601 /* Read the statistics */ 1602 nicvf_update_stats(nic); 1603 1604 callout_reset(&nic->stats_callout, hz, nicvf_tick_stats, nic); 1605 } 1606