1 /* 2 * Copyright (C) 2015 Cavium Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 */ 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include "opt_inet.h" 31 #include "opt_inet6.h" 32 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/bitset.h> 36 #include <sys/bitstring.h> 37 #include <sys/bus.h> 38 #include <sys/endian.h> 39 #include <sys/kernel.h> 40 #include <sys/malloc.h> 41 #include <sys/mbuf.h> 42 #include <sys/module.h> 43 #include <sys/rman.h> 44 #include <sys/pciio.h> 45 #include <sys/pcpu.h> 46 #include <sys/proc.h> 47 #include <sys/socket.h> 48 #include <sys/sockio.h> 49 #include <sys/stdatomic.h> 50 #include <sys/cpuset.h> 51 #include <sys/lock.h> 52 #include <sys/mutex.h> 53 #include <sys/smp.h> 54 #include <sys/taskqueue.h> 55 56 #include <net/bpf.h> 57 #include <net/ethernet.h> 58 #include <net/if.h> 59 #include <net/if_var.h> 60 #include <net/if_arp.h> 61 #include <net/if_dl.h> 62 #include <net/if_media.h> 63 #include <net/if_types.h> 64 #include <net/if_vlan_var.h> 65 66 #include <netinet/in.h> 67 #include <netinet/ip.h> 68 #include <netinet/if_ether.h> 69 #include <netinet/tcp_lro.h> 70 71 #include <dev/pci/pcireg.h> 72 #include <dev/pci/pcivar.h> 73 74 #include <sys/dnv.h> 75 #include <sys/nv.h> 76 #include <sys/iov_schema.h> 77 78 #include <machine/bus.h> 79 80 #include "thunder_bgx.h" 81 #include "nic_reg.h" 82 #include "nic.h" 83 #include "nicvf_queues.h" 84 85 #define VNIC_VF_DEVSTR "Cavium Thunder NIC Virtual Function Driver" 86 87 #define VNIC_VF_REG_RID PCIR_BAR(PCI_CFG_REG_BAR_NUM) 88 89 /* Lock for core interface settings */ 90 #define NICVF_CORE_LOCK_INIT(nic) \ 91 sx_init(&(nic)->core_sx, device_get_nameunit((nic)->dev)) 92 93 #define NICVF_CORE_LOCK_DESTROY(nic) \ 94 sx_destroy(&(nic)->core_sx) 95 96 #define NICVF_CORE_LOCK(nic) sx_xlock(&(nic)->core_sx) 97 #define NICVF_CORE_UNLOCK(nic) sx_xunlock(&(nic)->core_sx) 98 99 #define NICVF_CORE_LOCK_ASSERT(nic) sx_assert(&(nic)->core_sx, SA_XLOCKED) 100 101 #define SPEED_10 10 102 #define SPEED_100 100 103 #define SPEED_1000 1000 104 #define SPEED_10000 10000 105 #define SPEED_40000 40000 106 107 MALLOC_DEFINE(M_NICVF, "nicvf", "ThunderX VNIC VF dynamic memory"); 108 109 static int nicvf_probe(device_t); 110 static int nicvf_attach(device_t); 111 static int nicvf_detach(device_t); 112 113 static device_method_t nicvf_methods[] = { 114 /* Device interface */ 115 DEVMETHOD(device_probe, nicvf_probe), 116 DEVMETHOD(device_attach, nicvf_attach), 117 DEVMETHOD(device_detach, nicvf_detach), 118 119 DEVMETHOD_END, 120 }; 121 122 static driver_t nicvf_driver = { 123 "vnic", 124 nicvf_methods, 125 sizeof(struct nicvf), 126 }; 127 128 DRIVER_MODULE(vnicvf, pci, nicvf_driver, 0, 0); 129 MODULE_VERSION(vnicvf, 1); 130 MODULE_DEPEND(vnicvf, pci, 1, 1, 1); 131 MODULE_DEPEND(vnicvf, ether, 1, 1, 1); 132 MODULE_DEPEND(vnicvf, vnicpf, 1, 1, 1); 133 134 static int nicvf_allocate_misc_interrupt(struct nicvf *); 135 static int nicvf_enable_misc_interrupt(struct nicvf *); 136 static int nicvf_allocate_net_interrupts(struct nicvf *); 137 static void nicvf_release_all_interrupts(struct nicvf *); 138 static int nicvf_update_hw_max_frs(struct nicvf *, int); 139 static int nicvf_hw_set_mac_addr(struct nicvf *, uint8_t *); 140 static void nicvf_config_cpi(struct nicvf *); 141 static int nicvf_rss_init(struct nicvf *); 142 static int nicvf_init_resources(struct nicvf *); 143 144 static int nicvf_setup_ifnet(struct nicvf *); 145 static int nicvf_setup_ifmedia(struct nicvf *); 146 static void nicvf_hw_addr_random(uint8_t *); 147 148 static int nicvf_if_ioctl(if_t, u_long, caddr_t); 149 static void nicvf_if_init(void *); 150 static void nicvf_if_init_locked(struct nicvf *); 151 static int nicvf_if_transmit(if_t, struct mbuf *); 152 static void nicvf_if_qflush(if_t); 153 static uint64_t nicvf_if_getcounter(if_t, ift_counter); 154 155 static int nicvf_stop_locked(struct nicvf *); 156 157 static void nicvf_media_status(if_t, struct ifmediareq *); 158 static int nicvf_media_change(if_t); 159 160 static void nicvf_tick_stats(void *); 161 162 static int 163 nicvf_probe(device_t dev) 164 { 165 uint16_t vendor_id; 166 uint16_t device_id; 167 168 vendor_id = pci_get_vendor(dev); 169 device_id = pci_get_device(dev); 170 171 if (vendor_id != PCI_VENDOR_ID_CAVIUM) 172 return (ENXIO); 173 174 if (device_id == PCI_DEVICE_ID_THUNDER_NIC_VF || 175 device_id == PCI_DEVICE_ID_THUNDER_PASS1_NIC_VF) { 176 device_set_desc(dev, VNIC_VF_DEVSTR); 177 return (BUS_PROBE_DEFAULT); 178 } 179 180 return (ENXIO); 181 } 182 183 static int 184 nicvf_attach(device_t dev) 185 { 186 struct nicvf *nic; 187 int rid, qcount; 188 int err = 0; 189 uint8_t hwaddr[ETHER_ADDR_LEN]; 190 uint8_t zeromac[] = {[0 ... (ETHER_ADDR_LEN - 1)] = 0}; 191 192 nic = device_get_softc(dev); 193 nic->dev = dev; 194 nic->pnicvf = nic; 195 196 NICVF_CORE_LOCK_INIT(nic); 197 /* Enable HW TSO on Pass2 */ 198 if (!pass1_silicon(dev)) 199 nic->hw_tso = TRUE; 200 201 rid = VNIC_VF_REG_RID; 202 nic->reg_base = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 203 RF_ACTIVE); 204 if (nic->reg_base == NULL) { 205 device_printf(dev, "Could not allocate registers memory\n"); 206 return (ENXIO); 207 } 208 209 qcount = MAX_CMP_QUEUES_PER_QS; 210 nic->max_queues = qcount; 211 212 err = nicvf_set_qset_resources(nic); 213 if (err != 0) 214 goto err_free_res; 215 216 /* Check if PF is alive and get MAC address for this VF */ 217 err = nicvf_allocate_misc_interrupt(nic); 218 if (err != 0) 219 goto err_free_res; 220 221 NICVF_CORE_LOCK(nic); 222 err = nicvf_enable_misc_interrupt(nic); 223 NICVF_CORE_UNLOCK(nic); 224 if (err != 0) 225 goto err_release_intr; 226 227 err = nicvf_allocate_net_interrupts(nic); 228 if (err != 0) { 229 device_printf(dev, 230 "Could not allocate network interface interrupts\n"); 231 goto err_free_ifnet; 232 } 233 234 /* If no MAC address was obtained we generate random one */ 235 if (memcmp(nic->hwaddr, zeromac, ETHER_ADDR_LEN) == 0) { 236 nicvf_hw_addr_random(hwaddr); 237 memcpy(nic->hwaddr, hwaddr, ETHER_ADDR_LEN); 238 NICVF_CORE_LOCK(nic); 239 nicvf_hw_set_mac_addr(nic, hwaddr); 240 NICVF_CORE_UNLOCK(nic); 241 } 242 243 /* Configure CPI alorithm */ 244 nic->cpi_alg = CPI_ALG_NONE; 245 NICVF_CORE_LOCK(nic); 246 nicvf_config_cpi(nic); 247 /* Configure receive side scaling */ 248 if (nic->qs->rq_cnt > 1) 249 nicvf_rss_init(nic); 250 NICVF_CORE_UNLOCK(nic); 251 252 err = nicvf_setup_ifnet(nic); 253 if (err != 0) { 254 device_printf(dev, "Could not set-up ifnet\n"); 255 goto err_release_intr; 256 } 257 258 err = nicvf_setup_ifmedia(nic); 259 if (err != 0) { 260 device_printf(dev, "Could not set-up ifmedia\n"); 261 goto err_free_ifnet; 262 } 263 264 mtx_init(&nic->stats_mtx, "VNIC stats", NULL, MTX_DEF); 265 callout_init_mtx(&nic->stats_callout, &nic->stats_mtx, 0); 266 267 ether_ifattach(nic->ifp, nic->hwaddr); 268 269 return (0); 270 271 err_free_ifnet: 272 if_free(nic->ifp); 273 err_release_intr: 274 nicvf_release_all_interrupts(nic); 275 err_free_res: 276 bus_release_resource(dev, SYS_RES_MEMORY, rman_get_rid(nic->reg_base), 277 nic->reg_base); 278 279 return (err); 280 } 281 282 static int 283 nicvf_detach(device_t dev) 284 { 285 struct nicvf *nic; 286 287 nic = device_get_softc(dev); 288 289 NICVF_CORE_LOCK(nic); 290 /* Shut down the port and release ring resources */ 291 nicvf_stop_locked(nic); 292 /* Release stats lock */ 293 mtx_destroy(&nic->stats_mtx); 294 /* Release interrupts */ 295 nicvf_release_all_interrupts(nic); 296 /* Release memory resource */ 297 if (nic->reg_base != NULL) { 298 bus_release_resource(dev, SYS_RES_MEMORY, 299 rman_get_rid(nic->reg_base), nic->reg_base); 300 } 301 302 /* Remove all ifmedia configurations */ 303 ifmedia_removeall(&nic->if_media); 304 /* Free this ifnet */ 305 if_free(nic->ifp); 306 NICVF_CORE_UNLOCK(nic); 307 /* Finally destroy the lock */ 308 NICVF_CORE_LOCK_DESTROY(nic); 309 310 return (0); 311 } 312 313 static void 314 nicvf_hw_addr_random(uint8_t *hwaddr) 315 { 316 uint32_t rnd; 317 uint8_t addr[ETHER_ADDR_LEN]; 318 319 /* 320 * Create randomized MAC address. 321 * Set 'bsd' + random 24 low-order bits. 322 */ 323 rnd = arc4random() & 0x00ffffff; 324 addr[0] = 'b'; 325 addr[1] = 's'; 326 addr[2] = 'd'; 327 addr[3] = rnd >> 16; 328 addr[4] = rnd >> 8; 329 addr[5] = rnd >> 0; 330 331 memcpy(hwaddr, addr, ETHER_ADDR_LEN); 332 } 333 334 static int 335 nicvf_setup_ifnet(struct nicvf *nic) 336 { 337 if_t ifp; 338 339 ifp = if_alloc(IFT_ETHER); 340 if (ifp == NULL) { 341 device_printf(nic->dev, "Could not allocate ifnet structure\n"); 342 return (ENOMEM); 343 } 344 345 nic->ifp = ifp; 346 347 if_setsoftc(ifp, nic); 348 if_initname(ifp, device_get_name(nic->dev), device_get_unit(nic->dev)); 349 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); 350 351 if_settransmitfn(ifp, nicvf_if_transmit); 352 if_setqflushfn(ifp, nicvf_if_qflush); 353 if_setioctlfn(ifp, nicvf_if_ioctl); 354 if_setinitfn(ifp, nicvf_if_init); 355 if_setgetcounterfn(ifp, nicvf_if_getcounter); 356 357 if_setmtu(ifp, ETHERMTU); 358 359 /* Reset caps */ 360 if_setcapabilities(ifp, 0); 361 362 /* Set the default values */ 363 if_setcapabilitiesbit(ifp, IFCAP_VLAN_MTU | IFCAP_JUMBO_MTU, 0); 364 if_setcapabilitiesbit(ifp, IFCAP_LRO, 0); 365 if (nic->hw_tso) { 366 /* TSO */ 367 if_setcapabilitiesbit(ifp, IFCAP_TSO4, 0); 368 /* TSO parameters */ 369 if_sethwtsomax(ifp, NICVF_TSO_MAXSIZE); 370 if_sethwtsomaxsegcount(ifp, NICVF_TSO_NSEGS); 371 if_sethwtsomaxsegsize(ifp, MCLBYTES); 372 } 373 /* IP/TCP/UDP HW checksums */ 374 if_setcapabilitiesbit(ifp, IFCAP_HWCSUM, 0); 375 if_setcapabilitiesbit(ifp, IFCAP_HWSTATS, 0); 376 /* 377 * HW offload enable 378 */ 379 if_clearhwassist(ifp); 380 if_sethwassistbits(ifp, (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_SCTP), 0); 381 if (nic->hw_tso) 382 if_sethwassistbits(ifp, (CSUM_TSO), 0); 383 if_setcapenable(ifp, if_getcapabilities(ifp)); 384 385 return (0); 386 } 387 388 static int 389 nicvf_setup_ifmedia(struct nicvf *nic) 390 { 391 392 ifmedia_init(&nic->if_media, IFM_IMASK, nicvf_media_change, 393 nicvf_media_status); 394 395 /* 396 * Advertise availability of all possible connection types, 397 * even though not all are possible at the same time. 398 */ 399 400 ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_10_T | IFM_FDX), 401 0, NULL); 402 ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_100_TX | IFM_FDX), 403 0, NULL); 404 ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_1000_T | IFM_FDX), 405 0, NULL); 406 ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_10G_SR | IFM_FDX), 407 0, NULL); 408 ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_40G_CR4 | IFM_FDX), 409 0, NULL); 410 ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_AUTO | IFM_FDX), 411 0, NULL); 412 413 ifmedia_set(&nic->if_media, (IFM_ETHER | IFM_AUTO | IFM_FDX)); 414 415 return (0); 416 } 417 418 static int 419 nicvf_if_ioctl(if_t ifp, u_long cmd, caddr_t data) 420 { 421 struct nicvf *nic; 422 struct rcv_queue *rq; 423 struct ifreq *ifr; 424 uint32_t flags; 425 int mask, err; 426 int rq_idx; 427 #if defined(INET) || defined(INET6) 428 struct ifaddr *ifa; 429 boolean_t avoid_reset = FALSE; 430 #endif 431 432 nic = if_getsoftc(ifp); 433 ifr = (struct ifreq *)data; 434 #if defined(INET) || defined(INET6) 435 ifa = (struct ifaddr *)data; 436 #endif 437 err = 0; 438 switch (cmd) { 439 case SIOCSIFADDR: 440 #ifdef INET 441 if (ifa->ifa_addr->sa_family == AF_INET) 442 avoid_reset = TRUE; 443 #endif 444 #ifdef INET6 445 if (ifa->ifa_addr->sa_family == AF_INET6) 446 avoid_reset = TRUE; 447 #endif 448 449 #if defined(INET) || defined(INET6) 450 /* Avoid reinitialization unless it's necessary */ 451 if (avoid_reset) { 452 if_setflagbits(ifp, IFF_UP, 0); 453 if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) 454 nicvf_if_init(nic); 455 #ifdef INET 456 if (!(if_getflags(ifp) & IFF_NOARP)) 457 arp_ifinit(ifp, ifa); 458 #endif 459 460 return (0); 461 } 462 #endif 463 err = ether_ioctl(ifp, cmd, data); 464 break; 465 case SIOCSIFMTU: 466 if (ifr->ifr_mtu < NIC_HW_MIN_FRS || 467 ifr->ifr_mtu > NIC_HW_MAX_FRS) { 468 err = EINVAL; 469 } else { 470 NICVF_CORE_LOCK(nic); 471 err = nicvf_update_hw_max_frs(nic, ifr->ifr_mtu); 472 if (err == 0) 473 if_setmtu(ifp, ifr->ifr_mtu); 474 NICVF_CORE_UNLOCK(nic); 475 } 476 break; 477 case SIOCSIFFLAGS: 478 NICVF_CORE_LOCK(nic); 479 flags = if_getflags(ifp); 480 if (flags & IFF_UP) { 481 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 482 if ((flags ^ nic->if_flags) & IFF_PROMISC) { 483 /* Change promiscous mode */ 484 #if 0 /* XXX */ 485 nicvf_set_promiscous(nic); 486 #endif 487 } 488 489 if ((flags ^ nic->if_flags) & IFF_ALLMULTI) { 490 /* Change multicasting settings */ 491 #if 0 /* XXX */ 492 nicvf_set_multicast(nic); 493 #endif 494 } 495 } else { 496 nicvf_if_init_locked(nic); 497 } 498 } else if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) 499 nicvf_stop_locked(nic); 500 501 nic->if_flags = flags; 502 NICVF_CORE_UNLOCK(nic); 503 break; 504 505 case SIOCADDMULTI: 506 case SIOCDELMULTI: 507 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 508 #if 0 509 NICVF_CORE_LOCK(nic); 510 /* ARM64TODO */ 511 nicvf_set_multicast(nic); 512 NICVF_CORE_UNLOCK(nic); 513 #endif 514 } 515 break; 516 517 case SIOCSIFMEDIA: 518 case SIOCGIFMEDIA: 519 err = ifmedia_ioctl(ifp, ifr, &nic->if_media, cmd); 520 break; 521 522 case SIOCSIFCAP: 523 mask = if_getcapenable(ifp) ^ ifr->ifr_reqcap; 524 if (mask & IFCAP_VLAN_MTU) { 525 /* No work to do except acknowledge the change took. */ 526 if_togglecapenable(ifp, IFCAP_VLAN_MTU); 527 } 528 if (mask & IFCAP_TXCSUM) 529 if_togglecapenable(ifp, IFCAP_TXCSUM); 530 if (mask & IFCAP_RXCSUM) 531 if_togglecapenable(ifp, IFCAP_RXCSUM); 532 if ((mask & IFCAP_TSO4) && nic->hw_tso) 533 if_togglecapenable(ifp, IFCAP_TSO4); 534 if (mask & IFCAP_LRO) { 535 /* 536 * Lock the driver for a moment to avoid 537 * mismatch in per-queue settings. 538 */ 539 NICVF_CORE_LOCK(nic); 540 if_togglecapenable(ifp, IFCAP_LRO); 541 if ((if_getdrvflags(nic->ifp) & IFF_DRV_RUNNING) != 0) { 542 /* 543 * Now disable LRO for subsequent packets. 544 * Atomicity of this change is not necessary 545 * as we don't need precise toggle of this 546 * feature for all threads processing the 547 * completion queue. 548 */ 549 for (rq_idx = 0; 550 rq_idx < nic->qs->rq_cnt; rq_idx++) { 551 rq = &nic->qs->rq[rq_idx]; 552 rq->lro_enabled = !rq->lro_enabled; 553 } 554 } 555 NICVF_CORE_UNLOCK(nic); 556 } 557 558 break; 559 560 default: 561 err = ether_ioctl(ifp, cmd, data); 562 break; 563 } 564 565 return (err); 566 } 567 568 static void 569 nicvf_if_init_locked(struct nicvf *nic) 570 { 571 struct queue_set *qs = nic->qs; 572 if_t ifp; 573 int qidx; 574 int err; 575 caddr_t if_addr; 576 577 NICVF_CORE_LOCK_ASSERT(nic); 578 ifp = nic->ifp; 579 580 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) 581 nicvf_stop_locked(nic); 582 583 err = nicvf_enable_misc_interrupt(nic); 584 if (err != 0) { 585 if_printf(ifp, "Could not reenable Mbox interrupt\n"); 586 return; 587 } 588 589 /* Get the latest MAC address */ 590 if_addr = if_getlladdr(ifp); 591 /* Update MAC address if changed */ 592 if (memcmp(nic->hwaddr, if_addr, ETHER_ADDR_LEN) != 0) { 593 memcpy(nic->hwaddr, if_addr, ETHER_ADDR_LEN); 594 nicvf_hw_set_mac_addr(nic, if_addr); 595 } 596 597 /* Initialize the queues */ 598 err = nicvf_init_resources(nic); 599 if (err != 0) 600 goto error; 601 602 /* Make sure queue initialization is written */ 603 wmb(); 604 605 nicvf_reg_write(nic, NIC_VF_INT, ~0UL); 606 /* Enable Qset err interrupt */ 607 nicvf_enable_intr(nic, NICVF_INTR_QS_ERR, 0); 608 609 /* Enable completion queue interrupt */ 610 for (qidx = 0; qidx < qs->cq_cnt; qidx++) 611 nicvf_enable_intr(nic, NICVF_INTR_CQ, qidx); 612 613 /* Enable RBDR threshold interrupt */ 614 for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) 615 nicvf_enable_intr(nic, NICVF_INTR_RBDR, qidx); 616 617 nic->drv_stats.txq_stop = 0; 618 nic->drv_stats.txq_wake = 0; 619 620 /* Activate network interface */ 621 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); 622 623 /* Schedule callout to update stats */ 624 callout_reset(&nic->stats_callout, hz, nicvf_tick_stats, nic); 625 626 return; 627 628 error: 629 /* Something went very wrong. Disable this ifnet for good */ 630 if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); 631 } 632 633 static void 634 nicvf_if_init(void *if_softc) 635 { 636 struct nicvf *nic = if_softc; 637 638 NICVF_CORE_LOCK(nic); 639 nicvf_if_init_locked(nic); 640 NICVF_CORE_UNLOCK(nic); 641 } 642 643 static int 644 nicvf_if_transmit(if_t ifp, struct mbuf *mbuf) 645 { 646 struct nicvf *nic = if_getsoftc(ifp); 647 struct queue_set *qs = nic->qs; 648 struct snd_queue *sq; 649 struct mbuf *mtmp; 650 int qidx; 651 int err = 0; 652 653 if (__predict_false(qs == NULL)) { 654 panic("%s: missing queue set for %s", __func__, 655 device_get_nameunit(nic->dev)); 656 } 657 658 /* Select queue */ 659 if (M_HASHTYPE_GET(mbuf) != M_HASHTYPE_NONE) 660 qidx = mbuf->m_pkthdr.flowid % qs->sq_cnt; 661 else 662 qidx = curcpu % qs->sq_cnt; 663 664 sq = &qs->sq[qidx]; 665 666 if (mbuf->m_next != NULL && 667 (mbuf->m_pkthdr.csum_flags & 668 (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_SCTP)) != 0) { 669 if (M_WRITABLE(mbuf) == 0) { 670 mtmp = m_dup(mbuf, M_NOWAIT); 671 m_freem(mbuf); 672 if (mtmp == NULL) 673 return (ENOBUFS); 674 mbuf = mtmp; 675 } 676 } 677 678 err = drbr_enqueue(ifp, sq->br, mbuf); 679 if (((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 680 IFF_DRV_RUNNING) || !nic->link_up || (err != 0)) { 681 /* 682 * Try to enqueue packet to the ring buffer. 683 * If the driver is not active, link down or enqueue operation 684 * failed, return with the appropriate error code. 685 */ 686 return (err); 687 } 688 689 if (NICVF_TX_TRYLOCK(sq) != 0) { 690 err = nicvf_xmit_locked(sq); 691 NICVF_TX_UNLOCK(sq); 692 return (err); 693 } else 694 taskqueue_enqueue(sq->snd_taskq, &sq->snd_task); 695 696 return (0); 697 } 698 699 static void 700 nicvf_if_qflush(if_t ifp) 701 { 702 struct nicvf *nic; 703 struct queue_set *qs; 704 struct snd_queue *sq; 705 struct mbuf *mbuf; 706 size_t idx; 707 708 nic = if_getsoftc(ifp); 709 qs = nic->qs; 710 711 for (idx = 0; idx < qs->sq_cnt; idx++) { 712 sq = &qs->sq[idx]; 713 NICVF_TX_LOCK(sq); 714 while ((mbuf = buf_ring_dequeue_sc(sq->br)) != NULL) 715 m_freem(mbuf); 716 NICVF_TX_UNLOCK(sq); 717 } 718 if_qflush(ifp); 719 } 720 721 static uint64_t 722 nicvf_if_getcounter(if_t ifp, ift_counter cnt) 723 { 724 struct nicvf *nic; 725 struct nicvf_hw_stats *hw_stats; 726 struct nicvf_drv_stats *drv_stats; 727 728 nic = if_getsoftc(ifp); 729 hw_stats = &nic->hw_stats; 730 drv_stats = &nic->drv_stats; 731 732 switch (cnt) { 733 case IFCOUNTER_IPACKETS: 734 return (drv_stats->rx_frames_ok); 735 case IFCOUNTER_OPACKETS: 736 return (drv_stats->tx_frames_ok); 737 case IFCOUNTER_IBYTES: 738 return (hw_stats->rx_bytes); 739 case IFCOUNTER_OBYTES: 740 return (hw_stats->tx_bytes_ok); 741 case IFCOUNTER_IMCASTS: 742 return (hw_stats->rx_mcast_frames); 743 case IFCOUNTER_COLLISIONS: 744 return (0); 745 case IFCOUNTER_IQDROPS: 746 return (drv_stats->rx_drops); 747 case IFCOUNTER_OQDROPS: 748 return (drv_stats->tx_drops); 749 default: 750 return (if_get_counter_default(ifp, cnt)); 751 } 752 753 } 754 755 static void 756 nicvf_media_status(if_t ifp, struct ifmediareq *ifmr) 757 { 758 struct nicvf *nic = if_getsoftc(ifp); 759 760 NICVF_CORE_LOCK(nic); 761 762 ifmr->ifm_status = IFM_AVALID; 763 ifmr->ifm_active = IFM_ETHER; 764 765 if (nic->link_up) { 766 /* Device attached to working network */ 767 ifmr->ifm_status |= IFM_ACTIVE; 768 } 769 770 switch (nic->speed) { 771 case SPEED_10: 772 ifmr->ifm_active |= IFM_10_T; 773 break; 774 case SPEED_100: 775 ifmr->ifm_active |= IFM_100_TX; 776 break; 777 case SPEED_1000: 778 ifmr->ifm_active |= IFM_1000_T; 779 break; 780 case SPEED_10000: 781 ifmr->ifm_active |= IFM_10G_SR; 782 break; 783 case SPEED_40000: 784 ifmr->ifm_active |= IFM_40G_CR4; 785 break; 786 default: 787 ifmr->ifm_active |= IFM_AUTO; 788 break; 789 } 790 791 if (nic->duplex) 792 ifmr->ifm_active |= IFM_FDX; 793 else 794 ifmr->ifm_active |= IFM_HDX; 795 796 NICVF_CORE_UNLOCK(nic); 797 } 798 799 static int 800 nicvf_media_change(if_t ifp __unused) 801 { 802 803 return (0); 804 } 805 806 /* Register read/write APIs */ 807 void 808 nicvf_reg_write(struct nicvf *nic, bus_space_handle_t offset, uint64_t val) 809 { 810 811 bus_write_8(nic->reg_base, offset, val); 812 } 813 814 uint64_t 815 nicvf_reg_read(struct nicvf *nic, uint64_t offset) 816 { 817 818 return (bus_read_8(nic->reg_base, offset)); 819 } 820 821 void 822 nicvf_queue_reg_write(struct nicvf *nic, bus_space_handle_t offset, 823 uint64_t qidx, uint64_t val) 824 { 825 826 bus_write_8(nic->reg_base, offset + (qidx << NIC_Q_NUM_SHIFT), val); 827 } 828 829 uint64_t 830 nicvf_queue_reg_read(struct nicvf *nic, bus_space_handle_t offset, 831 uint64_t qidx) 832 { 833 834 return (bus_read_8(nic->reg_base, offset + (qidx << NIC_Q_NUM_SHIFT))); 835 } 836 837 /* VF -> PF mailbox communication */ 838 static void 839 nicvf_write_to_mbx(struct nicvf *nic, union nic_mbx *mbx) 840 { 841 uint64_t *msg = (uint64_t *)mbx; 842 843 nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 0, msg[0]); 844 nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 8, msg[1]); 845 } 846 847 int 848 nicvf_send_msg_to_pf(struct nicvf *nic, union nic_mbx *mbx) 849 { 850 int timeout = NIC_MBOX_MSG_TIMEOUT * 10; 851 int sleep = 2; 852 853 NICVF_CORE_LOCK_ASSERT(nic); 854 855 nic->pf_acked = FALSE; 856 nic->pf_nacked = FALSE; 857 858 nicvf_write_to_mbx(nic, mbx); 859 860 /* Wait for previous message to be acked, timeout 2sec */ 861 while (!nic->pf_acked) { 862 if (nic->pf_nacked) 863 return (EINVAL); 864 865 DELAY(sleep * 1000); 866 867 if (nic->pf_acked) 868 break; 869 timeout -= sleep; 870 if (!timeout) { 871 device_printf(nic->dev, 872 "PF didn't ack to mbox msg %d from VF%d\n", 873 (mbx->msg.msg & 0xFF), nic->vf_id); 874 875 return (EBUSY); 876 } 877 } 878 return (0); 879 } 880 881 /* 882 * Checks if VF is able to comminicate with PF 883 * and also gets the VNIC number this VF is associated to. 884 */ 885 static int 886 nicvf_check_pf_ready(struct nicvf *nic) 887 { 888 union nic_mbx mbx = {}; 889 890 mbx.msg.msg = NIC_MBOX_MSG_READY; 891 if (nicvf_send_msg_to_pf(nic, &mbx)) { 892 device_printf(nic->dev, 893 "PF didn't respond to READY msg\n"); 894 return 0; 895 } 896 897 return 1; 898 } 899 900 static void 901 nicvf_read_bgx_stats(struct nicvf *nic, struct bgx_stats_msg *bgx) 902 { 903 904 if (bgx->rx) 905 nic->bgx_stats.rx_stats[bgx->idx] = bgx->stats; 906 else 907 nic->bgx_stats.tx_stats[bgx->idx] = bgx->stats; 908 } 909 910 static void 911 nicvf_handle_mbx_intr(struct nicvf *nic) 912 { 913 union nic_mbx mbx = {}; 914 uint64_t *mbx_data; 915 uint64_t mbx_addr; 916 int i; 917 918 mbx_addr = NIC_VF_PF_MAILBOX_0_1; 919 mbx_data = (uint64_t *)&mbx; 920 921 for (i = 0; i < NIC_PF_VF_MAILBOX_SIZE; i++) { 922 *mbx_data = nicvf_reg_read(nic, mbx_addr); 923 mbx_data++; 924 mbx_addr += sizeof(uint64_t); 925 } 926 927 switch (mbx.msg.msg) { 928 case NIC_MBOX_MSG_READY: 929 nic->pf_acked = TRUE; 930 nic->vf_id = mbx.nic_cfg.vf_id & 0x7F; 931 nic->tns_mode = mbx.nic_cfg.tns_mode & 0x7F; 932 nic->node = mbx.nic_cfg.node_id; 933 memcpy(nic->hwaddr, mbx.nic_cfg.mac_addr, ETHER_ADDR_LEN); 934 nic->loopback_supported = mbx.nic_cfg.loopback_supported; 935 nic->link_up = FALSE; 936 nic->duplex = 0; 937 nic->speed = 0; 938 break; 939 case NIC_MBOX_MSG_ACK: 940 nic->pf_acked = TRUE; 941 break; 942 case NIC_MBOX_MSG_NACK: 943 nic->pf_nacked = TRUE; 944 break; 945 case NIC_MBOX_MSG_RSS_SIZE: 946 nic->rss_info.rss_size = mbx.rss_size.ind_tbl_size; 947 nic->pf_acked = TRUE; 948 break; 949 case NIC_MBOX_MSG_BGX_STATS: 950 nicvf_read_bgx_stats(nic, &mbx.bgx_stats); 951 nic->pf_acked = TRUE; 952 break; 953 case NIC_MBOX_MSG_BGX_LINK_CHANGE: 954 nic->pf_acked = TRUE; 955 nic->link_up = mbx.link_status.link_up; 956 nic->duplex = mbx.link_status.duplex; 957 nic->speed = mbx.link_status.speed; 958 if (nic->link_up) { 959 if_setbaudrate(nic->ifp, nic->speed * 1000000); 960 if_link_state_change(nic->ifp, LINK_STATE_UP); 961 } else { 962 if_setbaudrate(nic->ifp, 0); 963 if_link_state_change(nic->ifp, LINK_STATE_DOWN); 964 } 965 break; 966 default: 967 device_printf(nic->dev, 968 "Invalid message from PF, msg 0x%x\n", mbx.msg.msg); 969 break; 970 } 971 nicvf_clear_intr(nic, NICVF_INTR_MBOX, 0); 972 } 973 974 static int 975 nicvf_update_hw_max_frs(struct nicvf *nic, int mtu) 976 { 977 union nic_mbx mbx = {}; 978 979 mbx.frs.msg = NIC_MBOX_MSG_SET_MAX_FRS; 980 mbx.frs.max_frs = mtu; 981 mbx.frs.vf_id = nic->vf_id; 982 983 return nicvf_send_msg_to_pf(nic, &mbx); 984 } 985 986 static int 987 nicvf_hw_set_mac_addr(struct nicvf *nic, uint8_t *hwaddr) 988 { 989 union nic_mbx mbx = {}; 990 991 mbx.mac.msg = NIC_MBOX_MSG_SET_MAC; 992 mbx.mac.vf_id = nic->vf_id; 993 memcpy(mbx.mac.mac_addr, hwaddr, ETHER_ADDR_LEN); 994 995 return (nicvf_send_msg_to_pf(nic, &mbx)); 996 } 997 998 static void 999 nicvf_config_cpi(struct nicvf *nic) 1000 { 1001 union nic_mbx mbx = {}; 1002 1003 mbx.cpi_cfg.msg = NIC_MBOX_MSG_CPI_CFG; 1004 mbx.cpi_cfg.vf_id = nic->vf_id; 1005 mbx.cpi_cfg.cpi_alg = nic->cpi_alg; 1006 mbx.cpi_cfg.rq_cnt = nic->qs->rq_cnt; 1007 1008 nicvf_send_msg_to_pf(nic, &mbx); 1009 } 1010 1011 static void 1012 nicvf_get_rss_size(struct nicvf *nic) 1013 { 1014 union nic_mbx mbx = {}; 1015 1016 mbx.rss_size.msg = NIC_MBOX_MSG_RSS_SIZE; 1017 mbx.rss_size.vf_id = nic->vf_id; 1018 nicvf_send_msg_to_pf(nic, &mbx); 1019 } 1020 1021 static void 1022 nicvf_config_rss(struct nicvf *nic) 1023 { 1024 union nic_mbx mbx = {}; 1025 struct nicvf_rss_info *rss; 1026 int ind_tbl_len; 1027 int i, nextq; 1028 1029 rss = &nic->rss_info; 1030 ind_tbl_len = rss->rss_size; 1031 nextq = 0; 1032 1033 mbx.rss_cfg.vf_id = nic->vf_id; 1034 mbx.rss_cfg.hash_bits = rss->hash_bits; 1035 while (ind_tbl_len != 0) { 1036 mbx.rss_cfg.tbl_offset = nextq; 1037 mbx.rss_cfg.tbl_len = MIN(ind_tbl_len, 1038 RSS_IND_TBL_LEN_PER_MBX_MSG); 1039 mbx.rss_cfg.msg = mbx.rss_cfg.tbl_offset ? 1040 NIC_MBOX_MSG_RSS_CFG_CONT : NIC_MBOX_MSG_RSS_CFG; 1041 1042 for (i = 0; i < mbx.rss_cfg.tbl_len; i++) 1043 mbx.rss_cfg.ind_tbl[i] = rss->ind_tbl[nextq++]; 1044 1045 nicvf_send_msg_to_pf(nic, &mbx); 1046 1047 ind_tbl_len -= mbx.rss_cfg.tbl_len; 1048 } 1049 } 1050 1051 static void 1052 nicvf_set_rss_key(struct nicvf *nic) 1053 { 1054 struct nicvf_rss_info *rss; 1055 uint64_t key_addr; 1056 int idx; 1057 1058 rss = &nic->rss_info; 1059 key_addr = NIC_VNIC_RSS_KEY_0_4; 1060 1061 for (idx = 0; idx < RSS_HASH_KEY_SIZE; idx++) { 1062 nicvf_reg_write(nic, key_addr, rss->key[idx]); 1063 key_addr += sizeof(uint64_t); 1064 } 1065 } 1066 1067 static int 1068 nicvf_rss_init(struct nicvf *nic) 1069 { 1070 struct nicvf_rss_info *rss; 1071 int idx; 1072 1073 nicvf_get_rss_size(nic); 1074 1075 rss = &nic->rss_info; 1076 if (nic->cpi_alg != CPI_ALG_NONE) { 1077 rss->enable = FALSE; 1078 rss->hash_bits = 0; 1079 return (ENXIO); 1080 } 1081 1082 rss->enable = TRUE; 1083 1084 /* Using the HW reset value for now */ 1085 rss->key[0] = 0xFEED0BADFEED0BADUL; 1086 rss->key[1] = 0xFEED0BADFEED0BADUL; 1087 rss->key[2] = 0xFEED0BADFEED0BADUL; 1088 rss->key[3] = 0xFEED0BADFEED0BADUL; 1089 rss->key[4] = 0xFEED0BADFEED0BADUL; 1090 1091 nicvf_set_rss_key(nic); 1092 1093 rss->cfg = RSS_IP_HASH_ENA | RSS_TCP_HASH_ENA | RSS_UDP_HASH_ENA; 1094 nicvf_reg_write(nic, NIC_VNIC_RSS_CFG, rss->cfg); 1095 1096 rss->hash_bits = fls(rss->rss_size) - 1; 1097 for (idx = 0; idx < rss->rss_size; idx++) 1098 rss->ind_tbl[idx] = idx % nic->rx_queues; 1099 1100 nicvf_config_rss(nic); 1101 1102 return (0); 1103 } 1104 1105 static int 1106 nicvf_init_resources(struct nicvf *nic) 1107 { 1108 int err; 1109 union nic_mbx mbx = {}; 1110 1111 mbx.msg.msg = NIC_MBOX_MSG_CFG_DONE; 1112 1113 /* Enable Qset */ 1114 nicvf_qset_config(nic, TRUE); 1115 1116 /* Initialize queues and HW for data transfer */ 1117 err = nicvf_config_data_transfer(nic, TRUE); 1118 if (err) { 1119 device_printf(nic->dev, 1120 "Failed to alloc/config VF's QSet resources\n"); 1121 return (err); 1122 } 1123 1124 /* Send VF config done msg to PF */ 1125 nicvf_write_to_mbx(nic, &mbx); 1126 1127 return (0); 1128 } 1129 1130 static void 1131 nicvf_misc_intr_handler(void *arg) 1132 { 1133 struct nicvf *nic = (struct nicvf *)arg; 1134 uint64_t intr; 1135 1136 intr = nicvf_reg_read(nic, NIC_VF_INT); 1137 /* Check for spurious interrupt */ 1138 if (!(intr & NICVF_INTR_MBOX_MASK)) 1139 return; 1140 1141 nicvf_handle_mbx_intr(nic); 1142 } 1143 1144 static int 1145 nicvf_intr_handler(void *arg) 1146 { 1147 struct nicvf *nic; 1148 struct cmp_queue *cq; 1149 int qidx; 1150 1151 cq = (struct cmp_queue *)arg; 1152 nic = cq->nic; 1153 qidx = cq->idx; 1154 1155 /* Disable interrupts */ 1156 nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx); 1157 1158 taskqueue_enqueue(cq->cmp_taskq, &cq->cmp_task); 1159 1160 /* Clear interrupt */ 1161 nicvf_clear_intr(nic, NICVF_INTR_CQ, qidx); 1162 1163 return (FILTER_HANDLED); 1164 } 1165 1166 static void 1167 nicvf_rbdr_intr_handler(void *arg) 1168 { 1169 struct nicvf *nic; 1170 struct queue_set *qs; 1171 struct rbdr *rbdr; 1172 int qidx; 1173 1174 nic = (struct nicvf *)arg; 1175 1176 /* Disable RBDR interrupt and schedule softirq */ 1177 for (qidx = 0; qidx < nic->qs->rbdr_cnt; qidx++) { 1178 if (!nicvf_is_intr_enabled(nic, NICVF_INTR_RBDR, qidx)) 1179 continue; 1180 nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx); 1181 1182 qs = nic->qs; 1183 rbdr = &qs->rbdr[qidx]; 1184 taskqueue_enqueue(rbdr->rbdr_taskq, &rbdr->rbdr_task_nowait); 1185 /* Clear interrupt */ 1186 nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx); 1187 } 1188 } 1189 1190 static void 1191 nicvf_qs_err_intr_handler(void *arg) 1192 { 1193 struct nicvf *nic = (struct nicvf *)arg; 1194 struct queue_set *qs = nic->qs; 1195 1196 /* Disable Qset err interrupt and schedule softirq */ 1197 nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0); 1198 taskqueue_enqueue(qs->qs_err_taskq, &qs->qs_err_task); 1199 nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0); 1200 1201 } 1202 1203 static int 1204 nicvf_enable_msix(struct nicvf *nic) 1205 { 1206 struct pci_devinfo *dinfo; 1207 int rid, count; 1208 int ret; 1209 1210 dinfo = device_get_ivars(nic->dev); 1211 rid = dinfo->cfg.msix.msix_table_bar; 1212 nic->msix_table_res = 1213 bus_alloc_resource_any(nic->dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); 1214 if (nic->msix_table_res == NULL) { 1215 device_printf(nic->dev, 1216 "Could not allocate memory for MSI-X table\n"); 1217 return (ENXIO); 1218 } 1219 1220 count = nic->num_vec = NIC_VF_MSIX_VECTORS; 1221 1222 ret = pci_alloc_msix(nic->dev, &count); 1223 if ((ret != 0) || (count != nic->num_vec)) { 1224 device_printf(nic->dev, 1225 "Request for #%d msix vectors failed, error: %d\n", 1226 nic->num_vec, ret); 1227 return (ret); 1228 } 1229 1230 nic->msix_enabled = 1; 1231 return (0); 1232 } 1233 1234 static void 1235 nicvf_disable_msix(struct nicvf *nic) 1236 { 1237 1238 if (nic->msix_enabled) { 1239 pci_release_msi(nic->dev); 1240 nic->msix_enabled = 0; 1241 nic->num_vec = 0; 1242 } 1243 } 1244 1245 static void 1246 nicvf_release_all_interrupts(struct nicvf *nic) 1247 { 1248 struct resource *res; 1249 int irq; 1250 int err __diagused; 1251 1252 /* Free registered interrupts */ 1253 for (irq = 0; irq < nic->num_vec; irq++) { 1254 res = nic->msix_entries[irq].irq_res; 1255 if (res == NULL) 1256 continue; 1257 /* Teardown interrupt first */ 1258 if (nic->msix_entries[irq].handle != NULL) { 1259 err = bus_teardown_intr(nic->dev, 1260 nic->msix_entries[irq].irq_res, 1261 nic->msix_entries[irq].handle); 1262 KASSERT(err == 0, 1263 ("ERROR: Unable to teardown interrupt %d", irq)); 1264 nic->msix_entries[irq].handle = NULL; 1265 } 1266 1267 bus_release_resource(nic->dev, SYS_RES_IRQ, 1268 rman_get_rid(res), nic->msix_entries[irq].irq_res); 1269 nic->msix_entries[irq].irq_res = NULL; 1270 } 1271 /* Disable MSI-X */ 1272 nicvf_disable_msix(nic); 1273 } 1274 1275 /* 1276 * Initialize MSIX vectors and register MISC interrupt. 1277 * Send READY message to PF to check if its alive 1278 */ 1279 static int 1280 nicvf_allocate_misc_interrupt(struct nicvf *nic) 1281 { 1282 struct resource *res; 1283 int irq, rid; 1284 int ret = 0; 1285 1286 /* Return if mailbox interrupt is already registered */ 1287 if (nic->msix_enabled) 1288 return (0); 1289 1290 /* Enable MSI-X */ 1291 if (nicvf_enable_msix(nic) != 0) 1292 return (ENXIO); 1293 1294 irq = NICVF_INTR_ID_MISC; 1295 rid = irq + 1; 1296 nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev, 1297 SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE)); 1298 if (nic->msix_entries[irq].irq_res == NULL) { 1299 device_printf(nic->dev, 1300 "Could not allocate Mbox interrupt for VF%d\n", 1301 device_get_unit(nic->dev)); 1302 return (ENXIO); 1303 } 1304 1305 ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res, 1306 (INTR_MPSAFE | INTR_TYPE_MISC), NULL, nicvf_misc_intr_handler, nic, 1307 &nic->msix_entries[irq].handle); 1308 if (ret != 0) { 1309 res = nic->msix_entries[irq].irq_res; 1310 bus_release_resource(nic->dev, SYS_RES_IRQ, 1311 rman_get_rid(res), res); 1312 nic->msix_entries[irq].irq_res = NULL; 1313 return (ret); 1314 } 1315 1316 return (0); 1317 } 1318 1319 static int 1320 nicvf_enable_misc_interrupt(struct nicvf *nic) 1321 { 1322 1323 /* Enable mailbox interrupt */ 1324 nicvf_enable_intr(nic, NICVF_INTR_MBOX, 0); 1325 1326 /* Check if VF is able to communicate with PF */ 1327 if (!nicvf_check_pf_ready(nic)) { 1328 nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0); 1329 return (ENXIO); 1330 } 1331 1332 return (0); 1333 } 1334 1335 static void 1336 nicvf_release_net_interrupts(struct nicvf *nic) 1337 { 1338 struct resource *res; 1339 int irq; 1340 int err; 1341 1342 for_each_cq_irq(irq) { 1343 res = nic->msix_entries[irq].irq_res; 1344 if (res == NULL) 1345 continue; 1346 /* Teardown active interrupts first */ 1347 if (nic->msix_entries[irq].handle != NULL) { 1348 err = bus_teardown_intr(nic->dev, 1349 nic->msix_entries[irq].irq_res, 1350 nic->msix_entries[irq].handle); 1351 KASSERT(err == 0, 1352 ("ERROR: Unable to teardown CQ interrupt %d", 1353 (irq - NICVF_INTR_ID_CQ))); 1354 if (err != 0) 1355 continue; 1356 } 1357 1358 /* Release resource */ 1359 bus_release_resource(nic->dev, SYS_RES_IRQ, rman_get_rid(res), 1360 res); 1361 nic->msix_entries[irq].irq_res = NULL; 1362 } 1363 1364 for_each_rbdr_irq(irq) { 1365 res = nic->msix_entries[irq].irq_res; 1366 if (res == NULL) 1367 continue; 1368 /* Teardown active interrupts first */ 1369 if (nic->msix_entries[irq].handle != NULL) { 1370 err = bus_teardown_intr(nic->dev, 1371 nic->msix_entries[irq].irq_res, 1372 nic->msix_entries[irq].handle); 1373 KASSERT(err == 0, 1374 ("ERROR: Unable to teardown RDBR interrupt %d", 1375 (irq - NICVF_INTR_ID_RBDR))); 1376 if (err != 0) 1377 continue; 1378 } 1379 1380 /* Release resource */ 1381 bus_release_resource(nic->dev, SYS_RES_IRQ, rman_get_rid(res), 1382 res); 1383 nic->msix_entries[irq].irq_res = NULL; 1384 } 1385 1386 irq = NICVF_INTR_ID_QS_ERR; 1387 res = nic->msix_entries[irq].irq_res; 1388 if (res != NULL) { 1389 /* Teardown active interrupts first */ 1390 if (nic->msix_entries[irq].handle != NULL) { 1391 err = bus_teardown_intr(nic->dev, 1392 nic->msix_entries[irq].irq_res, 1393 nic->msix_entries[irq].handle); 1394 KASSERT(err == 0, 1395 ("ERROR: Unable to teardown QS Error interrupt %d", 1396 irq)); 1397 if (err != 0) 1398 return; 1399 } 1400 1401 /* Release resource */ 1402 bus_release_resource(nic->dev, SYS_RES_IRQ, rman_get_rid(res), 1403 res); 1404 nic->msix_entries[irq].irq_res = NULL; 1405 } 1406 } 1407 1408 static int 1409 nicvf_allocate_net_interrupts(struct nicvf *nic) 1410 { 1411 u_int cpuid; 1412 int irq, rid; 1413 int qidx; 1414 int ret = 0; 1415 1416 /* MSI-X must be configured by now */ 1417 if (!nic->msix_enabled) { 1418 device_printf(nic->dev, "Cannot alloacte queue interrups. " 1419 "MSI-X interrupts disabled.\n"); 1420 return (ENXIO); 1421 } 1422 1423 /* Register CQ interrupts */ 1424 for_each_cq_irq(irq) { 1425 if (irq >= (NICVF_INTR_ID_CQ + nic->qs->cq_cnt)) 1426 break; 1427 1428 qidx = irq - NICVF_INTR_ID_CQ; 1429 rid = irq + 1; 1430 nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev, 1431 SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE)); 1432 if (nic->msix_entries[irq].irq_res == NULL) { 1433 device_printf(nic->dev, 1434 "Could not allocate CQ interrupt %d for VF%d\n", 1435 (irq - NICVF_INTR_ID_CQ), device_get_unit(nic->dev)); 1436 ret = ENXIO; 1437 goto error; 1438 } 1439 ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res, 1440 (INTR_MPSAFE | INTR_TYPE_NET), nicvf_intr_handler, 1441 NULL, &nic->qs->cq[qidx], &nic->msix_entries[irq].handle); 1442 if (ret != 0) { 1443 device_printf(nic->dev, 1444 "Could not setup CQ interrupt %d for VF%d\n", 1445 (irq - NICVF_INTR_ID_CQ), device_get_unit(nic->dev)); 1446 goto error; 1447 } 1448 cpuid = (device_get_unit(nic->dev) * CMP_QUEUE_CNT) + qidx; 1449 cpuid %= mp_ncpus; 1450 /* 1451 * Save CPU ID for later use when system-wide RSS is enabled. 1452 * It will be used to pit the CQ task to the same CPU that got 1453 * interrupted. 1454 */ 1455 nic->qs->cq[qidx].cmp_cpuid = cpuid; 1456 if (bootverbose) { 1457 device_printf(nic->dev, "bind CQ%d IRQ to CPU%d\n", 1458 qidx, cpuid); 1459 } 1460 /* Bind interrupts to the given CPU */ 1461 bus_bind_intr(nic->dev, nic->msix_entries[irq].irq_res, cpuid); 1462 } 1463 1464 /* Register RBDR interrupt */ 1465 for_each_rbdr_irq(irq) { 1466 if (irq >= (NICVF_INTR_ID_RBDR + nic->qs->rbdr_cnt)) 1467 break; 1468 1469 rid = irq + 1; 1470 nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev, 1471 SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE)); 1472 if (nic->msix_entries[irq].irq_res == NULL) { 1473 device_printf(nic->dev, 1474 "Could not allocate RBDR interrupt %d for VF%d\n", 1475 (irq - NICVF_INTR_ID_RBDR), 1476 device_get_unit(nic->dev)); 1477 ret = ENXIO; 1478 goto error; 1479 } 1480 ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res, 1481 (INTR_MPSAFE | INTR_TYPE_NET), NULL, 1482 nicvf_rbdr_intr_handler, nic, 1483 &nic->msix_entries[irq].handle); 1484 if (ret != 0) { 1485 device_printf(nic->dev, 1486 "Could not setup RBDR interrupt %d for VF%d\n", 1487 (irq - NICVF_INTR_ID_RBDR), 1488 device_get_unit(nic->dev)); 1489 goto error; 1490 } 1491 } 1492 1493 /* Register QS error interrupt */ 1494 irq = NICVF_INTR_ID_QS_ERR; 1495 rid = irq + 1; 1496 nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev, 1497 SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE)); 1498 if (nic->msix_entries[irq].irq_res == NULL) { 1499 device_printf(nic->dev, 1500 "Could not allocate QS Error interrupt for VF%d\n", 1501 device_get_unit(nic->dev)); 1502 ret = ENXIO; 1503 goto error; 1504 } 1505 ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res, 1506 (INTR_MPSAFE | INTR_TYPE_NET), NULL, nicvf_qs_err_intr_handler, 1507 nic, &nic->msix_entries[irq].handle); 1508 if (ret != 0) { 1509 device_printf(nic->dev, 1510 "Could not setup QS Error interrupt for VF%d\n", 1511 device_get_unit(nic->dev)); 1512 goto error; 1513 } 1514 1515 return (0); 1516 error: 1517 nicvf_release_net_interrupts(nic); 1518 return (ret); 1519 } 1520 1521 static int 1522 nicvf_stop_locked(struct nicvf *nic) 1523 { 1524 if_t ifp; 1525 int qidx; 1526 struct queue_set *qs = nic->qs; 1527 union nic_mbx mbx = {}; 1528 1529 NICVF_CORE_LOCK_ASSERT(nic); 1530 /* Stop callout. Can block here since holding SX lock */ 1531 callout_drain(&nic->stats_callout); 1532 1533 ifp = nic->ifp; 1534 1535 mbx.msg.msg = NIC_MBOX_MSG_SHUTDOWN; 1536 nicvf_send_msg_to_pf(nic, &mbx); 1537 1538 /* Disable RBDR & QS error interrupts */ 1539 for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) { 1540 nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx); 1541 nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx); 1542 } 1543 nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0); 1544 nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0); 1545 1546 /* Deactivate network interface */ 1547 if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); 1548 1549 /* Free resources */ 1550 nicvf_config_data_transfer(nic, FALSE); 1551 1552 /* Disable HW Qset */ 1553 nicvf_qset_config(nic, FALSE); 1554 1555 /* disable mailbox interrupt */ 1556 nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0); 1557 1558 return (0); 1559 } 1560 1561 static void 1562 nicvf_update_stats(struct nicvf *nic) 1563 { 1564 int qidx; 1565 struct nicvf_hw_stats *stats = &nic->hw_stats; 1566 struct nicvf_drv_stats *drv_stats = &nic->drv_stats; 1567 struct queue_set *qs = nic->qs; 1568 1569 #define GET_RX_STATS(reg) \ 1570 nicvf_reg_read(nic, NIC_VNIC_RX_STAT_0_13 | ((reg) << 3)) 1571 #define GET_TX_STATS(reg) \ 1572 nicvf_reg_read(nic, NIC_VNIC_TX_STAT_0_4 | ((reg) << 3)) 1573 1574 stats->rx_bytes = GET_RX_STATS(RX_OCTS); 1575 stats->rx_ucast_frames = GET_RX_STATS(RX_UCAST); 1576 stats->rx_bcast_frames = GET_RX_STATS(RX_BCAST); 1577 stats->rx_mcast_frames = GET_RX_STATS(RX_MCAST); 1578 stats->rx_fcs_errors = GET_RX_STATS(RX_FCS); 1579 stats->rx_l2_errors = GET_RX_STATS(RX_L2ERR); 1580 stats->rx_drop_red = GET_RX_STATS(RX_RED); 1581 stats->rx_drop_red_bytes = GET_RX_STATS(RX_RED_OCTS); 1582 stats->rx_drop_overrun = GET_RX_STATS(RX_ORUN); 1583 stats->rx_drop_overrun_bytes = GET_RX_STATS(RX_ORUN_OCTS); 1584 stats->rx_drop_bcast = GET_RX_STATS(RX_DRP_BCAST); 1585 stats->rx_drop_mcast = GET_RX_STATS(RX_DRP_MCAST); 1586 stats->rx_drop_l3_bcast = GET_RX_STATS(RX_DRP_L3BCAST); 1587 stats->rx_drop_l3_mcast = GET_RX_STATS(RX_DRP_L3MCAST); 1588 1589 stats->tx_bytes_ok = GET_TX_STATS(TX_OCTS); 1590 stats->tx_ucast_frames_ok = GET_TX_STATS(TX_UCAST); 1591 stats->tx_bcast_frames_ok = GET_TX_STATS(TX_BCAST); 1592 stats->tx_mcast_frames_ok = GET_TX_STATS(TX_MCAST); 1593 stats->tx_drops = GET_TX_STATS(TX_DROP); 1594 1595 drv_stats->tx_frames_ok = stats->tx_ucast_frames_ok + 1596 stats->tx_bcast_frames_ok + stats->tx_mcast_frames_ok; 1597 drv_stats->rx_drops = stats->rx_drop_red + stats->rx_drop_overrun; 1598 drv_stats->tx_drops = stats->tx_drops; 1599 1600 /* Update RQ and SQ stats */ 1601 for (qidx = 0; qidx < qs->rq_cnt; qidx++) 1602 nicvf_update_rq_stats(nic, qidx); 1603 for (qidx = 0; qidx < qs->sq_cnt; qidx++) 1604 nicvf_update_sq_stats(nic, qidx); 1605 } 1606 1607 static void 1608 nicvf_tick_stats(void *arg) 1609 { 1610 struct nicvf *nic; 1611 1612 nic = (struct nicvf *)arg; 1613 1614 /* Read the statistics */ 1615 nicvf_update_stats(nic); 1616 1617 callout_reset(&nic->stats_callout, hz, nicvf_tick_stats, nic); 1618 } 1619