1 /* 2 * Copyright (C) 2015 Cavium Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 */ 27 #include <sys/cdefs.h> 28 #include "opt_inet.h" 29 #include "opt_inet6.h" 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/bitset.h> 34 #include <sys/bitstring.h> 35 #include <sys/bus.h> 36 #include <sys/endian.h> 37 #include <sys/kernel.h> 38 #include <sys/malloc.h> 39 #include <sys/mbuf.h> 40 #include <sys/module.h> 41 #include <sys/rman.h> 42 #include <sys/pciio.h> 43 #include <sys/pcpu.h> 44 #include <sys/proc.h> 45 #include <sys/socket.h> 46 #include <sys/sockio.h> 47 #include <sys/stdatomic.h> 48 #include <sys/cpuset.h> 49 #include <sys/lock.h> 50 #include <sys/mutex.h> 51 #include <sys/smp.h> 52 #include <sys/taskqueue.h> 53 54 #include <net/bpf.h> 55 #include <net/ethernet.h> 56 #include <net/if.h> 57 #include <net/if_var.h> 58 #include <net/if_arp.h> 59 #include <net/if_dl.h> 60 #include <net/if_media.h> 61 #include <net/if_types.h> 62 #include <net/if_vlan_var.h> 63 #include <net/rss_config.h> 64 65 #include <netinet/in.h> 66 #include <netinet/ip.h> 67 #include <netinet/if_ether.h> 68 #include <netinet/tcp_lro.h> 69 70 #include <dev/pci/pcireg.h> 71 #include <dev/pci/pcivar.h> 72 73 #include <sys/dnv.h> 74 #include <sys/nv.h> 75 #include <sys/iov_schema.h> 76 77 #include <machine/bus.h> 78 79 #include "thunder_bgx.h" 80 #include "nic_reg.h" 81 #include "nic.h" 82 #include "nicvf_queues.h" 83 84 #define VNIC_VF_DEVSTR "Cavium Thunder NIC Virtual Function Driver" 85 86 #define VNIC_VF_REG_RID PCIR_BAR(PCI_CFG_REG_BAR_NUM) 87 88 /* Lock for core interface settings */ 89 #define NICVF_CORE_LOCK_INIT(nic) \ 90 sx_init(&(nic)->core_sx, device_get_nameunit((nic)->dev)) 91 92 #define NICVF_CORE_LOCK_DESTROY(nic) \ 93 sx_destroy(&(nic)->core_sx) 94 95 #define NICVF_CORE_LOCK(nic) sx_xlock(&(nic)->core_sx) 96 #define NICVF_CORE_UNLOCK(nic) sx_xunlock(&(nic)->core_sx) 97 98 #define NICVF_CORE_LOCK_ASSERT(nic) sx_assert(&(nic)->core_sx, SA_XLOCKED) 99 100 #define SPEED_10 10 101 #define SPEED_100 100 102 #define SPEED_1000 1000 103 #define SPEED_10000 10000 104 #define SPEED_40000 40000 105 106 MALLOC_DEFINE(M_NICVF, "nicvf", "ThunderX VNIC VF dynamic memory"); 107 108 static int nicvf_probe(device_t); 109 static int nicvf_attach(device_t); 110 static int nicvf_detach(device_t); 111 112 static device_method_t nicvf_methods[] = { 113 /* Device interface */ 114 DEVMETHOD(device_probe, nicvf_probe), 115 DEVMETHOD(device_attach, nicvf_attach), 116 DEVMETHOD(device_detach, nicvf_detach), 117 118 DEVMETHOD_END, 119 }; 120 121 static driver_t nicvf_driver = { 122 "vnic", 123 nicvf_methods, 124 sizeof(struct nicvf), 125 }; 126 127 DRIVER_MODULE(vnicvf, pci, nicvf_driver, 0, 0); 128 MODULE_VERSION(vnicvf, 1); 129 MODULE_DEPEND(vnicvf, pci, 1, 1, 1); 130 MODULE_DEPEND(vnicvf, ether, 1, 1, 1); 131 MODULE_DEPEND(vnicvf, vnicpf, 1, 1, 1); 132 133 static int nicvf_allocate_misc_interrupt(struct nicvf *); 134 static int nicvf_enable_misc_interrupt(struct nicvf *); 135 static int nicvf_allocate_net_interrupts(struct nicvf *); 136 static void nicvf_release_all_interrupts(struct nicvf *); 137 static int nicvf_update_hw_max_frs(struct nicvf *, int); 138 static int nicvf_hw_set_mac_addr(struct nicvf *, uint8_t *); 139 static void nicvf_config_cpi(struct nicvf *); 140 static int nicvf_rss_init(struct nicvf *); 141 static int nicvf_init_resources(struct nicvf *); 142 143 static void nicvf_setup_ifnet(struct nicvf *); 144 static int nicvf_setup_ifmedia(struct nicvf *); 145 static void nicvf_hw_addr_random(uint8_t *); 146 147 static int nicvf_if_ioctl(if_t, u_long, caddr_t); 148 static void nicvf_if_init(void *); 149 static void nicvf_if_init_locked(struct nicvf *); 150 static int nicvf_if_transmit(if_t, struct mbuf *); 151 static void nicvf_if_qflush(if_t); 152 static uint64_t nicvf_if_getcounter(if_t, ift_counter); 153 154 static int nicvf_stop_locked(struct nicvf *); 155 156 static void nicvf_media_status(if_t, struct ifmediareq *); 157 static int nicvf_media_change(if_t); 158 159 static void nicvf_tick_stats(void *); 160 161 static int 162 nicvf_probe(device_t dev) 163 { 164 uint16_t vendor_id; 165 uint16_t device_id; 166 167 vendor_id = pci_get_vendor(dev); 168 device_id = pci_get_device(dev); 169 170 if (vendor_id != PCI_VENDOR_ID_CAVIUM) 171 return (ENXIO); 172 173 if (device_id == PCI_DEVICE_ID_THUNDER_NIC_VF || 174 device_id == PCI_DEVICE_ID_THUNDER_PASS1_NIC_VF) { 175 device_set_desc(dev, VNIC_VF_DEVSTR); 176 return (BUS_PROBE_DEFAULT); 177 } 178 179 return (ENXIO); 180 } 181 182 static int 183 nicvf_attach(device_t dev) 184 { 185 struct nicvf *nic; 186 int rid, qcount; 187 int err = 0; 188 uint8_t hwaddr[ETHER_ADDR_LEN]; 189 uint8_t zeromac[] = {[0 ... (ETHER_ADDR_LEN - 1)] = 0}; 190 191 nic = device_get_softc(dev); 192 nic->dev = dev; 193 nic->pnicvf = nic; 194 195 NICVF_CORE_LOCK_INIT(nic); 196 /* Enable HW TSO on Pass2 */ 197 if (!pass1_silicon(dev)) 198 nic->hw_tso = TRUE; 199 200 rid = VNIC_VF_REG_RID; 201 nic->reg_base = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 202 RF_ACTIVE); 203 if (nic->reg_base == NULL) { 204 device_printf(dev, "Could not allocate registers memory\n"); 205 return (ENXIO); 206 } 207 208 qcount = MAX_CMP_QUEUES_PER_QS; 209 nic->max_queues = qcount; 210 211 err = nicvf_set_qset_resources(nic); 212 if (err != 0) 213 goto err_free_res; 214 215 /* Check if PF is alive and get MAC address for this VF */ 216 err = nicvf_allocate_misc_interrupt(nic); 217 if (err != 0) 218 goto err_free_res; 219 220 NICVF_CORE_LOCK(nic); 221 err = nicvf_enable_misc_interrupt(nic); 222 NICVF_CORE_UNLOCK(nic); 223 if (err != 0) 224 goto err_release_intr; 225 226 err = nicvf_allocate_net_interrupts(nic); 227 if (err != 0) { 228 device_printf(dev, 229 "Could not allocate network interface interrupts\n"); 230 goto err_free_ifnet; 231 } 232 233 /* If no MAC address was obtained we generate random one */ 234 if (memcmp(nic->hwaddr, zeromac, ETHER_ADDR_LEN) == 0) { 235 nicvf_hw_addr_random(hwaddr); 236 memcpy(nic->hwaddr, hwaddr, ETHER_ADDR_LEN); 237 NICVF_CORE_LOCK(nic); 238 nicvf_hw_set_mac_addr(nic, hwaddr); 239 NICVF_CORE_UNLOCK(nic); 240 } 241 242 /* Configure CPI alorithm */ 243 nic->cpi_alg = CPI_ALG_NONE; 244 NICVF_CORE_LOCK(nic); 245 nicvf_config_cpi(nic); 246 /* Configure receive side scaling */ 247 if (nic->qs->rq_cnt > 1) 248 nicvf_rss_init(nic); 249 NICVF_CORE_UNLOCK(nic); 250 251 nicvf_setup_ifnet(nic); 252 253 err = nicvf_setup_ifmedia(nic); 254 if (err != 0) { 255 device_printf(dev, "Could not set-up ifmedia\n"); 256 goto err_free_ifnet; 257 } 258 259 mtx_init(&nic->stats_mtx, "VNIC stats", NULL, MTX_DEF); 260 callout_init_mtx(&nic->stats_callout, &nic->stats_mtx, 0); 261 262 ether_ifattach(nic->ifp, nic->hwaddr); 263 264 return (0); 265 266 err_free_ifnet: 267 if_free(nic->ifp); 268 err_release_intr: 269 nicvf_release_all_interrupts(nic); 270 err_free_res: 271 bus_release_resource(dev, SYS_RES_MEMORY, rman_get_rid(nic->reg_base), 272 nic->reg_base); 273 274 return (err); 275 } 276 277 static int 278 nicvf_detach(device_t dev) 279 { 280 struct nicvf *nic; 281 282 nic = device_get_softc(dev); 283 284 NICVF_CORE_LOCK(nic); 285 /* Shut down the port and release ring resources */ 286 nicvf_stop_locked(nic); 287 /* Release stats lock */ 288 mtx_destroy(&nic->stats_mtx); 289 /* Release interrupts */ 290 nicvf_release_all_interrupts(nic); 291 /* Release memory resource */ 292 if (nic->reg_base != NULL) { 293 bus_release_resource(dev, SYS_RES_MEMORY, 294 rman_get_rid(nic->reg_base), nic->reg_base); 295 } 296 297 /* Remove all ifmedia configurations */ 298 ifmedia_removeall(&nic->if_media); 299 /* Free this ifnet */ 300 if_free(nic->ifp); 301 NICVF_CORE_UNLOCK(nic); 302 /* Finally destroy the lock */ 303 NICVF_CORE_LOCK_DESTROY(nic); 304 305 return (0); 306 } 307 308 static void 309 nicvf_hw_addr_random(uint8_t *hwaddr) 310 { 311 uint32_t rnd; 312 uint8_t addr[ETHER_ADDR_LEN]; 313 314 /* 315 * Create randomized MAC address. 316 * Set 'bsd' + random 24 low-order bits. 317 */ 318 rnd = arc4random() & 0x00ffffff; 319 addr[0] = 'b'; 320 addr[1] = 's'; 321 addr[2] = 'd'; 322 addr[3] = rnd >> 16; 323 addr[4] = rnd >> 8; 324 addr[5] = rnd >> 0; 325 326 memcpy(hwaddr, addr, ETHER_ADDR_LEN); 327 } 328 329 static void 330 nicvf_setup_ifnet(struct nicvf *nic) 331 { 332 if_t ifp; 333 334 ifp = if_alloc(IFT_ETHER); 335 nic->ifp = ifp; 336 337 if_setsoftc(ifp, nic); 338 if_initname(ifp, device_get_name(nic->dev), device_get_unit(nic->dev)); 339 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); 340 341 if_settransmitfn(ifp, nicvf_if_transmit); 342 if_setqflushfn(ifp, nicvf_if_qflush); 343 if_setioctlfn(ifp, nicvf_if_ioctl); 344 if_setinitfn(ifp, nicvf_if_init); 345 if_setgetcounterfn(ifp, nicvf_if_getcounter); 346 347 if_setmtu(ifp, ETHERMTU); 348 349 /* Reset caps */ 350 if_setcapabilities(ifp, 0); 351 352 /* Set the default values */ 353 if_setcapabilitiesbit(ifp, IFCAP_VLAN_MTU | IFCAP_JUMBO_MTU, 0); 354 if_setcapabilitiesbit(ifp, IFCAP_LRO, 0); 355 if (nic->hw_tso) { 356 /* TSO */ 357 if_setcapabilitiesbit(ifp, IFCAP_TSO4, 0); 358 /* TSO parameters */ 359 if_sethwtsomax(ifp, NICVF_TSO_MAXSIZE); 360 if_sethwtsomaxsegcount(ifp, NICVF_TSO_NSEGS); 361 if_sethwtsomaxsegsize(ifp, MCLBYTES); 362 } 363 /* IP/TCP/UDP HW checksums */ 364 if_setcapabilitiesbit(ifp, IFCAP_HWCSUM, 0); 365 if_setcapabilitiesbit(ifp, IFCAP_HWSTATS, 0); 366 /* 367 * HW offload enable 368 */ 369 if_clearhwassist(ifp); 370 if_sethwassistbits(ifp, (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_SCTP), 0); 371 if (nic->hw_tso) 372 if_sethwassistbits(ifp, (CSUM_TSO), 0); 373 if_setcapenable(ifp, if_getcapabilities(ifp)); 374 } 375 376 static int 377 nicvf_setup_ifmedia(struct nicvf *nic) 378 { 379 380 ifmedia_init(&nic->if_media, IFM_IMASK, nicvf_media_change, 381 nicvf_media_status); 382 383 /* 384 * Advertise availability of all possible connection types, 385 * even though not all are possible at the same time. 386 */ 387 388 ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_10_T | IFM_FDX), 389 0, NULL); 390 ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_100_TX | IFM_FDX), 391 0, NULL); 392 ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_1000_T | IFM_FDX), 393 0, NULL); 394 ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_10G_SR | IFM_FDX), 395 0, NULL); 396 ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_40G_CR4 | IFM_FDX), 397 0, NULL); 398 ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_AUTO | IFM_FDX), 399 0, NULL); 400 401 ifmedia_set(&nic->if_media, (IFM_ETHER | IFM_AUTO | IFM_FDX)); 402 403 return (0); 404 } 405 406 static int 407 nicvf_if_ioctl(if_t ifp, u_long cmd, caddr_t data) 408 { 409 struct nicvf *nic; 410 struct rcv_queue *rq; 411 struct ifreq *ifr; 412 uint32_t flags; 413 int mask, err; 414 int rq_idx; 415 #if defined(INET) || defined(INET6) 416 struct ifaddr *ifa; 417 boolean_t avoid_reset = FALSE; 418 #endif 419 420 nic = if_getsoftc(ifp); 421 ifr = (struct ifreq *)data; 422 #if defined(INET) || defined(INET6) 423 ifa = (struct ifaddr *)data; 424 #endif 425 err = 0; 426 switch (cmd) { 427 case SIOCSIFADDR: 428 #ifdef INET 429 if (ifa->ifa_addr->sa_family == AF_INET) 430 avoid_reset = TRUE; 431 #endif 432 #ifdef INET6 433 if (ifa->ifa_addr->sa_family == AF_INET6) 434 avoid_reset = TRUE; 435 #endif 436 437 #if defined(INET) || defined(INET6) 438 /* Avoid reinitialization unless it's necessary */ 439 if (avoid_reset) { 440 if_setflagbits(ifp, IFF_UP, 0); 441 if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) 442 nicvf_if_init(nic); 443 #ifdef INET 444 if (!(if_getflags(ifp) & IFF_NOARP)) 445 arp_ifinit(ifp, ifa); 446 #endif 447 448 return (0); 449 } 450 #endif 451 err = ether_ioctl(ifp, cmd, data); 452 break; 453 case SIOCSIFMTU: 454 if (ifr->ifr_mtu < NIC_HW_MIN_FRS || 455 ifr->ifr_mtu > NIC_HW_MAX_FRS) { 456 err = EINVAL; 457 } else { 458 NICVF_CORE_LOCK(nic); 459 err = nicvf_update_hw_max_frs(nic, ifr->ifr_mtu); 460 if (err == 0) 461 if_setmtu(ifp, ifr->ifr_mtu); 462 NICVF_CORE_UNLOCK(nic); 463 } 464 break; 465 case SIOCSIFFLAGS: 466 NICVF_CORE_LOCK(nic); 467 flags = if_getflags(ifp); 468 if (flags & IFF_UP) { 469 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 470 if ((flags ^ nic->if_flags) & IFF_PROMISC) { 471 /* Change promiscous mode */ 472 #if 0 /* XXX */ 473 nicvf_set_promiscous(nic); 474 #endif 475 } 476 477 if ((flags ^ nic->if_flags) & IFF_ALLMULTI) { 478 /* Change multicasting settings */ 479 #if 0 /* XXX */ 480 nicvf_set_multicast(nic); 481 #endif 482 } 483 } else { 484 nicvf_if_init_locked(nic); 485 } 486 } else if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) 487 nicvf_stop_locked(nic); 488 489 nic->if_flags = flags; 490 NICVF_CORE_UNLOCK(nic); 491 break; 492 493 case SIOCADDMULTI: 494 case SIOCDELMULTI: 495 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 496 #if 0 497 NICVF_CORE_LOCK(nic); 498 /* ARM64TODO */ 499 nicvf_set_multicast(nic); 500 NICVF_CORE_UNLOCK(nic); 501 #endif 502 } 503 break; 504 505 case SIOCSIFMEDIA: 506 case SIOCGIFMEDIA: 507 err = ifmedia_ioctl(ifp, ifr, &nic->if_media, cmd); 508 break; 509 510 case SIOCSIFCAP: 511 mask = if_getcapenable(ifp) ^ ifr->ifr_reqcap; 512 if (mask & IFCAP_VLAN_MTU) { 513 /* No work to do except acknowledge the change took. */ 514 if_togglecapenable(ifp, IFCAP_VLAN_MTU); 515 } 516 if (mask & IFCAP_TXCSUM) 517 if_togglecapenable(ifp, IFCAP_TXCSUM); 518 if (mask & IFCAP_RXCSUM) 519 if_togglecapenable(ifp, IFCAP_RXCSUM); 520 if ((mask & IFCAP_TSO4) && nic->hw_tso) 521 if_togglecapenable(ifp, IFCAP_TSO4); 522 if (mask & IFCAP_LRO) { 523 /* 524 * Lock the driver for a moment to avoid 525 * mismatch in per-queue settings. 526 */ 527 NICVF_CORE_LOCK(nic); 528 if_togglecapenable(ifp, IFCAP_LRO); 529 if ((if_getdrvflags(nic->ifp) & IFF_DRV_RUNNING) != 0) { 530 /* 531 * Now disable LRO for subsequent packets. 532 * Atomicity of this change is not necessary 533 * as we don't need precise toggle of this 534 * feature for all threads processing the 535 * completion queue. 536 */ 537 for (rq_idx = 0; 538 rq_idx < nic->qs->rq_cnt; rq_idx++) { 539 rq = &nic->qs->rq[rq_idx]; 540 rq->lro_enabled = !rq->lro_enabled; 541 } 542 } 543 NICVF_CORE_UNLOCK(nic); 544 } 545 546 break; 547 548 default: 549 err = ether_ioctl(ifp, cmd, data); 550 break; 551 } 552 553 return (err); 554 } 555 556 static void 557 nicvf_if_init_locked(struct nicvf *nic) 558 { 559 struct queue_set *qs = nic->qs; 560 if_t ifp; 561 int qidx; 562 int err; 563 caddr_t if_addr; 564 565 NICVF_CORE_LOCK_ASSERT(nic); 566 ifp = nic->ifp; 567 568 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) 569 nicvf_stop_locked(nic); 570 571 err = nicvf_enable_misc_interrupt(nic); 572 if (err != 0) { 573 if_printf(ifp, "Could not reenable Mbox interrupt\n"); 574 return; 575 } 576 577 /* Get the latest MAC address */ 578 if_addr = if_getlladdr(ifp); 579 /* Update MAC address if changed */ 580 if (memcmp(nic->hwaddr, if_addr, ETHER_ADDR_LEN) != 0) { 581 memcpy(nic->hwaddr, if_addr, ETHER_ADDR_LEN); 582 nicvf_hw_set_mac_addr(nic, if_addr); 583 } 584 585 /* Initialize the queues */ 586 err = nicvf_init_resources(nic); 587 if (err != 0) 588 goto error; 589 590 /* Make sure queue initialization is written */ 591 wmb(); 592 593 nicvf_reg_write(nic, NIC_VF_INT, ~0UL); 594 /* Enable Qset err interrupt */ 595 nicvf_enable_intr(nic, NICVF_INTR_QS_ERR, 0); 596 597 /* Enable completion queue interrupt */ 598 for (qidx = 0; qidx < qs->cq_cnt; qidx++) 599 nicvf_enable_intr(nic, NICVF_INTR_CQ, qidx); 600 601 /* Enable RBDR threshold interrupt */ 602 for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) 603 nicvf_enable_intr(nic, NICVF_INTR_RBDR, qidx); 604 605 nic->drv_stats.txq_stop = 0; 606 nic->drv_stats.txq_wake = 0; 607 608 /* Activate network interface */ 609 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); 610 611 /* Schedule callout to update stats */ 612 callout_reset(&nic->stats_callout, hz, nicvf_tick_stats, nic); 613 614 return; 615 616 error: 617 /* Something went very wrong. Disable this ifnet for good */ 618 if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); 619 } 620 621 static void 622 nicvf_if_init(void *if_softc) 623 { 624 struct nicvf *nic = if_softc; 625 626 NICVF_CORE_LOCK(nic); 627 nicvf_if_init_locked(nic); 628 NICVF_CORE_UNLOCK(nic); 629 } 630 631 static int 632 nicvf_if_transmit(if_t ifp, struct mbuf *mbuf) 633 { 634 struct nicvf *nic = if_getsoftc(ifp); 635 struct queue_set *qs = nic->qs; 636 struct snd_queue *sq; 637 struct mbuf *mtmp; 638 int qidx; 639 int err = 0; 640 641 if (__predict_false(qs == NULL)) { 642 panic("%s: missing queue set for %s", __func__, 643 device_get_nameunit(nic->dev)); 644 } 645 646 /* Select queue */ 647 if (M_HASHTYPE_GET(mbuf) != M_HASHTYPE_NONE) 648 qidx = mbuf->m_pkthdr.flowid % qs->sq_cnt; 649 else 650 qidx = curcpu % qs->sq_cnt; 651 652 sq = &qs->sq[qidx]; 653 654 if (mbuf->m_next != NULL && 655 (mbuf->m_pkthdr.csum_flags & 656 (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_SCTP)) != 0) { 657 if (M_WRITABLE(mbuf) == 0) { 658 mtmp = m_dup(mbuf, M_NOWAIT); 659 m_freem(mbuf); 660 if (mtmp == NULL) 661 return (ENOBUFS); 662 mbuf = mtmp; 663 } 664 } 665 666 err = drbr_enqueue(ifp, sq->br, mbuf); 667 if (((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 668 IFF_DRV_RUNNING) || !nic->link_up || (err != 0)) { 669 /* 670 * Try to enqueue packet to the ring buffer. 671 * If the driver is not active, link down or enqueue operation 672 * failed, return with the appropriate error code. 673 */ 674 return (err); 675 } 676 677 if (NICVF_TX_TRYLOCK(sq) != 0) { 678 err = nicvf_xmit_locked(sq); 679 NICVF_TX_UNLOCK(sq); 680 return (err); 681 } else 682 taskqueue_enqueue(sq->snd_taskq, &sq->snd_task); 683 684 return (0); 685 } 686 687 static void 688 nicvf_if_qflush(if_t ifp) 689 { 690 struct nicvf *nic; 691 struct queue_set *qs; 692 struct snd_queue *sq; 693 struct mbuf *mbuf; 694 size_t idx; 695 696 nic = if_getsoftc(ifp); 697 qs = nic->qs; 698 699 for (idx = 0; idx < qs->sq_cnt; idx++) { 700 sq = &qs->sq[idx]; 701 NICVF_TX_LOCK(sq); 702 while ((mbuf = buf_ring_dequeue_sc(sq->br)) != NULL) 703 m_freem(mbuf); 704 NICVF_TX_UNLOCK(sq); 705 } 706 if_qflush(ifp); 707 } 708 709 static uint64_t 710 nicvf_if_getcounter(if_t ifp, ift_counter cnt) 711 { 712 struct nicvf *nic; 713 struct nicvf_hw_stats *hw_stats; 714 struct nicvf_drv_stats *drv_stats; 715 716 nic = if_getsoftc(ifp); 717 hw_stats = &nic->hw_stats; 718 drv_stats = &nic->drv_stats; 719 720 switch (cnt) { 721 case IFCOUNTER_IPACKETS: 722 return (drv_stats->rx_frames_ok); 723 case IFCOUNTER_OPACKETS: 724 return (drv_stats->tx_frames_ok); 725 case IFCOUNTER_IBYTES: 726 return (hw_stats->rx_bytes); 727 case IFCOUNTER_OBYTES: 728 return (hw_stats->tx_bytes_ok); 729 case IFCOUNTER_IMCASTS: 730 return (hw_stats->rx_mcast_frames); 731 case IFCOUNTER_COLLISIONS: 732 return (0); 733 case IFCOUNTER_IQDROPS: 734 return (drv_stats->rx_drops); 735 case IFCOUNTER_OQDROPS: 736 return (drv_stats->tx_drops); 737 default: 738 return (if_get_counter_default(ifp, cnt)); 739 } 740 741 } 742 743 static void 744 nicvf_media_status(if_t ifp, struct ifmediareq *ifmr) 745 { 746 struct nicvf *nic = if_getsoftc(ifp); 747 748 NICVF_CORE_LOCK(nic); 749 750 ifmr->ifm_status = IFM_AVALID; 751 ifmr->ifm_active = IFM_ETHER; 752 753 if (nic->link_up) { 754 /* Device attached to working network */ 755 ifmr->ifm_status |= IFM_ACTIVE; 756 } 757 758 switch (nic->speed) { 759 case SPEED_10: 760 ifmr->ifm_active |= IFM_10_T; 761 break; 762 case SPEED_100: 763 ifmr->ifm_active |= IFM_100_TX; 764 break; 765 case SPEED_1000: 766 ifmr->ifm_active |= IFM_1000_T; 767 break; 768 case SPEED_10000: 769 ifmr->ifm_active |= IFM_10G_SR; 770 break; 771 case SPEED_40000: 772 ifmr->ifm_active |= IFM_40G_CR4; 773 break; 774 default: 775 ifmr->ifm_active |= IFM_AUTO; 776 break; 777 } 778 779 if (nic->duplex) 780 ifmr->ifm_active |= IFM_FDX; 781 else 782 ifmr->ifm_active |= IFM_HDX; 783 784 NICVF_CORE_UNLOCK(nic); 785 } 786 787 static int 788 nicvf_media_change(if_t ifp __unused) 789 { 790 791 return (0); 792 } 793 794 /* Register read/write APIs */ 795 void 796 nicvf_reg_write(struct nicvf *nic, bus_space_handle_t offset, uint64_t val) 797 { 798 799 bus_write_8(nic->reg_base, offset, val); 800 } 801 802 uint64_t 803 nicvf_reg_read(struct nicvf *nic, uint64_t offset) 804 { 805 806 return (bus_read_8(nic->reg_base, offset)); 807 } 808 809 void 810 nicvf_queue_reg_write(struct nicvf *nic, bus_space_handle_t offset, 811 uint64_t qidx, uint64_t val) 812 { 813 814 bus_write_8(nic->reg_base, offset + (qidx << NIC_Q_NUM_SHIFT), val); 815 } 816 817 uint64_t 818 nicvf_queue_reg_read(struct nicvf *nic, bus_space_handle_t offset, 819 uint64_t qidx) 820 { 821 822 return (bus_read_8(nic->reg_base, offset + (qidx << NIC_Q_NUM_SHIFT))); 823 } 824 825 /* VF -> PF mailbox communication */ 826 static void 827 nicvf_write_to_mbx(struct nicvf *nic, union nic_mbx *mbx) 828 { 829 uint64_t *msg = (uint64_t *)mbx; 830 831 nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 0, msg[0]); 832 nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 8, msg[1]); 833 } 834 835 int 836 nicvf_send_msg_to_pf(struct nicvf *nic, union nic_mbx *mbx) 837 { 838 int timeout = NIC_MBOX_MSG_TIMEOUT * 10; 839 int sleep = 2; 840 841 NICVF_CORE_LOCK_ASSERT(nic); 842 843 nic->pf_acked = FALSE; 844 nic->pf_nacked = FALSE; 845 846 nicvf_write_to_mbx(nic, mbx); 847 848 /* Wait for previous message to be acked, timeout 2sec */ 849 while (!nic->pf_acked) { 850 if (nic->pf_nacked) 851 return (EINVAL); 852 853 DELAY(sleep * 1000); 854 855 if (nic->pf_acked) 856 break; 857 timeout -= sleep; 858 if (!timeout) { 859 device_printf(nic->dev, 860 "PF didn't ack to mbox msg %d from VF%d\n", 861 (mbx->msg.msg & 0xFF), nic->vf_id); 862 863 return (EBUSY); 864 } 865 } 866 return (0); 867 } 868 869 /* 870 * Checks if VF is able to comminicate with PF 871 * and also gets the VNIC number this VF is associated to. 872 */ 873 static int 874 nicvf_check_pf_ready(struct nicvf *nic) 875 { 876 union nic_mbx mbx = {}; 877 878 mbx.msg.msg = NIC_MBOX_MSG_READY; 879 if (nicvf_send_msg_to_pf(nic, &mbx)) { 880 device_printf(nic->dev, 881 "PF didn't respond to READY msg\n"); 882 return 0; 883 } 884 885 return 1; 886 } 887 888 static void 889 nicvf_read_bgx_stats(struct nicvf *nic, struct bgx_stats_msg *bgx) 890 { 891 892 if (bgx->rx) 893 nic->bgx_stats.rx_stats[bgx->idx] = bgx->stats; 894 else 895 nic->bgx_stats.tx_stats[bgx->idx] = bgx->stats; 896 } 897 898 static void 899 nicvf_handle_mbx_intr(struct nicvf *nic) 900 { 901 union nic_mbx mbx = {}; 902 uint64_t *mbx_data; 903 uint64_t mbx_addr; 904 int i; 905 906 mbx_addr = NIC_VF_PF_MAILBOX_0_1; 907 mbx_data = (uint64_t *)&mbx; 908 909 for (i = 0; i < NIC_PF_VF_MAILBOX_SIZE; i++) { 910 *mbx_data = nicvf_reg_read(nic, mbx_addr); 911 mbx_data++; 912 mbx_addr += sizeof(uint64_t); 913 } 914 915 switch (mbx.msg.msg) { 916 case NIC_MBOX_MSG_READY: 917 nic->pf_acked = TRUE; 918 nic->vf_id = mbx.nic_cfg.vf_id & 0x7F; 919 nic->tns_mode = mbx.nic_cfg.tns_mode & 0x7F; 920 nic->node = mbx.nic_cfg.node_id; 921 memcpy(nic->hwaddr, mbx.nic_cfg.mac_addr, ETHER_ADDR_LEN); 922 nic->loopback_supported = mbx.nic_cfg.loopback_supported; 923 nic->link_up = FALSE; 924 nic->duplex = 0; 925 nic->speed = 0; 926 break; 927 case NIC_MBOX_MSG_ACK: 928 nic->pf_acked = TRUE; 929 break; 930 case NIC_MBOX_MSG_NACK: 931 nic->pf_nacked = TRUE; 932 break; 933 case NIC_MBOX_MSG_RSS_SIZE: 934 nic->rss_info.rss_size = mbx.rss_size.ind_tbl_size; 935 nic->pf_acked = TRUE; 936 break; 937 case NIC_MBOX_MSG_BGX_STATS: 938 nicvf_read_bgx_stats(nic, &mbx.bgx_stats); 939 nic->pf_acked = TRUE; 940 break; 941 case NIC_MBOX_MSG_BGX_LINK_CHANGE: 942 nic->pf_acked = TRUE; 943 nic->link_up = mbx.link_status.link_up; 944 nic->duplex = mbx.link_status.duplex; 945 nic->speed = mbx.link_status.speed; 946 if (nic->link_up) { 947 if_setbaudrate(nic->ifp, nic->speed * 1000000); 948 if_link_state_change(nic->ifp, LINK_STATE_UP); 949 } else { 950 if_setbaudrate(nic->ifp, 0); 951 if_link_state_change(nic->ifp, LINK_STATE_DOWN); 952 } 953 break; 954 default: 955 device_printf(nic->dev, 956 "Invalid message from PF, msg 0x%x\n", mbx.msg.msg); 957 break; 958 } 959 nicvf_clear_intr(nic, NICVF_INTR_MBOX, 0); 960 } 961 962 static int 963 nicvf_update_hw_max_frs(struct nicvf *nic, int mtu) 964 { 965 union nic_mbx mbx = {}; 966 967 mbx.frs.msg = NIC_MBOX_MSG_SET_MAX_FRS; 968 mbx.frs.max_frs = mtu; 969 mbx.frs.vf_id = nic->vf_id; 970 971 return nicvf_send_msg_to_pf(nic, &mbx); 972 } 973 974 static int 975 nicvf_hw_set_mac_addr(struct nicvf *nic, uint8_t *hwaddr) 976 { 977 union nic_mbx mbx = {}; 978 979 mbx.mac.msg = NIC_MBOX_MSG_SET_MAC; 980 mbx.mac.vf_id = nic->vf_id; 981 memcpy(mbx.mac.mac_addr, hwaddr, ETHER_ADDR_LEN); 982 983 return (nicvf_send_msg_to_pf(nic, &mbx)); 984 } 985 986 static void 987 nicvf_config_cpi(struct nicvf *nic) 988 { 989 union nic_mbx mbx = {}; 990 991 mbx.cpi_cfg.msg = NIC_MBOX_MSG_CPI_CFG; 992 mbx.cpi_cfg.vf_id = nic->vf_id; 993 mbx.cpi_cfg.cpi_alg = nic->cpi_alg; 994 mbx.cpi_cfg.rq_cnt = nic->qs->rq_cnt; 995 996 nicvf_send_msg_to_pf(nic, &mbx); 997 } 998 999 static void 1000 nicvf_get_rss_size(struct nicvf *nic) 1001 { 1002 union nic_mbx mbx = {}; 1003 1004 mbx.rss_size.msg = NIC_MBOX_MSG_RSS_SIZE; 1005 mbx.rss_size.vf_id = nic->vf_id; 1006 nicvf_send_msg_to_pf(nic, &mbx); 1007 } 1008 1009 static void 1010 nicvf_config_rss(struct nicvf *nic) 1011 { 1012 union nic_mbx mbx = {}; 1013 struct nicvf_rss_info *rss; 1014 int ind_tbl_len; 1015 int i, nextq; 1016 1017 rss = &nic->rss_info; 1018 ind_tbl_len = rss->rss_size; 1019 nextq = 0; 1020 1021 mbx.rss_cfg.vf_id = nic->vf_id; 1022 mbx.rss_cfg.hash_bits = rss->hash_bits; 1023 while (ind_tbl_len != 0) { 1024 mbx.rss_cfg.tbl_offset = nextq; 1025 mbx.rss_cfg.tbl_len = MIN(ind_tbl_len, 1026 RSS_IND_TBL_LEN_PER_MBX_MSG); 1027 mbx.rss_cfg.msg = mbx.rss_cfg.tbl_offset ? 1028 NIC_MBOX_MSG_RSS_CFG_CONT : NIC_MBOX_MSG_RSS_CFG; 1029 1030 for (i = 0; i < mbx.rss_cfg.tbl_len; i++) 1031 mbx.rss_cfg.ind_tbl[i] = rss->ind_tbl[nextq++]; 1032 1033 nicvf_send_msg_to_pf(nic, &mbx); 1034 1035 ind_tbl_len -= mbx.rss_cfg.tbl_len; 1036 } 1037 } 1038 1039 static void 1040 nicvf_set_rss_key(struct nicvf *nic) 1041 { 1042 struct nicvf_rss_info *rss; 1043 uint64_t key_addr; 1044 int idx; 1045 1046 rss = &nic->rss_info; 1047 key_addr = NIC_VNIC_RSS_KEY_0_4; 1048 1049 for (idx = 0; idx < RSS_HASH_KEY_SIZE; idx++) { 1050 nicvf_reg_write(nic, key_addr, rss->key[idx]); 1051 key_addr += sizeof(uint64_t); 1052 } 1053 } 1054 1055 static int 1056 nicvf_rss_init(struct nicvf *nic) 1057 { 1058 struct nicvf_rss_info *rss; 1059 uint8_t rss_key[RSS_KEYSIZE]; 1060 int idx; 1061 1062 nicvf_get_rss_size(nic); 1063 1064 rss = &nic->rss_info; 1065 if (nic->cpi_alg != CPI_ALG_NONE) { 1066 rss->enable = FALSE; 1067 rss->hash_bits = 0; 1068 return (ENXIO); 1069 } 1070 1071 rss->enable = TRUE; 1072 1073 _Static_assert(sizeof(rss->key) == RSS_KEYSIZE, 1074 "RSS key size mismatch"); 1075 rss_getkey(rss_key); 1076 /* Each key register holds eight bytes in network order. */ 1077 for (idx = 0; idx < RSS_HASH_KEY_SIZE; idx++) 1078 rss->key[idx] = be64dec(rss_key + idx * sizeof(uint64_t)); 1079 1080 nicvf_set_rss_key(nic); 1081 1082 rss->cfg = RSS_IP_HASH_ENA | RSS_TCP_HASH_ENA | RSS_UDP_HASH_ENA; 1083 nicvf_reg_write(nic, NIC_VNIC_RSS_CFG, rss->cfg); 1084 1085 rss->hash_bits = fls(rss->rss_size) - 1; 1086 for (idx = 0; idx < rss->rss_size; idx++) 1087 rss->ind_tbl[idx] = idx % nic->rx_queues; 1088 1089 nicvf_config_rss(nic); 1090 1091 return (0); 1092 } 1093 1094 static int 1095 nicvf_init_resources(struct nicvf *nic) 1096 { 1097 int err; 1098 union nic_mbx mbx = {}; 1099 1100 mbx.msg.msg = NIC_MBOX_MSG_CFG_DONE; 1101 1102 /* Enable Qset */ 1103 nicvf_qset_config(nic, TRUE); 1104 1105 /* Initialize queues and HW for data transfer */ 1106 err = nicvf_config_data_transfer(nic, TRUE); 1107 if (err) { 1108 device_printf(nic->dev, 1109 "Failed to alloc/config VF's QSet resources\n"); 1110 return (err); 1111 } 1112 1113 /* Send VF config done msg to PF */ 1114 nicvf_write_to_mbx(nic, &mbx); 1115 1116 return (0); 1117 } 1118 1119 static void 1120 nicvf_misc_intr_handler(void *arg) 1121 { 1122 struct nicvf *nic = (struct nicvf *)arg; 1123 uint64_t intr; 1124 1125 intr = nicvf_reg_read(nic, NIC_VF_INT); 1126 /* Check for spurious interrupt */ 1127 if (!(intr & NICVF_INTR_MBOX_MASK)) 1128 return; 1129 1130 nicvf_handle_mbx_intr(nic); 1131 } 1132 1133 static int 1134 nicvf_intr_handler(void *arg) 1135 { 1136 struct nicvf *nic; 1137 struct cmp_queue *cq; 1138 int qidx; 1139 1140 cq = (struct cmp_queue *)arg; 1141 nic = cq->nic; 1142 qidx = cq->idx; 1143 1144 /* Disable interrupts */ 1145 nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx); 1146 1147 taskqueue_enqueue(cq->cmp_taskq, &cq->cmp_task); 1148 1149 /* Clear interrupt */ 1150 nicvf_clear_intr(nic, NICVF_INTR_CQ, qidx); 1151 1152 return (FILTER_HANDLED); 1153 } 1154 1155 static void 1156 nicvf_rbdr_intr_handler(void *arg) 1157 { 1158 struct nicvf *nic; 1159 struct queue_set *qs; 1160 struct rbdr *rbdr; 1161 int qidx; 1162 1163 nic = (struct nicvf *)arg; 1164 1165 /* Disable RBDR interrupt and schedule softirq */ 1166 for (qidx = 0; qidx < nic->qs->rbdr_cnt; qidx++) { 1167 if (!nicvf_is_intr_enabled(nic, NICVF_INTR_RBDR, qidx)) 1168 continue; 1169 nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx); 1170 1171 qs = nic->qs; 1172 rbdr = &qs->rbdr[qidx]; 1173 taskqueue_enqueue(rbdr->rbdr_taskq, &rbdr->rbdr_task_nowait); 1174 /* Clear interrupt */ 1175 nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx); 1176 } 1177 } 1178 1179 static void 1180 nicvf_qs_err_intr_handler(void *arg) 1181 { 1182 struct nicvf *nic = (struct nicvf *)arg; 1183 struct queue_set *qs = nic->qs; 1184 1185 /* Disable Qset err interrupt and schedule softirq */ 1186 nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0); 1187 taskqueue_enqueue(qs->qs_err_taskq, &qs->qs_err_task); 1188 nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0); 1189 1190 } 1191 1192 static int 1193 nicvf_enable_msix(struct nicvf *nic) 1194 { 1195 struct pci_devinfo *dinfo; 1196 int rid, count; 1197 int ret; 1198 1199 dinfo = device_get_ivars(nic->dev); 1200 rid = dinfo->cfg.msix.msix_table_bar; 1201 nic->msix_table_res = 1202 bus_alloc_resource_any(nic->dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); 1203 if (nic->msix_table_res == NULL) { 1204 device_printf(nic->dev, 1205 "Could not allocate memory for MSI-X table\n"); 1206 return (ENXIO); 1207 } 1208 1209 count = nic->num_vec = NIC_VF_MSIX_VECTORS; 1210 1211 ret = pci_alloc_msix(nic->dev, &count); 1212 if ((ret != 0) || (count != nic->num_vec)) { 1213 device_printf(nic->dev, 1214 "Request for #%d msix vectors failed, error: %d\n", 1215 nic->num_vec, ret); 1216 return (ret); 1217 } 1218 1219 nic->msix_enabled = 1; 1220 return (0); 1221 } 1222 1223 static void 1224 nicvf_disable_msix(struct nicvf *nic) 1225 { 1226 1227 if (nic->msix_enabled) { 1228 pci_release_msi(nic->dev); 1229 nic->msix_enabled = 0; 1230 nic->num_vec = 0; 1231 } 1232 } 1233 1234 static void 1235 nicvf_release_all_interrupts(struct nicvf *nic) 1236 { 1237 struct resource *res; 1238 int irq; 1239 int err __diagused; 1240 1241 /* Free registered interrupts */ 1242 for (irq = 0; irq < nic->num_vec; irq++) { 1243 res = nic->msix_entries[irq].irq_res; 1244 if (res == NULL) 1245 continue; 1246 /* Teardown interrupt first */ 1247 if (nic->msix_entries[irq].handle != NULL) { 1248 err = bus_teardown_intr(nic->dev, 1249 nic->msix_entries[irq].irq_res, 1250 nic->msix_entries[irq].handle); 1251 KASSERT(err == 0, 1252 ("ERROR: Unable to teardown interrupt %d", irq)); 1253 nic->msix_entries[irq].handle = NULL; 1254 } 1255 1256 bus_release_resource(nic->dev, SYS_RES_IRQ, 1257 rman_get_rid(res), nic->msix_entries[irq].irq_res); 1258 nic->msix_entries[irq].irq_res = NULL; 1259 } 1260 /* Disable MSI-X */ 1261 nicvf_disable_msix(nic); 1262 } 1263 1264 /* 1265 * Initialize MSIX vectors and register MISC interrupt. 1266 * Send READY message to PF to check if its alive 1267 */ 1268 static int 1269 nicvf_allocate_misc_interrupt(struct nicvf *nic) 1270 { 1271 struct resource *res; 1272 int irq, rid; 1273 int ret = 0; 1274 1275 /* Return if mailbox interrupt is already registered */ 1276 if (nic->msix_enabled) 1277 return (0); 1278 1279 /* Enable MSI-X */ 1280 if (nicvf_enable_msix(nic) != 0) 1281 return (ENXIO); 1282 1283 irq = NICVF_INTR_ID_MISC; 1284 rid = irq + 1; 1285 nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev, 1286 SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE)); 1287 if (nic->msix_entries[irq].irq_res == NULL) { 1288 device_printf(nic->dev, 1289 "Could not allocate Mbox interrupt for VF%d\n", 1290 device_get_unit(nic->dev)); 1291 return (ENXIO); 1292 } 1293 1294 ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res, 1295 (INTR_MPSAFE | INTR_TYPE_MISC), NULL, nicvf_misc_intr_handler, nic, 1296 &nic->msix_entries[irq].handle); 1297 if (ret != 0) { 1298 res = nic->msix_entries[irq].irq_res; 1299 bus_release_resource(nic->dev, SYS_RES_IRQ, 1300 rman_get_rid(res), res); 1301 nic->msix_entries[irq].irq_res = NULL; 1302 return (ret); 1303 } 1304 1305 return (0); 1306 } 1307 1308 static int 1309 nicvf_enable_misc_interrupt(struct nicvf *nic) 1310 { 1311 1312 /* Enable mailbox interrupt */ 1313 nicvf_enable_intr(nic, NICVF_INTR_MBOX, 0); 1314 1315 /* Check if VF is able to communicate with PF */ 1316 if (!nicvf_check_pf_ready(nic)) { 1317 nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0); 1318 return (ENXIO); 1319 } 1320 1321 return (0); 1322 } 1323 1324 static void 1325 nicvf_release_net_interrupts(struct nicvf *nic) 1326 { 1327 struct resource *res; 1328 int irq; 1329 int err; 1330 1331 for_each_cq_irq(irq) { 1332 res = nic->msix_entries[irq].irq_res; 1333 if (res == NULL) 1334 continue; 1335 /* Teardown active interrupts first */ 1336 if (nic->msix_entries[irq].handle != NULL) { 1337 err = bus_teardown_intr(nic->dev, 1338 nic->msix_entries[irq].irq_res, 1339 nic->msix_entries[irq].handle); 1340 KASSERT(err == 0, 1341 ("ERROR: Unable to teardown CQ interrupt %d", 1342 (irq - NICVF_INTR_ID_CQ))); 1343 if (err != 0) 1344 continue; 1345 } 1346 1347 /* Release resource */ 1348 bus_release_resource(nic->dev, SYS_RES_IRQ, rman_get_rid(res), 1349 res); 1350 nic->msix_entries[irq].irq_res = NULL; 1351 } 1352 1353 for_each_rbdr_irq(irq) { 1354 res = nic->msix_entries[irq].irq_res; 1355 if (res == NULL) 1356 continue; 1357 /* Teardown active interrupts first */ 1358 if (nic->msix_entries[irq].handle != NULL) { 1359 err = bus_teardown_intr(nic->dev, 1360 nic->msix_entries[irq].irq_res, 1361 nic->msix_entries[irq].handle); 1362 KASSERT(err == 0, 1363 ("ERROR: Unable to teardown RDBR interrupt %d", 1364 (irq - NICVF_INTR_ID_RBDR))); 1365 if (err != 0) 1366 continue; 1367 } 1368 1369 /* Release resource */ 1370 bus_release_resource(nic->dev, SYS_RES_IRQ, rman_get_rid(res), 1371 res); 1372 nic->msix_entries[irq].irq_res = NULL; 1373 } 1374 1375 irq = NICVF_INTR_ID_QS_ERR; 1376 res = nic->msix_entries[irq].irq_res; 1377 if (res != NULL) { 1378 /* Teardown active interrupts first */ 1379 if (nic->msix_entries[irq].handle != NULL) { 1380 err = bus_teardown_intr(nic->dev, 1381 nic->msix_entries[irq].irq_res, 1382 nic->msix_entries[irq].handle); 1383 KASSERT(err == 0, 1384 ("ERROR: Unable to teardown QS Error interrupt %d", 1385 irq)); 1386 if (err != 0) 1387 return; 1388 } 1389 1390 /* Release resource */ 1391 bus_release_resource(nic->dev, SYS_RES_IRQ, rman_get_rid(res), 1392 res); 1393 nic->msix_entries[irq].irq_res = NULL; 1394 } 1395 } 1396 1397 static int 1398 nicvf_allocate_net_interrupts(struct nicvf *nic) 1399 { 1400 u_int cpuid; 1401 int irq, rid; 1402 int qidx; 1403 int ret = 0; 1404 1405 /* MSI-X must be configured by now */ 1406 if (!nic->msix_enabled) { 1407 device_printf(nic->dev, "Cannot alloacte queue interrupts. " 1408 "MSI-X interrupts disabled.\n"); 1409 return (ENXIO); 1410 } 1411 1412 /* Register CQ interrupts */ 1413 for_each_cq_irq(irq) { 1414 if (irq >= (NICVF_INTR_ID_CQ + nic->qs->cq_cnt)) 1415 break; 1416 1417 qidx = irq - NICVF_INTR_ID_CQ; 1418 rid = irq + 1; 1419 nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev, 1420 SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE)); 1421 if (nic->msix_entries[irq].irq_res == NULL) { 1422 device_printf(nic->dev, 1423 "Could not allocate CQ interrupt %d for VF%d\n", 1424 (irq - NICVF_INTR_ID_CQ), device_get_unit(nic->dev)); 1425 ret = ENXIO; 1426 goto error; 1427 } 1428 ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res, 1429 (INTR_MPSAFE | INTR_TYPE_NET), nicvf_intr_handler, 1430 NULL, &nic->qs->cq[qidx], &nic->msix_entries[irq].handle); 1431 if (ret != 0) { 1432 device_printf(nic->dev, 1433 "Could not setup CQ interrupt %d for VF%d\n", 1434 (irq - NICVF_INTR_ID_CQ), device_get_unit(nic->dev)); 1435 goto error; 1436 } 1437 cpuid = (device_get_unit(nic->dev) * CMP_QUEUE_CNT) + qidx; 1438 cpuid %= mp_ncpus; 1439 /* 1440 * Save CPU ID for later use when system-wide RSS is enabled. 1441 * It will be used to pit the CQ task to the same CPU that got 1442 * interrupted. 1443 */ 1444 nic->qs->cq[qidx].cmp_cpuid = cpuid; 1445 if (bootverbose) { 1446 device_printf(nic->dev, "bind CQ%d IRQ to CPU%d\n", 1447 qidx, cpuid); 1448 } 1449 /* Bind interrupts to the given CPU */ 1450 bus_bind_intr(nic->dev, nic->msix_entries[irq].irq_res, cpuid); 1451 } 1452 1453 /* Register RBDR interrupt */ 1454 for_each_rbdr_irq(irq) { 1455 if (irq >= (NICVF_INTR_ID_RBDR + nic->qs->rbdr_cnt)) 1456 break; 1457 1458 rid = irq + 1; 1459 nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev, 1460 SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE)); 1461 if (nic->msix_entries[irq].irq_res == NULL) { 1462 device_printf(nic->dev, 1463 "Could not allocate RBDR interrupt %d for VF%d\n", 1464 (irq - NICVF_INTR_ID_RBDR), 1465 device_get_unit(nic->dev)); 1466 ret = ENXIO; 1467 goto error; 1468 } 1469 ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res, 1470 (INTR_MPSAFE | INTR_TYPE_NET), NULL, 1471 nicvf_rbdr_intr_handler, nic, 1472 &nic->msix_entries[irq].handle); 1473 if (ret != 0) { 1474 device_printf(nic->dev, 1475 "Could not setup RBDR interrupt %d for VF%d\n", 1476 (irq - NICVF_INTR_ID_RBDR), 1477 device_get_unit(nic->dev)); 1478 goto error; 1479 } 1480 } 1481 1482 /* Register QS error interrupt */ 1483 irq = NICVF_INTR_ID_QS_ERR; 1484 rid = irq + 1; 1485 nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev, 1486 SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE)); 1487 if (nic->msix_entries[irq].irq_res == NULL) { 1488 device_printf(nic->dev, 1489 "Could not allocate QS Error interrupt for VF%d\n", 1490 device_get_unit(nic->dev)); 1491 ret = ENXIO; 1492 goto error; 1493 } 1494 ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res, 1495 (INTR_MPSAFE | INTR_TYPE_NET), NULL, nicvf_qs_err_intr_handler, 1496 nic, &nic->msix_entries[irq].handle); 1497 if (ret != 0) { 1498 device_printf(nic->dev, 1499 "Could not setup QS Error interrupt for VF%d\n", 1500 device_get_unit(nic->dev)); 1501 goto error; 1502 } 1503 1504 return (0); 1505 error: 1506 nicvf_release_net_interrupts(nic); 1507 return (ret); 1508 } 1509 1510 static int 1511 nicvf_stop_locked(struct nicvf *nic) 1512 { 1513 if_t ifp; 1514 int qidx; 1515 struct queue_set *qs = nic->qs; 1516 union nic_mbx mbx = {}; 1517 1518 NICVF_CORE_LOCK_ASSERT(nic); 1519 /* Stop callout. Can block here since holding SX lock */ 1520 callout_drain(&nic->stats_callout); 1521 1522 ifp = nic->ifp; 1523 1524 mbx.msg.msg = NIC_MBOX_MSG_SHUTDOWN; 1525 nicvf_send_msg_to_pf(nic, &mbx); 1526 1527 /* Disable RBDR & QS error interrupts */ 1528 for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) { 1529 nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx); 1530 nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx); 1531 } 1532 nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0); 1533 nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0); 1534 1535 /* Deactivate network interface */ 1536 if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); 1537 1538 /* Free resources */ 1539 nicvf_config_data_transfer(nic, FALSE); 1540 1541 /* Disable HW Qset */ 1542 nicvf_qset_config(nic, FALSE); 1543 1544 /* disable mailbox interrupt */ 1545 nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0); 1546 1547 return (0); 1548 } 1549 1550 static void 1551 nicvf_update_stats(struct nicvf *nic) 1552 { 1553 int qidx; 1554 struct nicvf_hw_stats *stats = &nic->hw_stats; 1555 struct nicvf_drv_stats *drv_stats = &nic->drv_stats; 1556 struct queue_set *qs = nic->qs; 1557 1558 #define GET_RX_STATS(reg) \ 1559 nicvf_reg_read(nic, NIC_VNIC_RX_STAT_0_13 | ((reg) << 3)) 1560 #define GET_TX_STATS(reg) \ 1561 nicvf_reg_read(nic, NIC_VNIC_TX_STAT_0_4 | ((reg) << 3)) 1562 1563 stats->rx_bytes = GET_RX_STATS(RX_OCTS); 1564 stats->rx_ucast_frames = GET_RX_STATS(RX_UCAST); 1565 stats->rx_bcast_frames = GET_RX_STATS(RX_BCAST); 1566 stats->rx_mcast_frames = GET_RX_STATS(RX_MCAST); 1567 stats->rx_fcs_errors = GET_RX_STATS(RX_FCS); 1568 stats->rx_l2_errors = GET_RX_STATS(RX_L2ERR); 1569 stats->rx_drop_red = GET_RX_STATS(RX_RED); 1570 stats->rx_drop_red_bytes = GET_RX_STATS(RX_RED_OCTS); 1571 stats->rx_drop_overrun = GET_RX_STATS(RX_ORUN); 1572 stats->rx_drop_overrun_bytes = GET_RX_STATS(RX_ORUN_OCTS); 1573 stats->rx_drop_bcast = GET_RX_STATS(RX_DRP_BCAST); 1574 stats->rx_drop_mcast = GET_RX_STATS(RX_DRP_MCAST); 1575 stats->rx_drop_l3_bcast = GET_RX_STATS(RX_DRP_L3BCAST); 1576 stats->rx_drop_l3_mcast = GET_RX_STATS(RX_DRP_L3MCAST); 1577 1578 stats->tx_bytes_ok = GET_TX_STATS(TX_OCTS); 1579 stats->tx_ucast_frames_ok = GET_TX_STATS(TX_UCAST); 1580 stats->tx_bcast_frames_ok = GET_TX_STATS(TX_BCAST); 1581 stats->tx_mcast_frames_ok = GET_TX_STATS(TX_MCAST); 1582 stats->tx_drops = GET_TX_STATS(TX_DROP); 1583 1584 drv_stats->tx_frames_ok = stats->tx_ucast_frames_ok + 1585 stats->tx_bcast_frames_ok + stats->tx_mcast_frames_ok; 1586 drv_stats->rx_drops = stats->rx_drop_red + stats->rx_drop_overrun; 1587 drv_stats->tx_drops = stats->tx_drops; 1588 1589 /* Update RQ and SQ stats */ 1590 for (qidx = 0; qidx < qs->rq_cnt; qidx++) 1591 nicvf_update_rq_stats(nic, qidx); 1592 for (qidx = 0; qidx < qs->sq_cnt; qidx++) 1593 nicvf_update_sq_stats(nic, qidx); 1594 } 1595 1596 static void 1597 nicvf_tick_stats(void *arg) 1598 { 1599 struct nicvf *nic; 1600 1601 nic = (struct nicvf *)arg; 1602 1603 /* Read the statistics */ 1604 nicvf_update_stats(nic); 1605 1606 callout_reset(&nic->stats_callout, hz, nicvf_tick_stats, nic); 1607 } 1608